@gentbajko/slopify 0.8.1 → 0.8.2

package/dist/adapters/alignment/index.js

@@ -22,7 +22,7 @@ export const alignSubtitles = async (request) => {
             const pcmPath = join(working, "audio.f32");
             await decodeAudio(request.ffmpeg, request.audioPath, pcmPath, request.signal);
             request.onProgress?.(25, 100);
-            return await runAlignmentWorker({ modelPath, pcmPath, text: request.text }, request.signal, (current, total) => request.onProgress?.(25 + Math.round((current / total) * 75), 100));
+            return await runAlignmentWorker({ modelPath, pcmPath, text: request.text }, request.signal, (current, total) => request.onProgress?.(25 + Math.round((current / total) * 75), 100), undefined, request.onOmission);
         }
         finally {
             await rm(working, { recursive: true, force: true });

package/dist/adapters/alignment/protocol.js

@@ -4,6 +4,10 @@ export const workerInput = z.object({
     pcmPath: z.string(),
     text: z.string(),
 });
+export const omissionSchema = z.object({
+    start: z.number().finite().nonnegative(),
+    text: z.string().min(1).max(10000),
+});
 export const workerMessage = z.discriminatedUnion("type", [
     z.object({
         type: z.literal("progress"),
@@ -19,5 +23,6 @@ export const workerMessage = z.discriminatedUnion("type", [
             confidence: z.number().finite().min(0).max(1).optional(),
         })),
     }),
+    omissionSchema.extend({ type: z.literal("omission") }),
     z.object({ type: z.literal("error"), message: z.string() }),
 ]);

package/dist/adapters/alignment/quality.js

@@ -1,6 +1,6 @@
 // Compare acoustic recognition with the already aligned transcript. This catches extra
 // speech and missing phrases even when a few forced words individually score well.
-export function agreesWithSpeech(expected, observed) {
+export function agreesWithSpeech(expected, observed, maximumError = 0.42) {
     const left = expected.toUpperCase().replace(/[^A-Z']/g, "");
     const right = observed.toUpperCase().replace(/[^A-Z']/g, "");
     if (left.length === 0 || right.length === 0)
@@ -13,5 +13,5 @@ export function agreesWithSpeech(expected, observed) {
             current[column] = Math.min((previous[column] ?? 0) + 1, (current[column - 1] ?? 0) + 1, (previous[column - 1] ?? 0) + (left[row - 1] === right[column - 1] ? 0 : 1));
         previous = current;
     }
-    return (previous[right.length] ?? Infinity) / Math.max(left.length, right.length) <= 0.42;
+    return (previous[right.length] ?? Infinity) / Math.max(left.length, right.length) <= maximumError;
 }

package/dist/adapters/alignment/runner.js

@@ -1,6 +1,6 @@
 import { fork } from "node:child_process";
 import { workerMessage } from "./protocol.js";
-export async function runAlignmentWorker(input, signal, onProgress, worker = new URL("./worker.js", import.meta.url)) {
+export async function runAlignmentWorker(input, signal, onProgress, worker = new URL("./worker.js", import.meta.url), onOmission) {
     signal.throwIfAborted();
     return new Promise((resolve, reject) => {
         const options = {
@@ -48,7 +48,15 @@ export async function runAlignmentWorker(input, signal, onProgress, worker = new
                 return;
             }
             const message = parsed.data;
-            if (message.type === "progress") {
+            if (message.type === "omission") {
+                try {
+                    onOmission?.({ start: message.start, text: message.text });
+                }
+                catch (error) {
+                    finish(error instanceof Error ? error : new Error(String(error)));
+                }
+            }
+            else if (message.type === "progress") {
                 try {
                     onProgress?.(message.current, message.total);
                 }

package/dist/adapters/alignment/window.js

@@ -0,0 +1,60 @@
+import { alignWindow, frameSeconds, mismatch } from "./ctc.js";
+import { agreesWithSpeech } from "./quality.js";
+import { letters } from "./vocabulary.js";
+// Recovery can only omit a short transcript prefix, never insert guessed words or times.
+export function alignSpeechWindow(logits, frames, candidate, complete, cutoff, skipBudget) {
+    try {
+        return { words: accepted(logits, frames, candidate, complete, cutoff), skipped: 0 };
+    }
+    catch (error) {
+        if (!(error instanceof Error) || error.message !== mismatch)
+            throw error;
+    }
+    const heard = greedy(logits, frames).split(/\s+/);
+    for (let skipped = 1; skipped <= Math.min(40, skipBudget, candidate.length - 4); skipped += 1) {
+        const remaining = candidate.slice(skipped);
+        const anchor = remaining
+            .slice(0, 4)
+            .map((word) => word.spoken)
+            .join(" ");
+        if (anchor.replace(/[^A-Z]/g, "").length < 20 ||
+            !agreesWithSpeech(anchor, heard.slice(0, anchor.split(/\s+/).length).join(" "), 0.2))
+            continue;
+        try {
+            const words = accepted(logits, frames, remaining, complete, cutoff);
+            if (words.length < 4 || words.slice(0, 4).some((word) => (word.confidence ?? 0) < 0.75))
+                continue;
+            return { words, skipped };
+        }
+        catch (error) {
+            if (!(error instanceof Error) || error.message !== mismatch)
+                throw error;
+        }
+    }
+    throw new Error(mismatch);
+}
+function accepted(logits, frames, candidate, complete, cutoff) {
+    const words = alignWindow(logits, frames, candidate, complete).words.filter((word) => word.end <= cutoff);
+    const last = words.at(-1);
+    if (last === undefined ||
+        !agreesWithSpeech(candidate
+            .slice(0, words.length)
+            .map((word) => word.spoken)
+            .join(" "), greedy(logits, Math.min(frames, Math.ceil(last.end / frameSeconds)))))
+        throw new Error(mismatch);
+    return words;
+}
+export function greedy(logits, frames) {
+    let previous = -1;
+    let text = "";
+    for (let frame = 0; frame < frames; frame += 1) {
+        let best = 0;
+        for (let label = 1; label < 32; label += 1)
+            if ((logits[frame * 32 + label] ?? -Infinity) > (logits[frame * 32 + best] ?? -Infinity))
+                best = label;
+        if (best !== previous && best !== 0)
+            text += letters[best] ?? "";
+        previous = best;
+    }
+    return text.trim().replace(/\s+/g, " ");
+}

package/dist/adapters/alignment/worker.js

@@ -1,10 +1,9 @@
 import { open, readFile, stat } from "node:fs/promises";
 import * as ort from "onnxruntime-web/wasm";
-import { alignWindow, frameSeconds, mismatch } from "./ctc.js";
+import { mismatch } from "./ctc.js";
 import { workerInput } from "./protocol.js";
-import { agreesWithSpeech } from "./quality.js";
 import { speechWords } from "./text.js";
-import { letters } from "./vocabulary.js";
+import { alignSpeechWindow, greedy } from "./window.js";
 const sampleRate = 16000;
 const windowSeconds = 12;
 const overlapSeconds = 2;
@@ -31,6 +30,8 @@ async function run(input) {
         const source = speechWords(input.text);
         const output = [];
         let cursor = 0;
+        let omitted = 0;
+        const omissionBudget = Math.min(60, Math.floor(source.length * 0.05));
         let sampleAt = 0;
         while (sampleAt < totalSamples && cursor < source.length) {
             const count = Math.min(windowSeconds * sampleRate, totalSamples - sampleAt);
@@ -62,18 +63,24 @@ async function run(input) {
             const candidate = candidates(source, cursor, observed);
             const finalWindow = sampleAt + count >= totalSamples;
             const complete = finalWindow && cursor + candidate.length === source.length;
-            const aligned = alignWindow(logits.data, frames, candidate, complete).words;
             const cutoff = finalWindow ? count / sampleRate : count / sampleRate - overlapSeconds;
-            const accepted = aligned.filter((word) => word.end <= cutoff);
+            const recovered = alignSpeechWindow(logits.data, frames, candidate, complete, cutoff, cursor === 0 ? 0 : omissionBudget - omitted);
+            const accepted = recovered.words;
             const last = accepted.at(-1);
             if (last === undefined)
                 throw new Error(mismatch);
-            const heard = greedy(logits.data, Math.min(frames, Math.ceil(last.end / frameSeconds)));
-            if (!agreesWithSpeech(candidate
-                .slice(0, accepted.length)
-                .map((word) => word.spoken)
-                .join(" "), heard))
-                throw new Error(mismatch);
+            if (recovered.skipped > 0) {
+                omitted += recovered.skipped;
+                send({
+                    type: "omission",
+                    start: sampleAt / sampleRate,
+                    text: candidate
+                        .slice(0, recovered.skipped)
+                        .map((word) => word.text)
+                        .join(" "),
+                });
+                cursor += recovered.skipped;
+            }
             const offset = sampleAt / sampleRate;
             output.push(...accepted.map((word) => ({
                 ...word,
@@ -147,17 +154,3 @@ function normalize(audio) {
     const divisor = Math.sqrt(variance + 1e-7);
     return Float32Array.from(audio, (value) => (value - mean) / divisor);
 }
-function greedy(logits, frames) {
-    let previous = -1;
-    let text = "";
-    for (let frame = 0; frame < frames; frame += 1) {
-        let best = 0;
-        for (let label = 1; label < 32; label += 1)
-            if ((logits[frame * 32 + label] ?? -Infinity) > (logits[frame * 32 + best] ?? -Infinity))
-                best = label;
-        if (best !== previous && best !== 0)
-            text += letters[best] ?? "";
-        previous = best;
-    }
-    return text.trim().replace(/\s+/g, " ");
-}

package/dist/slices/storage/repo.js

@@ -1,6 +1,9 @@
 import { z } from "zod";
 import { outputRoles, stagedFileStates, stageKinds } from "./model.js";
 const metaSchema = z.object({
+    subtitleOmissions: z
+        .array(z.object({ start: z.number().finite().nonnegative(), text: z.string() }))
+        .optional(),
     subtitlesMode: z.enum(["off", "files", "burn-in"]).optional(),
     promptName: z.string().optional(),
     prompt: z.string().optional(),

package/dist/slices/subtitles/prepare.js

@@ -28,7 +28,14 @@ const fontSchema = z.object({
     assName: z.string(),
     extension: z.enum([".ttf", ".otf", ".ttc"]),
 });
-const cacheSchema = z.object({ key: z.string(), words: z.array(wordSchema), font: fontSchema });
+const cacheSchema = z.object({
+    key: z.string(),
+    words: z.array(wordSchema),
+    font: fontSchema,
+    omissions: z
+        .array(z.object({ start: z.number().finite().nonnegative(), text: z.string() }))
+        .default([]),
+});
 // Preparation writes into a new directory; only writeExport commits its output rows.
 // The old caption files/font/timing stay usable if alignment or rendering fails.
 export async function prepareSubtitles(deps, context, audio, frame) {
@@ -52,7 +59,8 @@ export async function prepareSubtitles(deps, context, audio, frame) {
     const directory = mkdtempSync(join(dir, "captions-"));
     try {
         const font = await snapshotFont(deps, projectId, outputs, config.fontId, cache, directory);
-        const words = cache?.key === key ? cache.words : await alignSegments(deps, context, segments);
+        const omissions = cache?.key === key ? [...cache.omissions] : [];
+        const words = cache?.key === key ? cache.words : await alignSegments(deps, context, segments, omissions);
         context.signal.throwIfAborted();
         const cues = captionCues(words);
         if (cues.length === 0)
@@ -65,7 +73,7 @@ export async function prepareSubtitles(deps, context, audio, frame) {
             fontName: font.assName,
             position: config.position,
         }), { mode: 0o600 });
-        writeFileSync(join(directory, "subtitles.json"), JSON.stringify({ key, words, font }), {
+        writeFileSync(join(directory, "subtitles.json"), JSON.stringify({ key, words, font, omissions }), {
             mode: 0o600,
         });
         const files = [
@@ -77,6 +85,7 @@ export async function prepareSubtitles(deps, context, audio, frame) {
         ];
         return {
             directory,
+            omissions,
             burnIn: config.mode === "burn-in" && project.config.sources.video !== "off",
             assets: files.map(([role, path]) => ({
                 role,
@@ -91,7 +100,7 @@ export async function prepareSubtitles(deps, context, audio, frame) {
 }
 async function timingKey(segments, signal) {
     // Bump when alignment normalization/model changes. Hash file contents, not timestamps.
-    const hash = createHash("sha256").update("wav2vec2-en-a19f851-v1");
+    const hash = createHash("sha256").update("wav2vec2-en-a19f851-v2-omissions");
     for (const segment of segments) {
         signal.throwIfAborted();
         hash.update(JSON.stringify({ kind: segment.kind, seconds: segment.seconds, text: segment.text }));
@@ -103,7 +112,7 @@ async function timingKey(segments, signal) {
     }
     return hash.digest("hex");
 }
-async function alignSegments(deps, context, segments) {
+async function alignSegments(deps, context, segments, omissions) {
     if (deps.alignSubtitles === undefined)
         throw new Error("Local subtitle alignment is unavailable in this build.");
     const words = [];
@@ -114,6 +123,7 @@ async function alignSegments(deps, context, segments) {
         if (segment.path !== null) {
             const aligned = await deps.alignSubtitles({
                 audioPath: segment.path,
+                onOmission: (omission) => omissions.push({ ...omission, start: omission.start + offset }),
                 text: segment.text,
                 cacheDir: join(deps.paths.dataDir, "models", "english-subtitles"),
                 ffmpeg: deps.ffmpeg,

package/dist/slices/video/write-export.js

@@ -90,6 +90,9 @@ export async function writeExport(deps, context, output) {
             }
             store(deps, projectId, "render_params", "render.json", null);
             store(deps, projectId, output.role, output.filename, totalMs, {
+                ...(output.subtitles?.omissions?.length
+                    ? { subtitleOmissions: output.subtitles.omissions }
+                    : {}),
                 subtitlesMode: output.subtitles === undefined ? "off" : output.subtitles.burnIn ? "burn-in" : "files",
             });
             for (const asset of output.subtitles?.assets ?? [])