@gentbajko/slopify 2.0.1 → 2.0.2

package/dist/adapters/alignment/cache.js

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, mkdtemp, rename, rm, stat } from "node:fs/promises";
+import { chmod, copyFile, mkdir, mkdtemp, rename, rm, stat } from "node:fs/promises";
 import { join } from "node:path";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
@@ -18,6 +18,8 @@ export async function prepareModel(input, deps = { model: alignmentModel, fetch:
     const target = join(input.cacheDir, deps.model.filename);
     if (await verified(target, deps.model, input.signal))
         return target;
+    if (input.seed !== undefined && (await verified(input.seed, deps.model, input.signal)))
+        return seedModel(input, input.seed, target);
     let lastFailure;
     for (let attempt = 1; attempt <= 3; attempt += 1) {
         try {
@@ -127,6 +129,20 @@ async function downloadModel(input, deps, target) {
         await rm(staging, { recursive: true, force: true });
     }
 }
+async function seedModel(input, seed, target) {
+    const staging = await mkdtemp(join(input.cacheDir, ".alignment-seed-"));
+    try {
+        const part = join(staging, "model.part");
+        await copyFile(seed, part);
+        await chmod(part, 0o600);
+        input.signal.throwIfAborted();
+        await rename(part, target);
+        return target;
+    }
+    finally {
+        await rm(staging, { recursive: true, force: true });
+    }
+}
 function interrupted(cause) {
     const error = new Error("The subtitle model download was interrupted.", { cause });
     error.name = "SubtitleModelDownloadInterrupted";

package/dist/adapters/alignment/prefetch.js

@@ -0,0 +1,35 @@
+import { mkdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { prepareModel } from "./cache.js";
+import { claimWorker } from "./lock.js";
+// Readies the subtitle model when the app starts, so the first captioned render never waits
+// on a 95 MB download in the middle of a run. It takes the worker lock, so a render that
+// starts meanwhile waits for this copy rather than fetching a second one.
+export async function prefetchModel(input, deps) {
+    await mkdir(input.cacheDir, { recursive: true, mode: 0o700 });
+    await clearOwnLock(input.cacheDir);
+    const release = await claimWorker(input.cacheDir, input.signal);
+    try {
+        return await prepareModel({ cacheDir: input.cacheDir, seed: input.seed, signal: input.signal }, deps);
+    }
+    finally {
+        await release();
+    }
+}
+// The lock names its holder by pid. A lock that names this process when the app has only
+// just started was left by an earlier run: in a container the app is always pid 1, so the
+// liveness check in claimWorker would take it for a live holder and wait forever.
+async function clearOwnLock(cacheDir) {
+    const path = join(cacheDir, "alignment-worker.lock");
+    let owner;
+    try {
+        owner = await readFile(path, "utf8");
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return;
+        throw error;
+    }
+    if (Number(owner) === process.pid)
+        await rm(path, { force: true });
+}

package/dist/edge/cli.js

@@ -80,7 +80,10 @@ try {
     const forwarded = await forwardManagedUpdate(config.dataDir, readVersion(), process.argv.slice(2));
     if (forwarded !== undefined)
         process.exit(forwarded);
-    const { paths, url, stop } = await boot(config);
+    const { paths, url, stop } = await boot(config, {
+        prefetchSubtitleModel: ["", "0", "false"].includes((process.env.SLOPIFY_NO_MODEL_PREFETCH ?? "").trim().toLowerCase()),
+        subtitleModelSeed: process.env.SLOPIFY_SUBTITLE_MODEL_SEED?.trim() || undefined,
+    });
     console.log(`Slopify is running at ${url}`);
     console.log(`Slopify data directory: ${paths.dataDir}`);
     console.log(`Database: ${paths.db}`);

package/dist/kernel/errors.js

@@ -0,0 +1,12 @@
+// A failure's own sentence and every cause it wraps, for the log. The page shows only the
+// sentence; the chain is the only place a "download interrupted" says the server closed the
+// stream.
+export function causedBy(error) {
+    const chain = [error instanceof Error ? error.message : String(error)];
+    let cause = error instanceof Error ? error.cause : undefined;
+    while (cause !== undefined && cause !== null && chain.length < 6) {
+        chain.push(cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause));
+        cause = cause instanceof Error ? cause.cause : undefined;
+    }
+    return chain.join(" <- ");
+}

package/dist/kernel/paths.js

@@ -19,3 +19,8 @@ export function ensureDirs(paths, options) {
         chmodSync(dir, options.mode);
     }
 }
+// The subtitle-timing model's cache. The app readies it at start and every captioned render
+// reads it, so both name it through here.
+export function subtitleModelDir(dataDir) {
+    return join(dataDir, "models", "english-subtitles");
+}

package/dist/kernel/runner/index.js

@@ -1,4 +1,6 @@
+import { causedBy } from "../errors.js";
 import { derive, deps as graph, satisfied } from "./graph.js";
+import { progressGate } from "./progress.js";
 const pauseReason = new Error("paused by user");
 export function createRunner(deps) {
     const inflight = new Map();
@@ -60,6 +62,12 @@ export function createRunner(deps) {
         emitStage(stage, state, reason);
     };
     async function execute(stage, controller) {
+        const tag = (event) => ({
+            ...event,
+            revisionId: stage.work.revisionId,
+            workId: stage.work.workId,
+        });
+        const progress = progressGate((event) => deps.emit(stage.projectId, tag(event)));
         try {
             const run = deps.runs[stage.kind];
             if (run === undefined) {
@@ -76,16 +84,18 @@ export function createRunner(deps) {
                     deps.stages.maySubmit(stage.work, pieceId),
                 signal: controller.signal,
                 emit: (event) => {
-                    deps.emit(stage.projectId, {
-                        ...event,
-                        revisionId: stage.work.revisionId,
-                        workId: stage.work.workId,
-                    });
+                    if (event.type === "stage.progress")
+                        progress.offer(event);
+                    else
+                        deps.emit(stage.projectId, tag(event));
                 },
             });
+            progress.flush();
+            progress.close();
             conclude(stage, result === "held" ? "pending" : "done", null);
         }
         catch (error) {
+            progress.close();
             if (controller.signal.aborted) {
                 // A late rejection from the aborted call is how a stage learns it was canceled, so
                 // it is not logged as a fault.
@@ -96,7 +106,7 @@ export function createRunner(deps) {
             deps.log.write("error", "stage.failed", {
                 projectId: stage.projectId,
                 stage: stage.kind,
-                detail: reasonOf(error),
+                detail: causedBy(error),
             });
             conclude(stage, "failed", reasonOf(error));
         }

package/dist/kernel/runner/progress.js

@@ -0,0 +1,56 @@
+// The most progress a single stage invocation reports per second. A stage may call emit on
+// every chunk it reads (a 95 MB download is ~6,000 of them) and each report writes the
+// database and reaches every open page; nobody can read a meter faster than this.
+export const progressIntervalMs = 500;
+// Repeats of the last value are dropped. The first report of a window goes out at once and
+// the newest one offered inside it goes out when the window ends, so a meter never sits on
+// a stale value longer than one interval.
+export function progressGate(send, intervalMs = progressIntervalMs) {
+    let last;
+    let held;
+    let timer;
+    let closed = false;
+    const same = (a, b) => a !== undefined && a.current === b.current && a.total === b.total;
+    const deliver = (event) => {
+        last = event;
+        send(event);
+        timer = setTimeout(() => {
+            timer = undefined;
+            const next = held;
+            held = undefined;
+            if (next !== undefined && !closed && !same(last, next))
+                deliver(next);
+        }, intervalMs);
+        timer.unref?.();
+    };
+    return {
+        offer: (event) => {
+            if (closed)
+                return;
+            if (timer !== undefined) {
+                held = event;
+                return;
+            }
+            if (!same(last, event))
+                deliver(event);
+        },
+        flush: () => {
+            if (timer !== undefined)
+                clearTimeout(timer);
+            timer = undefined;
+            const next = held;
+            held = undefined;
+            if (next !== undefined && !closed && !same(last, next)) {
+                last = next;
+                send(next);
+            }
+        },
+        close: () => {
+            closed = true;
+            held = undefined;
+            if (timer !== undefined)
+                clearTimeout(timer);
+            timer = undefined;
+        },
+    };
+}

package/dist/main.js

@@ -5,6 +5,7 @@ import { serve } from "@hono/node-server";
 import ffmpegStatic from "ffmpeg-static";
 import { buildRegistry } from "./adapter-registry.js";
 import { alignSubtitles } from "./adapters/alignment/index.js";
+import { prefetchModel } from "./adapters/alignment/prefetch.js";
 import { prepareFfmpeg } from "./adapters/ffmpeg.js";
 import { createHostCliClient } from "./adapters/host-cli/index.js";
 import { nodeRunCli } from "./adapters/llm/run-cli.js";
@@ -21,10 +22,11 @@ import { systemClock } from "./kernel/clock.js";
 import { openDb } from "./kernel/db/index.js";
 import { migrate } from "./kernel/db/migrate.js";
 import { transact } from "./kernel/db/tx.js";
+import { causedBy } from "./kernel/errors.js";
 import { ulidIds } from "./kernel/ids.js";
 import { acquireInstanceLock } from "./kernel/lock.js";
 import { openLog } from "./kernel/log.js";
-import { ensureDirs, layout } from "./kernel/paths.js";
+import { ensureDirs, layout, subtitleModelDir } from "./kernel/paths.js";
 import { sqliteAttempts } from "./kernel/runner/attempt-repo.js";
 import { createCheckpointAuthority, } from "./kernel/runner/checkpoint-authority.js";
 import { createRunner } from "./kernel/runner/index.js";
@@ -109,7 +111,7 @@ export function createScheduleTickLifecycle(deps) {
         },
     };
 }
-export async function boot(config) {
+export async function boot(config, options = {}) {
     const clock = systemClock;
     const ids = ulidIds;
     const paths = layout(config.dataDir);
@@ -364,6 +366,18 @@ export async function boot(config) {
         // Whatever last run left queued goes out at start. Nothing waits for
         // it, and an unreachable collector costs one refused socket.
         flusher.soon();
+        const modelPrefetch = new AbortController();
+        const prefetching = options.prefetchSubtitleModel
+            ? prefetchModel({
+                cacheDir: subtitleModelDir(paths.dataDir),
+                seed: options.subtitleModelSeed,
+                signal: modelPrefetch.signal,
+            }).then(() => log.write("info", "subtitle-model.ready"), (error) => {
+                // Not fatal: the first captioned render tries again and says why on its row.
+                if (!modelPrefetch.signal.aborted)
+                    log.write("warn", "subtitle-model.prefetch", { detail: causedBy(error) });
+            })
+            : Promise.resolve();
         const open = db;
         let stopping;
         let stopActivation = () => { };
@@ -376,6 +390,7 @@ export async function boot(config) {
                 const serverClose = beginServerClose(server);
                 stopActivation();
                 audioPreviews.close();
+                modelPrefetch.abort();
                 try {
                     // Admission closes before the listener. Requests already admitted keep the
                     // database until their response settles; anything racing shutdown receives 503.
@@ -394,6 +409,7 @@ export async function boot(config) {
                         // closes and fail to mark its batch delivered, which costs one re-send that
                         // the collector deduplicates by event id.
                         flusher.stop();
+                        await prefetching;
                         log.write("info", "shutdown");
                         open.close();
                         lock.release();

package/dist/slices/rebuild/runtime-store.js

@@ -94,12 +94,12 @@ export function invocationReady(deps, work) {
         });
     });
 }
-export function executionStandings(deps, projectId) {
+export function executionStandings(deps, projectId, derived) {
     const revisionId = currentRevisionId(deps.db, projectId);
     const view = revisionId === undefined ? undefined : getRevisionView(deps, projectId, revisionId);
     if (view === undefined)
         return [];
-    const work = executionStages(deps, projectId);
+    const work = derived ?? executionStages(deps, projectId);
     return stageKinds.map((kind) => {
         const source = view.revision.config.sources[kind];
         const group = work
@@ -136,20 +136,18 @@ export function executionStandings(deps, projectId) {
     });
 }
 export function projectStandings(deps, projectId) {
-    const standings = executionStandings(deps, projectId);
+    // One derivation serves every stage: each builds the whole execution plan.
+    const work = executionStages(deps, projectId);
+    const standings = executionStandings(deps, projectId, work);
     for (const stage of standings) {
-        const invocations = executionStages(deps, projectId).filter((work) => work.kind === stage.kind);
-        const current = invocations.reduce((total, work) => {
-            if (work.state === "done")
+        const invocations = work.filter((entry) => entry.kind === stage.kind);
+        const current = invocations.reduce((total, entry) => {
+            if (entry.state === "done")
                 return total + 1;
             const progress = deps.db
                 .prepare("SELECT progress_current,progress_total FROM revision_work WHERE id=?")
-                .get(work.work.workId);
-            if (typeof progress?.progress_current !== "number" ||
-                typeof progress.progress_total !== "number" ||
-                progress.progress_total <= 0)
-                return total;
-            return total + Math.max(0, Math.min(1, progress.progress_current / progress.progress_total));
+                .get(entry.work.workId);
+            return total + fraction(progress?.progress_current, progress?.progress_total);
         }, 0);
         const failed = invocations.find((work) => work.state === "failed");
         const failure = failed === undefined
@@ -163,6 +161,11 @@ export function projectStandings(deps, projectId) {
     }
     settleScheduleRunsForProject(deps.db, projectId, deps.clock.now().toISOString());
 }
+// Progress moves one number and nothing else. The stage row already counts this invocation's
+// last fraction, from the claim or the previous event, so it takes the difference instead
+// of re-deriving the project: that derivation rebuilds every stage's plan and ran to seconds
+// on a long article, which froze the process once per downloaded chunk. State changes still
+// go through projectStandings.
 export function recordWorkProgress(deps, event) {
     if (event.workId === undefined ||
         event.revisionId === undefined ||
@@ -170,8 +173,35 @@ export function recordWorkProgress(deps, event) {
         !Number.isFinite(event.total) ||
         event.total <= 0)
         return;
+    const before = deps.db
+        .prepare("SELECT progress_current,progress_total FROM revision_work WHERE id=? AND revision_id=? AND project_id=? AND kind=? AND state='running'")
+        .get(event.workId, event.revisionId, event.projectId, event.stage);
+    if (before === undefined)
+        return;
+    const current = Math.max(0, Math.min(event.current, event.total));
     deps.db
-        .prepare("UPDATE revision_work SET progress_current=?,progress_total=? WHERE id=? AND revision_id=? AND project_id=? AND kind=? AND state='running'")
-        .run(Math.max(0, Math.min(event.current, event.total)), event.total, event.workId, event.revisionId, event.projectId, event.stage);
-    projectStandings(deps, event.projectId);
+        .prepare("UPDATE revision_work SET progress_current=?,progress_total=? WHERE id=?")
+        .run(current, event.total, event.workId);
+    // Work an edit retired keeps running until it notices, but the stage row belongs to the
+    // current revision's work and must not move with it.
+    const head = currentRevisionId(deps.db, event.projectId);
+    if (head === undefined ||
+        deps.db
+            .prepare("SELECT 1 FROM revision_work_reservations WHERE work_id=? AND revision_id=? LIMIT 1")
+            .get(event.workId, head) === undefined)
+        return;
+    const delta = current / event.total - fraction(before.progress_current, before.progress_total);
+    if (delta === 0)
+        return;
+    const moved = deps.db
+        .prepare("UPDATE stages SET progress_current=MIN(progress_total,MAX(0,progress_current+?)) WHERE project_id=? AND kind=? AND progress_current IS NOT NULL AND progress_total IS NOT NULL")
+        .run(delta, event.projectId, event.stage);
+    // A row no derivation has counted yet has nothing to adjust: derive it once.
+    if (moved.changes === 0)
+        projectStandings(deps, event.projectId);
+}
+function fraction(current, total) {
+    if (typeof current !== "number" || typeof total !== "number" || total <= 0)
+        return 0;
+    return Math.max(0, Math.min(1, current / total));
 }

package/dist/slices/rebuild/runtime-subtitles.js

@@ -1,6 +1,6 @@
 import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { z } from "zod";
+import { subtitleModelDir } from "../../kernel/paths.js";
 import { resolveFont } from "../fonts/index.js";
 import { validateCues } from "../revisions/rules.js";
 import { discardPreparedAssets, writeAsset } from "../storage/assets.js";
@@ -59,7 +59,7 @@ async function timing(deps, context, piece, snapshot) {
             const aligned = await deps.alignSubtitles({
                 audioPath: segment.path,
                 text: revisionTranscript(deps, snapshot, segment.kind),
-                cacheDir: join(deps.paths.dataDir, "models", "english-subtitles"),
+                cacheDir: subtitleModelDir(deps.paths.dataDir),
                 ffmpeg: deps.ffmpeg,
                 signal: context.signal,
                 onOmission: (value) => omissions.push({ ...value, start: value.start + offset }),

package/dist/slices/subtitles/prepare.js

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { copyFileSync, createReadStream, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
 import { join, relative, sep } from "node:path";
 import { z } from "zod";
+import { subtitleModelDir } from "../../kernel/paths.js";
 import { projectById, setStageProgress } from "../admission/repo.js";
 import { resolveFont } from "../fonts/index.js";
 import { outputPath, projectDir } from "../storage/layout.js";
@@ -128,7 +129,7 @@ async function alignSegments(deps, context, segments, omissions) {
                 audioPath: segment.path,
                 onOmission: (omission) => omissions.push({ ...omission, start: omission.start + offset }),
                 text: segment.text,
-                cacheDir: join(deps.paths.dataDir, "models", "english-subtitles"),
+                cacheDir: subtitleModelDir(deps.paths.dataDir),
                 ffmpeg: deps.ffmpeg,
                 signal: context.signal,
                 onProgress: (current, maximum) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gentbajko/slopify",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "description": "Turn a prompt into a finished video, locally, with your own provider keys",
   "license": "MIT",
   "author": "Gent Bajko",