@gentbajko/slopify 0.8.4 → 1.0.0

package/dist/slices/rebuild/runtime-publication.js

@@ -0,0 +1,172 @@
+import { transact } from "../../kernel/db/tx.js";
+import { fingerprint } from "../../kernel/runner/work.js";
+import { publicationAuthority } from "../revisions/publication-rules.js";
+import { commitRevisionOutputs } from "../revisions/publish.js";
+import { insertAsset, insertManifestOutput, revisionById } from "../revisions/repo.js";
+import { discardPreparedAssets, writeAsset } from "../storage/assets.js";
+import { executionPlan, executionView, savedCatalogue } from "./runtime-plan.js";
+export function preparedResult(deps, context, piece, role, asset, durationMs = null, meta = {}) {
+    return {
+        slot: role === "image" ? piece.key : `${context.work.kind}:${role}`,
+        workKey: piece.key,
+        fingerprint: piece.logicalFingerprint ?? piece.fingerprint,
+        asset,
+        output: {
+            id: deps.ids.next(),
+            projectId: context.work.projectId,
+            stageKind: context.work.kind,
+            role,
+            path: asset.path,
+            originalFilename: null,
+            bytes: asset.bytes,
+            durationMs,
+            meta,
+            createdAt: asset.createdAt,
+        },
+    };
+}
+export function preparedText(deps, context, piece, role, filename, text) {
+    const output = preparedTexts(deps, context, piece, [[role, filename, text]])[0];
+    if (output === undefined)
+        throw new Error("The text output was not prepared.");
+    return output;
+}
+export function preparedTexts(deps, context, piece, texts) {
+    const assets = [];
+    try {
+        return texts.map(([role, filename, text]) => {
+            const asset = writeAsset(deps, context.work.projectId, filename, Buffer.from(text));
+            assets.push(asset);
+            return preparedResult(deps, context, piece, role, asset);
+        });
+    }
+    catch (error) {
+        discardPreparedAssets(deps, assets);
+        throw error;
+    }
+}
+export async function publishResult(deps, context, piece, outputs, payload, asset = null) {
+    try {
+        const record = {
+            key: piece.key,
+            stageKind: context.work.kind,
+            asset,
+            fingerprint: piece.fingerprint,
+            piece: {
+                id: piece.id,
+                stageId: context.work.stageId,
+                kind: pieceKind(piece),
+                idx: pieceIndex(deps, context, piece),
+                state: "done",
+                payload: JSON.stringify({
+                    ...payload,
+                    requestFingerprint: piece.requestFingerprint,
+                    ...(asset === null ? {} : { file: asset.path }),
+                }),
+            },
+        };
+        const result = await commitRevisionOutputs(deps, { work: context.work, pieceId: piece.id, publicationId: piece.id }, outputs, [record]);
+        deps.db
+            .prepare("UPDATE revision_work_pieces SET state='done' WHERE id=? AND work_id=?")
+            .run(piece.id, context.work.workId);
+        if (result.currentAttached)
+            context.emit({ type: "project.updated", projectId: context.work.projectId });
+    }
+    finally {
+        discardPreparedAssets(deps, [
+            ...outputs.map((output) => output.asset),
+            ...(asset === null ? [] : [asset]),
+        ]);
+    }
+}
+function pieceKind(piece) {
+    if (piece.input.kind === "tts")
+        return piece.input.segment === "body" ? "chunk" : "segment";
+    if (piece.input.kind === "provided" && /^audio:body:.+:\d+$/.test(piece.key))
+        return "chunk";
+    if (piece.input.kind === "provided" && /^audio:(intro|outro):\d+$/.test(piece.key))
+        return "segment";
+    if (piece.key.startsWith("image:"))
+        return "image";
+    if (piece.key.startsWith("research:chapter:"))
+        return "chapter";
+    if (piece.key.startsWith("entry:"))
+        return "segment";
+    if (piece.key === "research:planner" || piece.key === "thumbnail:prompt")
+        return "prompt_written";
+    return "article_written";
+}
+function pieceIndex(deps, context, piece) {
+    // Intro and outro can resolve independently. Reserve alternating ordinals so a
+    // late intro part cannot collide with an outro that was already published.
+    const segment = /^audio:(intro|outro):(\d+)$/.exec(piece.key);
+    if (segment !== null)
+        return Number(segment[2]) * 2 - (segment[1] === "intro" ? 1 : 0);
+    if (piece.key === "entry:intro:text")
+        return 1;
+    if (piece.key === "entry:outro:text")
+        return 2;
+    const localOrder = [
+        "audio:body:concat",
+        "audio:intro",
+        "audio:outro",
+        "audio:provided",
+        "subtitles:timing",
+        "subtitles:cues",
+        "subtitles:files",
+        "export:wav",
+        "export:video",
+    ];
+    const localIndex = localOrder.indexOf(piece.key);
+    if (localIndex >= 0)
+        return localIndex + 1;
+    const view = executionView(deps, context.work.projectId, context.work.revisionId);
+    const row = deps.db
+        .prepare("SELECT recipe_context FROM revision_work WHERE id=?")
+        .get(context.work.workId);
+    if (view !== undefined && row?.recipe_context !== null && row?.recipe_context !== undefined) {
+        const keys = executionPlan(deps, view, savedCatalogue(row.recipe_context)).recipes.filter((recipe) => recipe.stage === context.work.kind &&
+            pieceKind({ ...piece, key: recipe.key, input: recipe.input }) === pieceKind(piece));
+        const index = keys.findIndex((recipe) => recipe.key === piece.key);
+        if (index >= 0)
+            return index + 1;
+    }
+    const last = piece.key.split(":").at(-1);
+    return last !== undefined && /^\d+$/.test(last) ? Number(last) : 1;
+}
+export function retainPartialArticle(deps, context, piece, text) {
+    const publicationId = fingerprint(["partial-article", piece.id]);
+    if (deps.db
+        .prepare("SELECT 1 FROM revision_outputs WHERE revision_id=? AND publication_id=?")
+        .get(context.work.revisionId, publicationId) !== undefined)
+        return;
+    const result = preparedText(deps, context, piece, "article_md", "partial-article.md", text);
+    try {
+        transact(deps.db, () => {
+            const authority = publicationAuthority(deps.db, {
+                work: context.work,
+                pieceId: piece.id,
+                publicationId: piece.id,
+            });
+            if (authority === undefined)
+                return;
+            if (authority.key !== piece.key || authority.fingerprint !== piece.fingerprint)
+                throw new Error("Partial article differs from its durable piece authority.");
+            const revision = revisionById(deps.db, context.work.projectId, context.work.revisionId);
+            if (revision === undefined)
+                throw new Error("Partial article origin disappeared.");
+            insertAsset(deps.db, result.asset);
+            insertManifestOutput(deps.db, revision, {
+                slot: `article:partial:${piece.id}`,
+                workKey: `article:partial:${piece.id}`,
+                assetId: result.asset.id,
+                output: { ...result.output, originalFilename: "partial-article.md" },
+                fingerprint: piece.fingerprint,
+                state: "outdated",
+            }, deps.ids.next(), publicationId);
+        });
+    }
+    finally {
+        discardPreparedAssets(deps, [result.asset]);
+    }
+}

package/dist/slices/rebuild/runtime-run.js

@@ -0,0 +1,35 @@
+import { executeExportRecipe } from "./runtime-export.js";
+import { executeLocalRecipe } from "./runtime-local.js";
+import { executeProviderRecipe } from "./runtime-provider.js";
+import { executeSubtitleRecipe } from "./runtime-subtitles.js";
+import { workPieces } from "./work-records.js";
+export async function runRevisionInvocation(deps, context, providers) {
+    const pieces = workPieces(deps.db, context.work.workId).filter((piece) => !piece.key.startsWith("article:continuation:"));
+    for (const piece of pieces) {
+        if (piece.state === "done")
+            continue;
+        if (piece.input.kind === "deferred")
+            return "held";
+        try {
+            const outcome = piece.key.startsWith("subtitles:")
+                ? await executeSubtitleRecipe(deps, context, piece)
+                : piece.key.startsWith("export:")
+                    ? await executeExportRecipe(deps, context, piece)
+                    : piece.input.kind === "llm" || piece.input.kind === "tts" || piece.input.kind === "image"
+                        ? await executeProviderRecipe(deps, context, providers, piece)
+                        : await executeLocalRecipe(deps, context, piece);
+            if (outcome === "held")
+                return outcome;
+        }
+        catch (error) {
+            deps.db
+                .prepare("UPDATE revision_work_pieces SET state=? WHERE id=? AND work_id=? AND state!='done'")
+                .run(context.signal.aborted ? "pending" : "failed", piece.id, context.work.workId);
+            throw error;
+        }
+    }
+    deps.db
+        .prepare("UPDATE revision_work_pieces SET state='done' WHERE work_id=? AND result_json IS NOT NULL")
+        .run(context.work.workId);
+    return "done";
+}

package/dist/slices/rebuild/runtime-store.js

@@ -0,0 +1,166 @@
+import { z } from "zod";
+import { stageKinds, stageStates } from "../../kernel/pipeline.js";
+import { currentRevisionId } from "../revisions/repo.js";
+import { getRevisionView } from "../revisions/view.js";
+import { settleScheduleRunsForProject } from "../schedules/repo.js";
+import { executionPlan, executionView, savedCatalogue } from "./runtime-plan.js";
+import { workPieces } from "./work-records.js";
+const workRow = z.object({
+    id: z.string(),
+    project_id: z.string(),
+    revision_id: z.string(),
+    stage_id: z.string(),
+    kind: z.enum(stageKinds),
+    state: z.enum(stageStates),
+    fingerprint: z.string(),
+});
+export function executionStages(deps, projectId) {
+    const head = currentRevisionId(deps.db, projectId);
+    if (head === undefined)
+        return [];
+    const view = getRevisionView(deps, projectId, head);
+    const rows = deps.db
+        .prepare(`SELECT DISTINCT w.* FROM revision_work w JOIN revision_work_reservations r ON r.work_id=w.id WHERE r.project_id=? AND r.revision_id=?`)
+        .all(projectId, head);
+    const keys = new Set(deps.db
+        .prepare("SELECT work_key FROM revision_work_reservations WHERE revision_id=?")
+        .all(head)
+        .map((row) => String(row.work_key)));
+    return rows
+        .filter((row) => workPieces(deps.db, String(row.id)).some((piece) => {
+        if (!keys.has(piece.key))
+            return false;
+        if (piece.key === "subtitles:timing" && view?.revision.content.subtitleCues !== undefined)
+            return false;
+        if (piece.input.kind === "deferred" && piece.key.endsWith(":future")) {
+            const prefix = piece.key.slice(0, -"future".length);
+            return !rows.some((candidate) => workPieces(deps.db, String(candidate.id)).some((part) => part.input.kind === "tts" && part.key.startsWith(prefix)));
+        }
+        return true;
+    }))
+        .map((value) => {
+        const row = workRow.parse(value);
+        return {
+            id: row.stage_id,
+            projectId,
+            kind: row.kind,
+            state: row.state,
+            work: {
+                projectId,
+                revisionId: row.revision_id,
+                workId: row.id,
+                stageId: row.stage_id,
+                kind: row.kind,
+                fingerprint: row.fingerprint,
+            },
+        };
+    });
+}
+export function invocationReady(deps, work) {
+    const row = deps.db
+        .prepare("SELECT recipe_context FROM revision_work WHERE id=? AND project_id=? AND revision_id=?")
+        .get(work.workId, work.projectId, work.revisionId);
+    if (row?.recipe_context === null || row === undefined)
+        return false;
+    const view = executionView(deps, work.projectId, work.revisionId);
+    if (view === undefined)
+        return false;
+    const plan = executionPlan(deps, view, savedCatalogue(row.recipe_context));
+    const pieces = workPieces(deps.db, work.workId);
+    return pieces.some((piece) => {
+        if (piece.state === "done" ||
+            piece.dispatchState !== "allowed" ||
+            piece.input.kind === "deferred")
+            return false;
+        if (piece.continuation !== null)
+            return true;
+        const recipe = plan.recipes.find((value) => value.key === piece.key && value.fingerprint === piece.fingerprint);
+        if (recipe === undefined || recipe.unresolved || recipe.deferred)
+            return false;
+        return recipe.dependsOn.every((key) => {
+            const dependency = plan.work.find((value) => value.key === key);
+            return dependency?.disposition === "reuse" && !dependency.inflight;
+        });
+    });
+}
+export function executionStandings(deps, projectId) {
+    const revisionId = currentRevisionId(deps.db, projectId);
+    const view = revisionId === undefined ? undefined : getRevisionView(deps, projectId, revisionId);
+    if (view === undefined)
+        return [];
+    const work = executionStages(deps, projectId);
+    return stageKinds.map((kind) => {
+        const source = view.revision.config.sources[kind];
+        const group = work
+            .filter((entry) => entry.kind === kind)
+            .map((entry) => {
+            const pieces = workPieces(deps.db, entry.work.workId);
+            const retained = pieces.length > 0 &&
+                pieces.every((piece) => view.outputs.some((output) => output.workKey === piece.key &&
+                    output.selected &&
+                    output.available &&
+                    output.state === "ready") ||
+                    view.pieces.some((output) => output.key === piece.key &&
+                        output.selected &&
+                        output.available &&
+                        output.piece.state === "done" &&
+                        output.fingerprint === piece.fingerprint));
+            // Save can bind an existing asset without granting its placeholder invocation.
+            return retained ? { ...entry, state: "done" } : entry;
+        });
+        let state = source === "off" ? "skipped" : source === "provide" ? "provided" : "pending";
+        if (group.some((entry) => entry.state === "running"))
+            state = "running";
+        else if (group.some((entry) => entry.state === "failed"))
+            state = "failed";
+        else if (group.some((entry) => entry.state === "canceled"))
+            state = "canceled";
+        else if (group.length > 0)
+            state = group.every((entry) => entry.state === "done")
+                ? source === "provide"
+                    ? "provided"
+                    : "done"
+                : "pending";
+        return { kind, state };
+    });
+}
+export function projectStandings(deps, projectId) {
+    const standings = executionStandings(deps, projectId);
+    for (const stage of standings) {
+        const invocations = executionStages(deps, projectId).filter((work) => work.kind === stage.kind);
+        const current = invocations.reduce((total, work) => {
+            if (work.state === "done")
+                return total + 1;
+            const progress = deps.db
+                .prepare("SELECT progress_current,progress_total FROM revision_work WHERE id=?")
+                .get(work.work.workId);
+            if (typeof progress?.progress_current !== "number" ||
+                typeof progress.progress_total !== "number" ||
+                progress.progress_total <= 0)
+                return total;
+            return total + Math.max(0, Math.min(1, progress.progress_current / progress.progress_total));
+        }, 0);
+        const failed = invocations.find((work) => work.state === "failed");
+        const failure = failed === undefined
+            ? null
+            : (deps.db
+                .prepare("SELECT failure_reason FROM revision_work WHERE id=?")
+                .get(failed.work.workId)?.failure_reason ?? null);
+        deps.db
+            .prepare("UPDATE stages SET state=?,failure_reason=?,progress_current=?,progress_total=?,finished_at=CASE WHEN ?='done' THEN ? ELSE NULL END WHERE project_id=? AND kind=?")
+            .run(stage.state, failure, invocations.length === 0 ? null : current, invocations.length === 0 ? null : invocations.length, stage.state, deps.clock.now().toISOString(), projectId, stage.kind);
+    }
+    settleScheduleRunsForProject(deps.db, projectId, deps.clock.now().toISOString());
+}
+export function recordWorkProgress(deps, event) {
+    if (event.workId === undefined ||
+        event.revisionId === undefined ||
+        !Number.isFinite(event.current) ||
+        !Number.isFinite(event.total) ||
+        event.total <= 0)
+        return;
+    deps.db
+        .prepare("UPDATE revision_work SET progress_current=?,progress_total=? WHERE id=? AND revision_id=? AND project_id=? AND kind=? AND state='running'")
+        .run(Math.max(0, Math.min(event.current, event.total)), event.total, event.workId, event.revisionId, event.projectId, event.stage);
+    projectStandings(deps, event.projectId);
+}

package/dist/slices/rebuild/runtime-subtitles.js

@@ -0,0 +1,202 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { resolveFont } from "../fonts/index.js";
+import { validateCues } from "../revisions/rules.js";
+import { discardPreparedAssets, writeAsset } from "../storage/assets.js";
+import { outputPath } from "../storage/layout.js";
+import { captionCues, serializeAss, serializeSrt, serializeVtt } from "../subtitles/captions.js";
+import { exportSnapshot, retainedOutput, revisionAudio, revisionTranscript, } from "./runtime-export-inputs.js";
+import { preparedResult, preparedText, publishResult } from "./runtime-publication.js";
+const wordsSchema = z.object({
+    key: z.string(),
+    words: z.array(z.object({
+        text: z.string(),
+        start: z.number().finite().nonnegative(),
+        end: z.number().finite().positive(),
+        confidence: z.number().optional(),
+    })),
+    omissions: z
+        .array(z.object({ start: z.number().finite().nonnegative(), text: z.string() }))
+        .default([]),
+});
+const cuesSchema = z.object({
+    cues: z.array(z.object({
+        text: z.string(),
+        start: z.number().finite().nonnegative(),
+        end: z.number().finite().positive(),
+    })),
+    omissions: wordsSchema.shape.omissions,
+});
+export async function executeSubtitleRecipe(deps, context, piece) {
+    if (!context.maySubmit(piece.id))
+        return "held";
+    context.signal.throwIfAborted();
+    const snapshot = exportSnapshot(deps, context, piece);
+    if (piece.key === "subtitles:timing")
+        return timing(deps, context, piece, snapshot);
+    if (piece.key === "subtitles:cues")
+        return cues(deps, context, piece, snapshot);
+    if (piece.key === "subtitles:files")
+        return files(deps, context, piece, snapshot);
+    throw new Error("Unknown subtitle recipe.");
+}
+async function timing(deps, context, piece, snapshot) {
+    const audio = await revisionAudio(deps, context, snapshot.view);
+    if (audio.length === 0)
+        throw new Error("Subtitles need narration audio.");
+    if (deps.alignSubtitles === undefined)
+        throw new Error("Local subtitle alignment is unavailable in this build.");
+    if (!context.maySubmit(piece.id))
+        return "held";
+    const words = [];
+    const omissions = [];
+    let offset = 0;
+    const total = audio.reduce((sum, segment) => sum + segment.seconds, 0);
+    for (const segment of audio) {
+        context.signal.throwIfAborted();
+        if (segment.path !== null && segment.kind !== "gap") {
+            const aligned = await deps.alignSubtitles({
+                audioPath: segment.path,
+                text: revisionTranscript(deps, snapshot, segment.kind),
+                cacheDir: join(deps.paths.dataDir, "models", "english-subtitles"),
+                ffmpeg: deps.ffmpeg,
+                signal: context.signal,
+                onOmission: (value) => omissions.push({ ...value, start: value.start + offset }),
+                onProgress: (current, maximum) => context.emit({
+                    type: "stage.progress",
+                    projectId: context.work.projectId,
+                    stage: "video",
+                    current: Math.round((100 * (offset + segment.seconds * (maximum > 0 ? current / maximum : 0))) / total),
+                    total: 100,
+                }),
+            });
+            for (const word of aligned) {
+                if (word.end > segment.seconds + 0.1)
+                    throw new Error("Subtitle timing exceeds the narration duration.");
+                words.push({
+                    ...word,
+                    start: word.start + offset,
+                    end: Math.min(word.end, segment.seconds) + offset,
+                });
+            }
+        }
+        offset += segment.seconds;
+    }
+    if (captionCues(words).length === 0)
+        throw new Error("No spoken words could be aligned. Check that the English article matches the audio.");
+    context.signal.throwIfAborted();
+    const output = preparedText(deps, context, piece, "subtitle_words", "subtitles.json", JSON.stringify({ key: piece.logicalFingerprint ?? piece.fingerprint, words, omissions }));
+    await publishResult(deps, context, piece, [output], { words, omissions }, output.asset);
+    return "done";
+}
+async function cues(deps, context, piece, snapshot) {
+    const { view, plan } = snapshot;
+    const manual = view.revision.content.subtitleCues;
+    let value;
+    if (manual !== undefined) {
+        if (plan.work.find((one) => one.key === piece.key)?.disposition === "review")
+            throw new Error("Review the saved captions against the current narration before rebuilding.");
+        const audio = await revisionAudio(deps, context, view);
+        const errors = validateCues(manual.cues, audio.reduce((sum, segment) => sum + segment.seconds, 0));
+        if (errors.length > 0)
+            throw new Error(errors.map((error) => error.message).join(" "));
+        value = { cues: [...manual.cues], omissions: [] };
+    }
+    else {
+        const output = view.outputs.find((one) => one.selected &&
+            one.available &&
+            one.state === "ready" &&
+            one.workKey === "subtitles:timing");
+        if (output === undefined)
+            throw new Error("The revision has no aligned subtitle timing.");
+        const timed = wordsSchema.parse(JSON.parse(readFileSync(outputPath(deps.paths, context.work.projectId, output.output.path), "utf8")));
+        value = { cues: [...captionCues(timed.words)], omissions: timed.omissions };
+    }
+    if (!context.maySubmit(piece.id))
+        return "held";
+    const asset = writeAsset(deps, context.work.projectId, "cues.json", Buffer.from(JSON.stringify(value)));
+    await publishResult(deps, context, piece, [], value, asset);
+    return "done";
+}
+async function files(deps, context, piece, snapshot) {
+    const { view } = snapshot;
+    const config = view.revision.config.subtitles;
+    if (config === undefined || config.mode === "off")
+        throw new Error("The admitted revision has no subtitles.");
+    const row = view.pieces.find((one) => one.key === "subtitles:cues" && one.selected && one.available && one.piece.state === "done");
+    if (row?.piece.payload === null || row?.piece.payload === undefined)
+        throw new Error("The revision has no prepared captions.");
+    const value = cuesSchema.parse(JSON.parse(row.piece.payload));
+    const previous = view.outputs.find((one) => one.output.role === "subtitle_font" && one.selected && one.available);
+    const previousFiles = view.pieces.find((one) => one.key === "subtitles:files" &&
+        one.piece.state === "done" &&
+        (one.selected ||
+            (one.publicationId !== null && one.publicationId === previous?.publicationId)));
+    const savedFont = z
+        .object({
+        font: z
+            .object({
+            id: z.string(),
+            name: z.string(),
+            assName: z.string(),
+            extension: z.enum([".ttf", ".otf", ".ttc"]),
+        })
+            .optional(),
+    })
+        .parse(JSON.parse(previousFiles?.piece.payload ?? "{}")).font;
+    const font = savedFont?.id === config.fontId && previous !== undefined
+        ? { ...savedFont, path: outputPath(deps.paths, context.work.projectId, previous.output.path) }
+        : await resolveFont(deps.paths, config.fontId);
+    if (!context.maySubmit(piece.id))
+        return "held";
+    const prepared = [];
+    try {
+        const frame = view.revision.config.format === "16:9"
+            ? { width: 1920, height: 1080 }
+            : { width: 1080, height: 1920 };
+        for (const [role, filename, text] of [
+            ["subtitles_srt", "subtitles.srt", serializeSrt(value.cues)],
+            ["subtitles_vtt", "subtitles.vtt", serializeVtt(value.cues)],
+            [
+                "subtitle_ass",
+                "subtitles.ass",
+                serializeAss(value.cues, {
+                    ...frame,
+                    fontName: font.assName,
+                    fontSize: config.fontSize,
+                    position: config.position,
+                }),
+            ],
+        ])
+            prepared.push(preparedText(deps, context, piece, role, filename, text));
+        prepared.push(preparedResult(deps, context, piece, "subtitle_font", writeAsset(deps, context.work.projectId, `selected${font.extension}`, readFileSync(font.path))));
+        const media = view.outputs.find((one) => one.selected &&
+            one.available &&
+            one.state === "ready" &&
+            (one.output.role === "audio_export" ||
+                (one.output.role === "video" && config.mode === "files")));
+        if (media !== undefined) {
+            const retained = retainedOutput(deps, view, media);
+            prepared.push({
+                ...retained,
+                output: {
+                    ...retained.output,
+                    meta: {
+                        ...retained.output.meta,
+                        subtitlesMode: "files",
+                        subtitleOmissions: value.omissions,
+                    },
+                },
+            });
+        }
+        await publishResult(deps, context, piece, prepared, {
+            ...value,
+            font: { id: font.id, name: font.name, assName: font.assName, extension: font.extension },
+        });
+        return "done";
+    }
+    finally {
+        discardPreparedAssets(deps, prepared.map((one) => one.asset));
+    }
+}