@gentbajko/slopify 0.8.4 → 1.0.0

package/dist/slices/narration/legacy-plan.js

@@ -0,0 +1,58 @@
+import { z } from "zod";
+import { transact } from "../../kernel/db/tx.js";
+import { splitText } from "../../kernel/ports/text.js";
+import { insertPiece, piecesOf } from "../../kernel/runner/piece-repo.js";
+const payload = z.object({ text: z.string(), file: z.string().optional() });
+// Old whole-text plans can exceed a newly known account limit. Preserve finished
+// pieces and their paths, and split only unfinished work before any request starts.
+export function planLegacyNarration(deps, stageId, texts, maxCharacters, finished) {
+    const old = piecesOf(deps.db, stageId, "chunk");
+    const source = old.length
+        ? old
+        : texts.map((text, idx) => ({
+            id: deps.ids.next(),
+            stageId,
+            kind: "chunk",
+            idx: idx + 1,
+            state: "pending",
+            payload: JSON.stringify({ text }),
+        }));
+    const planned = [];
+    for (const piece of source) {
+        const data = payload.parse(JSON.parse(piece.payload ?? "null"));
+        if (finished(piece)) {
+            planned.push({ ...piece, idx: planned.length + 1 });
+            continue;
+        }
+        for (const [part, text] of splitText(data.text, maxCharacters).entries())
+            planned.push({
+                ...piece,
+                id: part === 0 ? piece.id : deps.ids.next(),
+                idx: planned.length + 1,
+                state: "pending",
+                payload: JSON.stringify({ text }),
+            });
+    }
+    if (old.length &&
+        planned.length === old.length &&
+        planned.every((p, i) => p.id === old[i]?.id &&
+            payload.parse(JSON.parse(p.payload ?? "null")).text ===
+                payload.parse(JSON.parse(old[i]?.payload ?? "null")).text))
+        return old;
+    transact(deps.db, () => {
+        // Negative temporary indices avoid colliding with a retained completed piece.
+        deps.db
+            .prepare("UPDATE stage_pieces SET idx=-idx WHERE stage_id=? AND kind='chunk'")
+            .run(stageId);
+        const known = new Set(old.map((p) => p.id));
+        for (const piece of planned) {
+            if (known.has(piece.id))
+                deps.db
+                    .prepare("UPDATE stage_pieces SET idx=?,state=?,payload=? WHERE id=?")
+                    .run(piece.idx, piece.state, piece.payload, piece.id);
+            else
+                insertPiece(deps.db, piece);
+        }
+    });
+    return planned;
+}

package/dist/slices/narration/live.js

@@ -1,11 +1,11 @@
-export function observeNarration(store, projectId, key, label) {
+export function observeNarration(store, projectId, key, label, origin) {
     if (store === undefined)
         return undefined;
     let sink;
     return (event) => {
         switch (event.type) {
             case "start":
-                sink = store.begin(projectId, key, label);
+                sink = store.begin(projectId, key, label, origin);
                 break;
             case "chunk":
                 sink?.append(event.bytes);

package/dist/slices/narration/plan.js

@@ -1,58 +1,52 @@
-import { z } from "zod";
-import { transact } from "../../kernel/db/tx.js";
 import { splitText } from "../../kernel/ports/text.js";
-import { insertPiece, piecesOf } from "../../kernel/runner/piece-repo.js";
-const payload = z.object({ text: z.string(), file: z.string().optional() });
-// Old whole-text plans can exceed a newly known account limit. Preserve finished
-// pieces and their paths, and split only unfinished work before any request starts.
-export function planNarration(deps, stageId, texts, maxCharacters, finished) {
-    const old = piecesOf(deps.db, stageId, "chunk");
-    const source = old.length
-        ? old
-        : texts.map((text, idx) => ({
-            id: deps.ids.next(),
-            stageId,
-            kind: "chunk",
-            idx: idx + 1,
-            state: "pending",
-            payload: JSON.stringify({ text }),
-        }));
-    const planned = [];
-    for (const piece of source) {
-        const data = payload.parse(JSON.parse(piece.payload ?? "null"));
-        if (finished(piece)) {
-            planned.push({ ...piece, idx: planned.length + 1 });
-            continue;
-        }
-        for (const [part, text] of splitText(data.text, maxCharacters).entries())
-            planned.push({
-                ...piece,
-                id: part === 0 ? piece.id : deps.ids.next(),
-                idx: planned.length + 1,
-                state: "pending",
-                payload: JSON.stringify({ text }),
+import { fingerprint } from "../../kernel/runner/work.js";
+export function normalizeNarrationText(text) {
+    return text.replace(/\r\n?/g, "\n").trim();
+}
+export function narrationRequestFingerprint(input) {
+    return fingerprint([
+        "narration-request-v1",
+        input.provider,
+        input.model,
+        input.voice,
+        input.text,
+        null,
+        input.segment,
+        input.wholeText === null ? null : fingerprint(["whole-v1", input.wholeText]),
+    ]);
+}
+export function planNarration(input) {
+    return input.groups.flatMap((group) => {
+        const text = normalizeNarrationText(group.text);
+        return splitText(text, input.maxCharacters).map((part, index) => {
+            const requestFingerprint = narrationRequestFingerprint({
+                ...input,
+                text: part,
+                segment: group.segment,
+                wholeText: group.wholeRequest ? text : null,
             });
-    }
-    if (old.length &&
-        planned.length === old.length &&
-        planned.every((p, i) => p.id === old[i]?.id &&
-            payload.parse(JSON.parse(p.payload ?? "null")).text ===
-                payload.parse(JSON.parse(old[i]?.payload ?? "null")).text))
-        return old;
-    transact(deps.db, () => {
-        // Negative temporary indices avoid colliding with a retained completed piece.
-        deps.db
-            .prepare("UPDATE stage_pieces SET idx=-idx WHERE stage_id=? AND kind='chunk'")
-            .run(stageId);
-        const known = new Set(old.map((p) => p.id));
-        for (const piece of planned) {
-            if (known.has(piece.id))
-                deps.db
-                    .prepare("UPDATE stage_pieces SET idx=?,state=?,payload=? WHERE id=?")
-                    .run(piece.idx, piece.state, piece.payload, piece.id);
-            else
-                insertPiece(deps.db, piece);
-        }
+            const workFingerprint = fingerprint([requestFingerprint, group.regenerationToken]);
+            const retained = group.regenerationToken === null
+                ? input.retained.find((one) => one.available && one.requestFingerprint === requestFingerprint)
+                : input.retained.find((one) => one.available && one.fingerprint === workFingerprint);
+            return {
+                key: `${group.key}:${index + 1}`,
+                logicalKey: group.key,
+                logicalText: text,
+                segment: group.segment,
+                text: part,
+                requestFingerprint,
+                fingerprint: workFingerprint,
+                assetId: retained?.assetId ?? null,
+            };
+        });
     });
-    return planned;
+}
+export function narrationRegenerationToken(tokens, logicalKey, segment) {
+    return tokens[logicalKey] ?? tokens[`audio:${segment}:future`] ?? null;
+}
+export function narrationRegenerationKey(key) {
+    return /^audio:(body:.+|intro|outro):[0-9]+$/.test(key)
+        ? key.slice(0, key.lastIndexOf(":"))
+        : key;
 }

package/dist/slices/narration/run.js

@@ -10,8 +10,8 @@ import { insertOutput, outputsOf } from "../storage/repo.js";
 import { probeDurationMs } from "../video/ffmpeg.js";
 import { chunkNarration, defaultChunking } from "./chunk.js";
 import { joinNarration } from "./concat.js";
+import { planLegacyNarration } from "./legacy-plan.js";
 import { observeNarration } from "./live.js";
-import { planNarration } from "./plan.js";
 // An empty narration source fails the stage immediately with no retries. Thrown from the
 // slice rather than from a provider call, so the attempt wrapper never sees it.
 export const nothingToNarrate = "nothing to narrate";
@@ -53,15 +53,19 @@ export async function runNarration(deps, context, providers) {
     if (texts.length === 0) {
         throw new Error(nothingToNarrate);
     }
-    const planned = planNarration(deps, context.stage.id, texts, deps.maxCharacters?.(choice.provider, choice.model) ?? 1000000, (piece) => finished(deps, projectId, piece) !== undefined);
-    const files = await speakChunks(deps, context, providers, choice, planned);
-    await storeBody(deps, context, choice, files, outputs);
-    await speakSegments(deps, context, providers, choice, outputs);
+    const planned = planLegacyNarration(deps, context.stage.id, texts, deps.maxCharacters?.(choice.provider, choice.model) ?? 1000000, (piece) => finished(deps, projectId, piece) !== undefined);
+    const result = await speakChunks(deps, context, providers, choice, planned);
+    if (!result.ok)
+        return "held";
+    await storeBody(deps, context, choice, result.value, outputs);
+    if ((await speakSegments(deps, context, providers, choice, outputs)) === "held")
+        return "held";
     deps.log.write("info", "narration.done", {
         projectId,
         stage: "audio",
         detail: `${String(texts.length)} chunks in ${chunking.mode} mode`,
     });
+    return "done";
 }
 // The narration source: the plain-text article, written by the article stage or pasted in
 // by a user whose article stage was set to Provide.
@@ -95,7 +99,11 @@ async function speakChunks(deps, context, providers, choice, pieces) {
                 voiceId: choice.voice,
                 text: carried.text,
             }, observeNarration(deps.audioPreviews, projectId, piece.id, total === 1 ? "Body" : `Body part ${piece.idx} of ${total}`));
-            write(deps, projectId, file, spoken.bytes);
+            if (!spoken.ok) {
+                setPiece(deps.db, piece.id, "pending", piece.payload);
+                return spoken;
+            }
+            write(deps, projectId, file, spoken.value.bytes);
             // The file is on disk before the row says so: a crash between the two leaves a
             // file the reconcile collects, where the other order would leave a `done` chunk
             // whose audio never existed and a concat that fails on every retry.
@@ -114,11 +122,13 @@ async function speakChunks(deps, context, providers, choice, pieces) {
     const files = [];
     for (const outcome of outcomes) {
         if (!outcome.ok) {
+            if ("reason" in outcome)
+                return outcome;
             throw outcome.error;
         }
         files.push(outcome.file);
     }
-    return files;
+    return { ok: true, value: files };
 }
 // One body file, in chunk order, with its measured duration on the row.
 async function storeBody(deps, context, choice, files, outputs) {
@@ -166,8 +176,10 @@ async function speakSegments(deps, context, providers, choice, outputs) {
             voiceId: choice.voice,
             text,
         }, observeNarration(deps.audioPreviews, projectId, piece.id, segment.category === "intro" ? "Intro" : "Outro"));
+        if (!spoken.ok)
+            return "held";
         const name = outputFileName(role, 1, ".mp3", "audio");
-        write(deps, projectId, name, spoken.bytes);
+        write(deps, projectId, name, spoken.value.bytes);
         // Measured the same way the body is, because the render adds all three and the gaps to
         // get the length of the video.
         const durationMs = await probeDurationMs(deps.ffmpeg, outputPath(deps.paths, projectId, name), context.signal, deps.log);
@@ -175,6 +187,7 @@ async function speakSegments(deps, context, providers, choice, outputs) {
         store(deps, projectId, role, name, durationMs, choice);
         counted(deps, segment.category, durationMs, choice);
     }
+    return "done";
 }
 // The segment pieces belong to the *article* stage, which is where they are written, so
 // they are read under its id and never under this stage's own. They stay there because the

package/dist/slices/play-drafts/convert.js

@@ -0,0 +1,120 @@
+import { z } from "zod";
+import { numberPerPromptMax } from "../admission/rules.js";
+import { runDraftSchema } from "../admission/schema.js";
+import { defaultSubtitles } from "../subtitles/model.js";
+export function toAdmissionDraft(input) {
+    const { form } = input.document;
+    const fields = [];
+    const sources = {
+        ...form.sources,
+        ...(form.sources.article === "provide" ? { research: "off" } : {}),
+        ...(form.sources.images === "off" ? { video: "off" } : {}),
+    };
+    const number = (raw, field, max, min = 1) => {
+        const parsed = z
+            .number()
+            .int()
+            .min(min)
+            .max(max)
+            .safeParse(raw.trim() === "" ? Number.NaN : Number(raw));
+        if (parsed.success)
+            return parsed.data;
+        fields.push({ field, message: `Enter a whole number between ${min} and ${max}.` });
+        return min;
+    };
+    const entry = (category) => {
+        const name = form[category];
+        if (sources.audio !== "generate" || name === "")
+            return undefined;
+        const saved = input.entries.find((candidate) => candidate.category === category && candidate.name.toLowerCase() === name.toLowerCase());
+        if (saved)
+            return { name: saved.name, mode: saved.mode };
+        fields.push({
+            field: category,
+            message: `That ${category} entry no longer exists; pick another.`,
+        });
+        return undefined;
+    };
+    const file = (ref, kind, field) => {
+        const attachment = input.attachments.find((item) => item.id === ref?.attachmentId && item.kind === kind && item.name === ref.name);
+        if (attachment?.state === "ready" && attachment.stagedFileId !== null)
+            return attachment.stagedFileId;
+        fields.push({
+            field,
+            message: attachment?.state === "copying" || attachment?.state === "pending"
+                ? "This upload is still copying."
+                : "That upload is no longer available; pick the file again.",
+        });
+        return undefined;
+    };
+    const mode = sources.audio === "off"
+        ? "off"
+        : sources.video === "off" && form.subtitles.mode === "burn-in"
+            ? "files"
+            : form.subtitles.mode;
+    const chunkMode = sources.audio === "generate" ? form.chunking.mode : "whole";
+    const draft = {
+        ...(form.checkpoints === undefined ? {} : { checkpoints: form.checkpoints }),
+        title: form.title,
+        format: form.format,
+        sources,
+        llm: form.llm,
+        audio: sources.audio === "generate" ? form.audio : undefined,
+        images: form.images,
+        articlePrompt: sources.article === "generate" ? form.articlePrompt : undefined,
+        imagePrompts: sources.images === "generate"
+            ? form.imagePrompts.map((prompt, index) => ({
+                name: prompt.name,
+                number: number(prompt.number, `imagePrompts.${index}.number`, numberPerPromptMax),
+            }))
+            : [],
+        thumbnailPrompt: ["from_prompt", "prompt_by_llm"].includes(sources.thumbnail)
+            ? form.thumbnailPrompt
+            : undefined,
+        intro: entry("intro"),
+        outro: entry("outro"),
+        values: form.values,
+        provided: {
+            research: sources.research === "provide" ? form.provided.research : undefined,
+            article: sources.article === "provide" ? form.provided.article : undefined,
+            audio: sources.audio === "provide"
+                ? file(form.provided.audio, "audio", "provided.audio")
+                : undefined,
+            thumbnail: sources.thumbnail === "provide"
+                ? file(form.provided.thumbnail, "thumbnail", "provided.thumbnail")
+                : undefined,
+            images: sources.images === "provide"
+                ? form.provided.images.flatMap((ref, index) => {
+                    const id = file(ref, "images", `provided.images.${index}`);
+                    return id === undefined ? [] : [id];
+                })
+                : [],
+        },
+        chunking: {
+            mode: chunkMode,
+            ...(chunkMode === "words"
+                ? { words: number(form.chunking.words, "chunking.words", 10000) }
+                : {}),
+            ...(chunkMode === "characters"
+                ? { characters: number(form.chunking.characters, "chunking.characters", 1000000) }
+                : {}),
+        },
+        subtitles: mode === "off"
+            ? defaultSubtitles
+            : {
+                ...form.subtitles,
+                mode,
+                fontSize: number(form.subtitles.fontSize, "subtitles.fontSize", 120, 16),
+            },
+        silenceGapSeconds: input.silenceGapSeconds,
+    };
+    const parsed = runDraftSchema.safeParse(draft);
+    if (!parsed.success)
+        fields.push(...parsed.error.issues.map((issue) => ({
+            field: issue.path.join("."),
+            message: issue.message,
+        })));
+    return fields.length || !parsed.success
+        ? { ok: false, fields }
+        : { ok: true, draft: parsed.data };
+}

package/dist/slices/play-drafts/model.js

@@ -0,0 +1 @@
+export {};

package/dist/slices/play-drafts/readiness.js

@@ -0,0 +1,80 @@
+import { modelFields } from "../../catalog/validate.js";
+import { readinessIsUsable } from "../../kernel/ports/model.js";
+import { cliPathStatus } from "../settings/cli-paths.js";
+import { hasKey, listVoices } from "../settings/repo.js";
+function choices(runs) {
+    const selected = runs
+        .flatMap(({ draft: d }) => [
+        {
+            field: "llm",
+            family: "llm",
+            choice: d.llm,
+            needed: d.sources.research === "generate" ||
+                d.sources.article === "generate" ||
+                d.sources.thumbnail === "prompt_by_llm" ||
+                d.intro?.mode === "llm" ||
+                d.outro?.mode === "llm",
+        },
+        {
+            field: "audio",
+            family: "tts",
+            choice: d.audio,
+            needed: d.sources.audio === "generate",
+        },
+        {
+            field: "images",
+            family: "image",
+            choice: d.images,
+            needed: d.sources.images === "generate" ||
+                ["from_prompt", "prompt_by_llm"].includes(d.sources.thumbnail),
+        },
+    ])
+        .flatMap((c) => (c.needed && c.choice ? [{ ...c, ...c.choice }] : []));
+    return [...new Map(selected.map((c) => [`${c.family}:${c.provider}:${c.model}`, c])).values()];
+}
+export async function checkDraftReadiness(deps, runs) {
+    const selected = choices(runs);
+    if (selected.length === 0)
+        return { fields: [], providers: [] };
+    const providers = await deps.providers();
+    const fields = [];
+    for (const c of selected) {
+        const provider = providers.find((p) => p.id === c.provider && p.family === c.family);
+        if (!provider || !readinessIsUsable(provider.readiness)) {
+            fields.push({
+                field: c.field,
+                message: provider?.readiness.kind === "cli" && provider.readiness.issue
+                    ? provider.readiness.issue
+                    : "Configure this provider before starting.",
+            });
+            continue;
+        }
+        const models = await deps.modelsFor(c.provider, c.family);
+        if (!models.some((m) => m.id === c.model))
+            fields.push({
+                field: `${c.field}.model`,
+                message: "Choose an available model before starting.",
+            });
+    }
+    return { fields: [...fields, ...localDraftReadiness(deps, runs, providers)], providers };
+}
+export function localDraftReadiness(deps, runs, providers) {
+    const fields = runs.flatMap((run) => modelFields(run.draft, deps.catalogue));
+    for (const c of choices(runs)) {
+        const provider = providers.find((p) => p.id === c.provider && p.family === c.family);
+        if (provider?.cliPath !== undefined &&
+            cliPathStatus(deps.db, provider.id).command !== provider.cliPath.command)
+            fields.push({
+                field: `${c.field}.cliPath`,
+                message: "The configured CLI command changed. Check readiness again.",
+            });
+        if (provider?.readiness.kind === "keyed" && !hasKey(deps.db, provider.id))
+            fields.push({ field: c.field, message: "Save the provider key before starting." });
+    }
+    for (const { draft } of runs)
+        if (draft.sources.audio === "generate" &&
+            draft.audio &&
+            !listVoices(deps.db).some((v) => v.provider === draft.audio?.provider && v.voiceId === draft.audio.voice))
+            fields.push({ field: "audio.voice", message: "Choose a saved voice before starting." });
+    return fields;
+}

package/dist/slices/play-drafts/repo.js

@@ -0,0 +1,76 @@
+import { z } from "zod";
+import { fingerprint } from "../../kernel/runner/work.js";
+const rowSchema = z.object({
+    id: z.string(),
+    schema_version: z.number(),
+    version: z.number(),
+    title: z.string(),
+    document_json: z.string(),
+    creation_hash: z.string(),
+    save_mutation_id: z.string().nullable(),
+    save_request_hash: z.string().nullable(),
+    created_at: z.string(),
+    updated_at: z.string(),
+    state: z.enum(["active", "starting", "started"]),
+    review_id: z.string().nullable(),
+    review_json: z.string().nullable(),
+    review_fingerprint: z.string().nullable(),
+    start_id: z.string().nullable(),
+});
+const attachmentSchema = z.object({
+    id: z.string(),
+    draft_id: z.string(),
+    staged_file_id: z.string().nullable(),
+    kind: z.enum(["audio", "images", "thumbnail"]),
+    original_filename: z.string(),
+    status: z.enum(["pending", "ready", "reattach"]),
+    error: z.string().nullable(),
+});
+export function draftRow(db, id) {
+    const row = db.prepare("SELECT * FROM play_drafts WHERE id=?").get(id);
+    return row === undefined ? undefined : rowSchema.parse(row);
+}
+export function draftRows(db) {
+    return db
+        .prepare("SELECT * FROM play_drafts WHERE state IN ('active','starting') ORDER BY updated_at DESC,id")
+        .all()
+        .map((row) => rowSchema.parse(row));
+}
+export function attachmentRow(db, id) {
+    const row = db.prepare("SELECT * FROM play_draft_attachments WHERE id=?").get(id);
+    return row === undefined ? undefined : attachmentSchema.parse(row);
+}
+export function attachmentRows(db, id) {
+    return db
+        .prepare("SELECT * FROM play_draft_attachments WHERE draft_id=? ORDER BY rowid")
+        .all(id)
+        .map((row) => attachmentSchema.parse(row));
+}
+export function attachmentRefs(document) {
+    const p = document.form.provided;
+    return [
+        ...(p.audio === null ? [] : [{ ...p.audio, kind: "audio" }]),
+        ...p.images.map((file) => ({ ...file, kind: "images" })),
+        ...(p.thumbnail === null ? [] : [{ ...p.thumbnail, kind: "thumbnail" }]),
+    ];
+}
+export function requestHash(value) {
+    return fingerprint(z.json().parse(JSON.parse(JSON.stringify(value))));
+}
+export function insertDraft(db, id, document, hash, at) {
+    db.prepare("INSERT INTO play_drafts (id,schema_version,version,title,document_json,creation_hash,created_at,updated_at,state) VALUES (?,1,1,?,?,?,?,?,'active')").run(id, document.form.title, JSON.stringify(document), hash, at, at);
+}
+export function insertAttachment(db, id, ref, status, stagedId = null) {
+    db.prepare("INSERT INTO play_draft_attachments (id,draft_id,kind,original_filename,status,staged_file_id) VALUES (?,?,?,?,?,?)").run(ref.attachmentId, id, ref.kind, ref.name, status, stagedId);
+}
+export function syncAttachments(db, id, document) {
+    const refs = attachmentRefs(document);
+    const retained = new Set(refs.map((ref) => ref.attachmentId));
+    for (const old of attachmentRows(db, id))
+        if (!retained.has(old.id))
+            db.prepare("DELETE FROM play_draft_attachments WHERE id=?").run(old.id);
+    // Removed staged bytes remain until reference-aware storage cleanup runs.
+    for (const ref of refs)
+        if (attachmentRow(db, ref.attachmentId) === undefined)
+            insertAttachment(db, id, ref, "pending");
+}