@gentbajko/slopify 0.8.4 → 1.0.0

package/dist/slices/rebuild/runtime-materialize.js

@@ -0,0 +1,105 @@
+import { z } from "zod";
+import { transact } from "../../kernel/db/tx.js";
+import { currentRevisionId } from "../revisions/repo.js";
+import { getRevisionView } from "../revisions/view.js";
+import { insertInvocation } from "./runtime-admission.js";
+import { bindNarrationReuse, narrationOrdinal } from "./runtime-narration-reuse.js";
+import { executionPlan, executionView, savedCatalogue } from "./runtime-plan.js";
+import { workPieces } from "./work-records.js";
+const reservation = z.object({
+    revision_id: z.string(),
+    work_id: z.string(),
+    work_key: z.string(),
+    fingerprint: z.string(),
+    logical_key: z.string().nullable(),
+    desired_fingerprint: z.string().nullable(),
+    origin_revision: z.string(),
+    recipe_context: z.string(),
+    admission_id: z.string().nullable(),
+});
+export function materializeAdmittedWork(deps, projectId) {
+    transact(deps.db, () => {
+        const head = currentRevisionId(deps.db, projectId);
+        if (head === undefined)
+            return;
+        const reservations = deps.db
+            .prepare(`SELECT r.*,w.revision_id AS origin_revision,w.recipe_context,w.admission_id FROM revision_work_reservations r JOIN revision_work w ON w.id=r.work_id WHERE r.project_id=? AND r.revision_id=? AND w.dispatch_state='allowed' AND w.recipe_context IS NOT NULL`)
+            .all(projectId, head)
+            .map((row) => reservation.parse(row));
+        const origins = new Map(reservations.map((row) => [row.origin_revision, row]));
+        for (const origin of origins.values()) {
+            const view = executionView(deps, projectId, origin.origin_revision);
+            if (view === undefined)
+                continue;
+            const catalogue = savedCatalogue(origin.recipe_context);
+            const plan = executionPlan(deps, view, catalogue);
+            for (const recipe of plan.recipes) {
+                if (!plan.work.some((row) => row.key === recipe.key))
+                    continue;
+                if (recipe.deferred || recipe.unresolved)
+                    continue;
+                if (recipe.input.kind === "tts") {
+                    const current = getRevisionView(deps, projectId, head);
+                    const desired = current?.revision.fingerprints[`${recipe.input.logicalKey}:1`];
+                    if (desired !== undefined && desired !== recipe.logicalFingerprint)
+                        continue;
+                }
+                const own = reservations.find((row) => row.work_key === recipe.key && row.origin_revision === origin.origin_revision);
+                if (own?.fingerprint === recipe.fingerprint)
+                    continue;
+                const anchor = own ??
+                    reservations.find((row) => row.origin_revision === origin.origin_revision && row.work_key === futureKey(recipe));
+                if (anchor === undefined || !desiredAnchor(deps, projectId, head, anchor))
+                    continue;
+                if (own !== undefined &&
+                    workPieces(deps.db, own.work_id).some((piece) => piece.submittedAt !== null || piece.state === "done"))
+                    continue;
+                if (plan.work.find((row) => row.key === recipe.key)?.disposition === "review")
+                    continue;
+                if (own !== undefined) {
+                    deps.db
+                        .prepare("UPDATE revision_work SET state='done' WHERE id=? AND state='pending'")
+                        .run(own.work_id);
+                    deps.db
+                        .prepare("DELETE FROM revision_work_reservations WHERE revision_id=? AND work_key=?")
+                        .run(head, recipe.key);
+                }
+                const placeholder = deps.db
+                    .prepare("SELECT w.dispatch_state FROM revision_work_reservations r JOIN revision_work w ON w.id=r.work_id WHERE r.revision_id=? AND r.work_key=?")
+                    .get(head, recipe.key);
+                if (placeholder !== undefined && placeholder.dispatch_state !== "held")
+                    continue;
+                deps.db
+                    .prepare("DELETE FROM revision_work_reservations WHERE revision_id=? AND work_key=?")
+                    .run(head, recipe.key);
+                if (plan.work.find((row) => row.key === recipe.key)?.disposition === "reuse") {
+                    const ordinal = narrationOrdinal(plan.recipes, recipe.key);
+                    bindNarrationReuse(deps, view, recipe, ordinal);
+                    if (head !== view.revision.id) {
+                        const current = getRevisionView(deps, projectId, head);
+                        if (current !== undefined)
+                            bindNarrationReuse(deps, current, recipe, ordinal);
+                    }
+                }
+                insertInvocation(deps, view, recipe, catalogue, {
+                    key: anchor.logical_key ?? anchor.work_key,
+                    fingerprint: anchor.desired_fingerprint ?? anchor.fingerprint,
+                }, plan.work.find((row) => row.key === recipe.key)?.disposition === "reuse", head, anchor.admission_id);
+            }
+        }
+    });
+}
+function desiredAnchor(deps, projectId, head, row) {
+    const view = getRevisionView(deps, projectId, head);
+    return (view?.revision.fingerprints[row.logical_key ?? row.work_key] ===
+        (row.desired_fingerprint ?? row.fingerprint));
+}
+function futureKey(recipe) {
+    if (recipe.input.kind === "tts")
+        return `audio:${recipe.input.segment}:future`;
+    if (recipe.key.startsWith("research:chapter:"))
+        return "research:planner";
+    if (recipe.key === "article:continuation")
+        return "article:body";
+    return recipe.key;
+}

package/dist/slices/rebuild/runtime-narration-reuse.js

@@ -0,0 +1,134 @@
+import { z } from "zod";
+import { insertManifestOutput, insertManifestPiece, selectOutputRecord, selectPieceRecord, } from "../revisions/manifest-repo.js";
+import { retainedNarrationPieces } from "./narration-history.js";
+import { matchingNarrationPiece } from "./narration-reuse.js";
+export function bindNarrationReuse(deps, view, recipe, ordinal) {
+    if (recipe.input.kind === "local" && recipe.input.operation === "concat-narration") {
+        const completed = view.pieces.find((one) => one.available &&
+            one.piece.state === "done" &&
+            one.key === recipe.key &&
+            one.fingerprint === recipe.fingerprint);
+        const output = completed === undefined
+            ? undefined
+            : view.outputs.find((one) => one.available && one.assetId === completed.assetId && one.workKey === recipe.key);
+        if (output !== undefined &&
+            !(output.selected &&
+                output.state === "ready" &&
+                output.fingerprint === recipe.logicalFingerprint)) {
+            const id = deps.ids.next();
+            insertManifestOutput(deps.db, view.revision, {
+                slot: output.slot,
+                workKey: output.workKey,
+                assetId: output.assetId,
+                output: output.output,
+                fingerprint: recipe.logicalFingerprint,
+                state: "ready",
+            }, id);
+            selectOutputRecord(deps.db, view.revision.id, output.slot, id);
+        }
+        return;
+    }
+    if (recipe.input.kind === "provided" && /^audio:(body:.+|intro|outro):[0-9]+$/.test(recipe.key)) {
+        if (view.pieces.some((one) => one.selected &&
+            one.available &&
+            one.key === recipe.key &&
+            one.fingerprint === recipe.fingerprint &&
+            one.piece.state === "done" &&
+            one.piece.idx === ordinal))
+            return;
+        const asset = deps.db
+            .prepare("SELECT path FROM project_assets WHERE id=? AND project_id=?")
+            .get(recipe.input.assetId, view.revision.projectId);
+        if (asset === undefined)
+            return;
+        const semantic = z
+            .tuple([z.string(), z.array(z.string().nullable())])
+            .parse(recipe.input.semantic);
+        const segment = recipe.key.startsWith("audio:body:")
+            ? "body"
+            : recipe.key.startsWith("audio:intro:")
+                ? "intro"
+                : "outro";
+        const stage = deps.db
+            .prepare("SELECT id FROM stages WHERE project_id=? AND kind='audio'")
+            .get(view.revision.projectId);
+        const id = deps.ids.next();
+        insertManifestPiece(deps.db, view.revision, {
+            key: recipe.key,
+            stageKind: "audio",
+            assetId: recipe.input.assetId,
+            fingerprint: recipe.fingerprint,
+            piece: {
+                id: deps.ids.next(),
+                stageId: z.string().parse(stage?.id),
+                kind: segment === "body" ? "chunk" : "segment",
+                idx: ordinal,
+                state: "done",
+                payload: JSON.stringify({
+                    provided: true,
+                    file: asset.path,
+                    logicalKey: recipe.key.slice(0, recipe.key.lastIndexOf(":")),
+                    logicalText: semantic[0],
+                    text: semantic[0],
+                    segment,
+                    provider: semantic[1][0],
+                    model: semantic[1][1],
+                    voice: semantic[1][2],
+                    requestFingerprint: recipe.requestFingerprint,
+                }),
+            },
+        }, id);
+        selectPieceRecord(deps.db, view.revision.id, recipe.key, id);
+        return;
+    }
+    if (recipe.input.kind !== "tts")
+        return;
+    if (view.pieces.some((one) => one.selected &&
+        one.available &&
+        one.key === recipe.key &&
+        one.fingerprint === recipe.fingerprint &&
+        one.piece.state === "done" &&
+        one.piece.idx === ordinal))
+        return;
+    const history = [...view.pieces, ...retainedNarrationPieces(deps, view.revision.projectId)];
+    const available = new Set(history
+        .filter((one) => one.available)
+        .flatMap((one) => (one.assetId === null ? [] : [one.assetId])));
+    const old = matchingNarrationPiece(recipe, history, available);
+    if (old === undefined)
+        return;
+    const input = recipe.input;
+    const recordId = deps.ids.next();
+    const payload = JSON.parse(old.piece.payload ?? "{}");
+    insertManifestPiece(deps.db, view.revision, {
+        key: recipe.key,
+        stageKind: "audio",
+        assetId: old.assetId,
+        fingerprint: recipe.fingerprint,
+        piece: {
+            ...old.piece,
+            id: deps.ids.next(),
+            idx: ordinal,
+            payload: JSON.stringify({
+                ...(typeof payload === "object" && payload !== null ? payload : {}),
+                text: input.text,
+                logicalKey: input.logicalKey,
+                logicalText: input.logicalText,
+                segment: input.segment,
+                requestFingerprint: recipe.requestFingerprint,
+                provider: input.provider,
+                model: input.model,
+                voice: input.voice,
+            }),
+        },
+    }, recordId);
+    selectPieceRecord(deps.db, view.revision.id, recipe.key, recordId);
+}
+export function narrationOrdinal(recipes, key) {
+    const entry = /^audio:(intro|outro):([0-9]+)$/.exec(key);
+    if (entry !== null)
+        return Number(entry[2]) * 2 - (entry[1] === "intro" ? 1 : 0);
+    return (recipes
+        .filter((row) => /^audio:body:.+:[0-9]+$/.test(row.key))
+        .findIndex((row) => row.key === key) + 1);
+}

package/dist/slices/rebuild/runtime-plan.js

@@ -0,0 +1,103 @@
+import { readFileSync } from "node:fs";
+import { z } from "zod";
+import { catalogueSchema } from "../../catalog/schema.js";
+import { getRevisionView } from "../revisions/view.js";
+import { outputPath } from "../storage/layout.js";
+import { referencedAssetsAvailable, retainedNarrationPieces } from "./narration-history.js";
+import { applyProvidedReviews } from "./provided-review.js";
+import { textRecipes } from "./recipe-text.js";
+import { planRevisionWork } from "./recipe-work.js";
+export function executionCatalogue(catalogue, config) {
+    const choices = [config.llm, config.audio, config.images];
+    const uses = (model) => choices.some((choice) => choice?.provider === model.provider && choice.model === model.id);
+    const providerIds = new Set(choices.flatMap((choice) => (choice ? [choice.provider] : [])));
+    return {
+        ...catalogue,
+        providers: Object.fromEntries(Object.entries(catalogue.providers).filter(([id]) => providerIds.has(id))),
+        llm: catalogue.llm.filter(uses),
+        tts: catalogue.tts.filter(uses),
+        image: catalogue.image.filter(uses),
+    };
+}
+export function savedCatalogue(value) {
+    return catalogueSchema.parse(JSON.parse(z.string().parse(value)));
+}
+export function executionPlan(deps, view, catalogue) {
+    const textOutput = (role) => {
+        const row = view.outputs.find((output) => output.selected && output.available && output.output.role === role);
+        return row === undefined
+            ? null
+            : readFileSync(outputPath(deps.paths, view.revision.projectId, row.output.path), "utf8");
+    };
+    const payloads = view.pieces.filter((piece) => piece.selected && piece.piece.state === "done" && piece.piece.payload !== null);
+    const resolved = {
+        articleMarkdown: view.articleMarkdown ?? textOutput("article_md"),
+        researchNotes: textOutput("notes"),
+    };
+    const research = matchingResearch({
+        config: view.revision.config,
+        content: view.revision.content,
+        manifest: view,
+        catalogue,
+        resolved,
+    }, payloads);
+    const history = retainedNarrationPieces(deps, view.revision.projectId);
+    const available = new Set([
+        ...history
+            .filter((row) => row.available && row.assetId !== null)
+            .flatMap((row) => (row.assetId === null ? [] : [row.assetId])),
+        ...referencedAssetsAvailable(deps, view.revision.projectId, Object.values(view.revision.content.narrationOverrides).flatMap((value) => value.kind === "asset" ? [value.assetId] : [])),
+        ...view.outputs.filter((row) => row.available).map((row) => row.assetId),
+        ...view.pieces
+            .filter((row) => row.available && row.assetId !== null)
+            .flatMap((row) => (row.assetId === null ? [] : [row.assetId])),
+    ]);
+    return applyProvidedReviews(deps.db, view.revision.id, planRevisionWork(view.revision, view, catalogue, available, {
+        ...resolved,
+        ...(research === undefined ? {} : { research }),
+    }, history));
+}
+export function executionView(deps, projectId, revisionId) {
+    const view = getRevisionView(deps, projectId, revisionId);
+    if (view === undefined)
+        return undefined;
+    const carried = new Set(deps.db
+        .prepare("SELECT r.piece_id FROM revision_work_reservations r JOIN project_heads h ON h.revision_id=r.revision_id AND h.project_id=r.project_id JOIN revision_work w ON w.id=r.work_id WHERE r.project_id=? AND w.revision_id=?")
+        .all(projectId, revisionId)
+        .map((row) => row.piece_id));
+    return {
+        ...view,
+        outputs: view.outputs.map((row) => ({
+            ...row,
+            selected: row.selected || (row.publicationId !== null && carried.has(row.publicationId)),
+        })),
+        pieces: view.pieces.map((row) => ({
+            ...row,
+            selected: row.selected || (row.publicationId !== null && carried.has(row.publicationId)),
+        })),
+    };
+}
+function matchingResearch(context, payloads) {
+    const planner = textRecipes(context).recipes.find((row) => row.key === "research:planner");
+    if (planner === undefined)
+        return undefined;
+    const savedPlanner = payloads.find((row) => row.key === planner.key && row.fingerprint === planner.fingerprint);
+    if (savedPlanner?.piece.payload === undefined || savedPlanner.piece.payload === null)
+        return undefined;
+    const { outline } = z
+        .object({ outline: z.array(z.string()) })
+        .parse(JSON.parse(savedPlanner.piece.payload));
+    const chapters = textRecipes({
+        ...context,
+        resolved: { ...context.resolved, research: { outline, findings: [] } },
+    }).recipes.filter((row) => row.key.startsWith("research:chapter:"));
+    const findings = chapters.flatMap((chapter) => {
+        const saved = payloads.find((row) => row.key === chapter.key && row.fingerprint === chapter.fingerprint);
+        if (saved?.piece.payload === undefined || saved.piece.payload === null)
+            return [];
+        return [
+            z.object({ title: z.string(), notes: z.string() }).parse(JSON.parse(saved.piece.payload)),
+        ];
+    });
+    return { outline, findings };
+}

package/dist/slices/rebuild/runtime-provider.js

@@ -0,0 +1,184 @@
+import { plainText } from "../article/plain.js";
+import { splitEndMatter } from "../article/split.js";
+import { observeNarration } from "../narration/live.js";
+import { chaptersFrom } from "../research/planner.js";
+import { sourcedAnswer } from "../research/synthesis.js";
+import { discardPreparedAssets, writeAsset } from "../storage/assets.js";
+import { outputPath } from "../storage/layout.js";
+import { probeDurationMs } from "../video/ffmpeg.js";
+import { executeArticleRequests } from "./runtime-article.js";
+import { frozenInstructions } from "./runtime-instructions.js";
+import { executionPlan, executionView, savedCatalogue } from "./runtime-plan.js";
+import { preparedResult, preparedTexts, publishResult } from "./runtime-publication.js";
+export async function executeProviderRecipe(deps, context, providers, piece) {
+    if (deps.db
+        .prepare("SELECT state FROM revision_work_pieces WHERE id=? AND work_id=?")
+        .get(piece.id, context.work.workId)?.state === "done")
+        return "done";
+    const input = piece.input;
+    const wrapped = providers.forPiece(piece.id);
+    if (input.kind === "llm") {
+        const answer = piece.key === "article:body"
+            ? await executeArticleRequests(deps, context, providers, piece)
+            : await wrapped.llm({
+                provider: input.provider,
+                model: input.model,
+                ...(input.thinking === null ? {} : { thinking: input.thinking }),
+                thinkingConfig: input.thinkingConfig,
+                messages: input.messages,
+                webSearch: input.webSearch,
+                previewLabel: piece.key,
+                check: (value) => checkAnswer(piece, value),
+            });
+        if (!answer.ok)
+            return "held";
+        await publishText(deps, context, piece, answer.value);
+        const segment = piece.key === "entry:intro:text"
+            ? "intro"
+            : piece.key === "entry:outro:text"
+                ? "outro"
+                : undefined;
+        deps.count?.("stage.completed", {
+            stage: context.work.kind,
+            provider: input.provider,
+            model: input.model,
+            ...(segment === undefined ? {} : { segment }),
+            tokensIn: answer.value.usage?.inputTokens ?? 0,
+            tokensOut: answer.value.usage?.outputTokens ?? 0,
+        });
+        return "done";
+    }
+    if (input.kind === "tts") {
+        const spoken = await wrapped.tts({ provider: input.provider, model: input.model, voiceId: input.voice, text: input.text }, observeNarration(deps.audioPreviews, context.work.projectId, piece.id, narrationLabel(deps, context, piece), { revisionId: context.work.revisionId, workId: context.work.workId, workPieceId: piece.id }));
+        if (!spoken.ok)
+            return "held";
+        const asset = writeAsset(deps, context.work.projectId, "narration.mp3", spoken.value.bytes);
+        try {
+            const durationMs = deps.count === undefined
+                ? undefined
+                : deps.measureAudio !== undefined
+                    ? await deps.measureAudio(outputPath(deps.paths, context.work.projectId, asset.path), context.signal)
+                    : deps.ffmpeg === undefined
+                        ? undefined
+                        : await probeDurationMs(deps.ffmpeg, outputPath(deps.paths, context.work.projectId, asset.path), context.signal, deps.log);
+            await publishResult(deps, context, piece, [], {
+                text: input.text,
+                logicalKey: input.logicalKey,
+                logicalText: input.logicalText,
+                segment: input.segment,
+                provider: input.provider,
+                model: input.model,
+                voice: input.voice,
+                ...(durationMs === undefined ? {} : { durationMs }),
+            }, asset);
+            deps.count?.("stage.completed", {
+                stage: "audio",
+                segment: input.segment,
+                provider: input.provider,
+                model: input.model,
+                ...(durationMs === undefined ? {} : { audioSeconds: durationMs / 1000 }),
+            });
+        }
+        finally {
+            discardPreparedAssets(deps, [asset]);
+        }
+        return "done";
+    }
+    if (input.kind === "image") {
+        const image = await wrapped.image({
+            provider: input.provider,
+            model: input.model,
+            prompt: input.prompt,
+            aspect: input.aspect,
+        });
+        if (!image.ok)
+            return "held";
+        const asset = writeAsset(deps, context.work.projectId, image.value.mime === "image/jpeg" ? "image.jpg" : "image.png", image.value.bytes);
+        const view = executionView(deps, context.work.projectId, context.work.revisionId);
+        const index = piece.key.startsWith("image:")
+            ? (view?.revision.content.imageOrder.indexOf(piece.key.slice(6)) ?? 0) + 1
+            : undefined;
+        const output = preparedResult(deps, context, piece, piece.key === "thumbnail:image" ? "thumbnail" : "image", asset, null, {
+            prompt: input.prompt,
+            provider: input.provider,
+            model: input.model,
+            ...(index === undefined ? {} : { index }),
+        });
+        await publishResult(deps, context, piece, [output], { prompt: input.prompt }, asset);
+        deps.count?.("stage.completed", {
+            stage: context.work.kind,
+            provider: input.provider,
+            model: input.model,
+            ...(piece.key === "thumbnail:image" ? { thumbnails: 1 } : { images: 1 }),
+        });
+        return "done";
+    }
+    throw new Error("Expected an exact provider recipe.");
+}
+function checkAnswer(piece, answer) {
+    if (answer.text.trim() === "")
+        return "The provider answered with nothing.";
+    if (piece.key === "research:planner")
+        return chaptersFrom(answer.text).length === 0 ? "The planner named no chapters." : undefined;
+    if (piece.key === "research:notes" || piece.key.startsWith("research:chapter:"))
+        return sourcedAnswer(piece.key, answer.text);
+    return undefined;
+}
+async function publishText(deps, context, piece, answer) {
+    if (piece.key === "article:body") {
+        const parts = splitEndMatter(answer.text);
+        await publishResult(deps, context, piece, preparedTexts(deps, context, piece, [
+            ["instructions", "instructions.md", frozenInstructions(deps, context, piece)],
+            ["article_md", "article.md", answer.text],
+            ["article_txt", "article.txt", plainText(parts.body)],
+            ...(parts.sources ? [["sources", "sources.md", parts.sources]] : []),
+            ...(parts.glossary ? [["glossary", "glossary.md", parts.glossary]] : []),
+        ]), { text: answer.text });
+        return;
+    }
+    if (piece.key === "research:notes") {
+        await publishResult(deps, context, piece, preparedTexts(deps, context, piece, [
+            ["notes", "notes.md", answer.text],
+            ["instructions", "instructions.md", frozenInstructions(deps, context, piece)],
+        ]), { text: answer.text });
+        return;
+    }
+    if (piece.key === "research:planner") {
+        await publishResult(deps, context, piece, [], { outline: chaptersFrom(answer.text) });
+        return;
+    }
+    if (piece.key.startsWith("research:chapter:")) {
+        const view = executionView(deps, context.work.projectId, context.work.revisionId);
+        const planner = view?.pieces.find((row) => row.selected && row.key === "research:planner");
+        const outline = JSON.parse(planner?.piece.payload ?? "{}");
+        const titles = typeof outline === "object" &&
+            outline !== null &&
+            "outline" in outline &&
+            Array.isArray(outline.outline)
+            ? outline.outline
+            : [];
+        const title = titles[Number(piece.key.split(":").at(-1)) - 1];
+        if (typeof title !== "string")
+            throw new Error("Research chapter has no pinned title.");
+        await publishResult(deps, context, piece, [], { title, notes: answer.text });
+        return;
+    }
+    await publishResult(deps, context, piece, preparedTexts(deps, context, piece, [
+        ["instructions", "instructions.md", frozenInstructions(deps, context, piece)],
+    ]), { text: answer.text, prompt: answer.text });
+}
+function narrationLabel(deps, context, piece) {
+    if (piece.input.kind !== "tts")
+        return "Narration";
+    const name = piece.input.segment === "body" ? "Body" : piece.input.segment === "intro" ? "Intro" : "Outro";
+    const view = executionView(deps, context.work.projectId, context.work.revisionId);
+    const row = deps.db
+        .prepare("SELECT recipe_context FROM revision_work WHERE id=?")
+        .get(context.work.workId);
+    if (view === undefined || row?.recipe_context === undefined || row.recipe_context === null)
+        return name;
+    const segment = piece.input.segment;
+    const parts = executionPlan(deps, view, savedCatalogue(row.recipe_context)).recipes.filter((recipe) => recipe.input.kind === "tts" && recipe.input.segment === segment);
+    const index = parts.findIndex((recipe) => recipe.key === piece.key);
+    return parts.length > 1 && index >= 0 ? `${name} part ${index + 1} of ${parts.length}` : name;
+}