@gentbajko/slopify 0.8.4 → 1.0.0

package/dist/slices/rebuild/recipe-legacy.js

@@ -0,0 +1,135 @@
+import { z } from "zod";
+import { buildRecipes } from "./recipe-build.js";
+export function baselineFingerprints(project, outputs, pieces, content, stages = []) {
+    const recipes = buildRecipes({
+        config: project.config,
+        content,
+        manifest: { outputs: [], pieces: [] },
+        resolved: {
+            articleMarkdown: content.articleMarkdown ?? project.config.provided.article ?? null,
+            researchNotes: project.config.provided.research ?? null,
+        },
+    });
+    const fingerprints = Object.fromEntries(recipes.map((value) => [value.key, value.fingerprint]));
+    for (const output of outputs) {
+        const key = legacyOutputWorkKey(output, project.config);
+        if (fingerprints[key] === undefined)
+            fingerprints[key] = `legacy:${output.id}`;
+    }
+    for (const piece of pieces) {
+        const key = legacyPieceKey(piece, legacyStageKind(piece, stages), outputs);
+        if (fingerprints[key] === undefined)
+            fingerprints[key] = `legacy:${piece.id}`;
+    }
+    return fingerprints;
+}
+export function legacyOutputSlot(output) {
+    return output.role === "image" ? `image:${output.id}` : `${output.stageKind}:${output.role}`;
+}
+export function legacyOutputWorkKey(output, config) {
+    switch (output.role) {
+        case "image":
+            return `image:${output.id}`;
+        case "thumbnail":
+            return "thumbnail:image";
+        case "notes":
+            return "research:notes";
+        case "article_md":
+        case "article_txt":
+        case "sources":
+        case "glossary":
+            return "article:body";
+        case "audio_body":
+            return config?.sources.audio === "provide" ? "audio:provided" : "audio:body:concat";
+        case "audio_intro":
+            return "audio:intro";
+        case "audio_outro":
+            return "audio:outro";
+        case "audio_export":
+            return "export:wav";
+        case "video":
+            return "export:video";
+        case "subtitles_srt":
+        case "subtitles_vtt":
+        case "subtitle_ass":
+        case "subtitle_font":
+            return "subtitles:files";
+        case "subtitle_words":
+            return "subtitles:timing";
+        case "render_params":
+        case "instructions":
+            return `${output.stageKind}:${output.role}`;
+    }
+}
+export function legacyPieceKey(piece, stageKind, outputs = []) {
+    if (piece.kind === "chunk")
+        return `audio:body:${piece.id}:1`;
+    if (piece.kind === "image")
+        return `image:${legacyImageOutput(piece, outputs)?.id ?? piece.id}`;
+    if (piece.kind === "prompt_written" && stageKind === "thumbnail")
+        return "thumbnail:prompt";
+    if (piece.kind === "segment") {
+        const category = piece.idx === 1 ? "intro" : "outro";
+        if (stageKind === "article")
+            return `entry:${category}:text`;
+        if (stageKind === "audio")
+            return `audio:${category}`;
+    }
+    return `${stageKind}:${piece.kind}:${piece.id}`;
+}
+function legacyStageKind(piece, stages) {
+    const stage = stages.find((row) => row.id === piece.stageId);
+    if (stage !== undefined)
+        return stage.kind;
+    switch (piece.kind) {
+        case "chunk":
+            return "audio";
+        case "chapter":
+            return "research";
+        case "image":
+            return "images";
+        case "prompt_written":
+            return "thumbnail";
+        case "article_written":
+        case "segment":
+            return "article";
+    }
+}
+export function legacyImageOutput(piece, outputs) {
+    if (piece.kind !== "image")
+        return undefined;
+    const parsed = legacyImageMetadata.safeParse(piece.payload);
+    if (!parsed.success)
+        return undefined;
+    const { file, prompt, promptName } = parsed.data;
+    const matches = outputs.filter((row) => row.role === "image" &&
+        (file !== undefined
+            ? row.path === file
+            : typeof prompt === "string" &&
+                row.meta.index === piece.idx &&
+                row.meta.prompt === prompt &&
+                (typeof promptName !== "string" ||
+                    row.meta.promptName === undefined ||
+                    row.meta.promptName === promptName)));
+    if (matches.length > 1)
+        throw new Error(`Legacy image piece ${piece.id} matches multiple outputs.`);
+    return matches[0];
+}
+const legacyImageMetadata = z
+    .string()
+    .transform((value, context) => {
+    try {
+        return JSON.parse(value);
+    }
+    catch (error) {
+        if (!(error instanceof SyntaxError))
+            throw error;
+        context.addIssue({ code: "custom", message: "Invalid legacy piece JSON." });
+        return z.NEVER;
+    }
+})
+    .pipe(z.object({
+    file: z.string().optional(),
+    prompt: z.unknown().optional(),
+    promptName: z.unknown().optional(),
+}));

package/dist/slices/rebuild/recipe-model.js

@@ -0,0 +1,77 @@
+import { z } from "zod";
+import { fingerprint } from "../../kernel/runner/work.js";
+import { narrationRegenerationToken, narrationRequestFingerprint } from "../narration/plan.js";
+export const localOperations = [
+    "export-wav",
+    "wav2vec2-en-a19f851-v2-omissions",
+    "automatic-cues-v1",
+    "manual-cues-v1",
+    "subtitle-files-v1",
+    "render-video",
+    "render-selected-video",
+    "provided-notes",
+    "provided-article",
+    "manual-article",
+    "entry-text",
+    "concat-narration",
+];
+export const deferredOperations = [
+    "research-synthesis",
+    "article",
+    "entry:intro:text",
+    "entry:outro:text",
+    "thumbnail-prompt",
+    "thumbnail-image",
+    "body-narration",
+    "intro-narration",
+    "outro-narration",
+    "resolve-revision-recipe",
+];
+export function recipe(context, key, stage, input, dependsOn = [], options = {}) {
+    const requestFingerprint = input.kind === "tts"
+        ? narrationRequestFingerprint({
+            ...input,
+            wholeText: input.wholeRequest === true ? input.logicalText : null,
+        })
+        : fingerprint(JSON.parse(JSON.stringify(input, (_key, value) => typeof value === "number" ? z.number().finite().parse(value) : value)));
+    const workFingerprint = fingerprint([
+        requestFingerprint,
+        input.kind === "tts"
+            ? narrationRegenerationToken(context.content.regenerationTokens, input.logicalKey, input.segment)
+            : (context.content.regenerationTokens[options.tokenKey ?? key] ?? null),
+    ]);
+    return {
+        key,
+        stage,
+        kind: options.kind ??
+            (input.kind === "llm" ||
+                input.kind === "tts" ||
+                input.kind === "image" ||
+                input.kind === "deferred"
+                ? "provider"
+                : input.kind),
+        input,
+        requestFingerprint,
+        fingerprint: workFingerprint,
+        logicalFingerprint: options.logicalFingerprint ?? workFingerprint,
+        dependsOn,
+        unresolved: options.unresolved ?? false,
+        deferred: input.kind === "deferred",
+    };
+}
+export function selectedAsset(context, key, desired) {
+    const output = context.manifest.outputs.find((row) => row.workKey === key && row.state === "ready" && selectedReference(row));
+    if (output !== undefined)
+        return output.assetId;
+    const piece = context.manifest.pieces.find((row) => row.key === key &&
+        row.piece.state === "done" &&
+        selectedReference(row) &&
+        (desired === undefined || row.fingerprint === desired));
+    return piece?.assetId ?? null;
+}
+export function resourceIdentity(context, value) {
+    return [value.fingerprint, selectedAsset(context, value.key, value.fingerprint)];
+}
+export function selectedReference(value) {
+    return !("selected" in value) || value.selected === true;
+}

package/dist/slices/rebuild/recipe-provider-choice.js

@@ -0,0 +1,14 @@
+export function recipeProviderChoice(recipe, config) {
+    const input = recipe.input;
+    if (input.kind === "llm" || input.kind === "tts" || input.kind === "image")
+        return { ...input, family: input.kind };
+    if (input.kind !== "deferred")
+        return undefined;
+    const family = recipe.stage === "audio"
+        ? "tts"
+        : recipe.stage === "images" || input.operation === "thumbnail-image"
+            ? "image"
+            : "llm";
+    const choice = family === "tts" ? config.audio : family === "image" ? config.images : config.llm;
+    return choice === undefined ? undefined : { ...choice, family };
+}

package/dist/slices/rebuild/recipe-save.js

@@ -0,0 +1,221 @@
+import { stageKinds } from "../../kernel/pipeline.js";
+import { allowedSources, normaliseDraft, silenceGapSecondsMax, } from "../admission/rules.js";
+import { render } from "../admission/substitute.js";
+import { buildRecipes } from "./recipe-build.js";
+import { selectedReference } from "./recipe-model.js";
+import { validateRecipeInputs } from "./recipe-validation.js";
+export function normalizeArticleIntent(base, edit) {
+    const restart = edit.regenerate?.includes("article:body") === true ||
+        (base.revision.config.sources.article === "provide" &&
+            edit.config.sources.article === "generate");
+    const submittedText = edit.content.articleEdited === true ||
+        edit.content.articleMarkdown !== base.revision.content.articleMarkdown;
+    const changed = submittedText && edit.content.articleMarkdown !== base.articleMarkdown;
+    return {
+        ...edit.content,
+        articleEdited: restart
+            ? false
+            : changed
+                ? true
+                : (base.revision.content.articleEdited ?? false),
+    };
+}
+export function planRevision(base, edit, prepared = { outputs: [], pieces: [] }, inspected = []) {
+    const content = normalizeImages(normalizeArticleIntent(base, edit));
+    const fields = [
+        ...validateRevisionEdit(edit.config, content),
+        ...validateCues(content),
+        ...validateRecipeInputs(edit.config, content),
+    ];
+    if (fields.length > 0)
+        return { ok: false, fields };
+    const normalized = normaliseDraft(edit.config);
+    const rendered = { ...edit.config.rendered };
+    for (const [key, raw] of Object.entries(content.promptTemplates))
+        if (raw !== null)
+            rendered[key] = render(raw, normalized.values);
+    const config = {
+        ...normalized,
+        rendered,
+        sources: {
+            ...normalized.sources,
+            images: edit.config.sources.images === "off"
+                ? "off"
+                : content.imageOrder.some((key) => content.imageDefinitions[key]?.source === "generate")
+                    ? "generate"
+                    : "provide",
+        },
+        provided: {
+            ...edit.config.provided,
+            ...(edit.config.sources.article === "provide"
+                ? { article: content.articleMarkdown ?? edit.config.provided.article ?? "" }
+                : {}),
+        },
+    };
+    const manifest = {
+        outputs: base.outputs.filter(selectedReference),
+        pieces: base.pieces.filter(selectedReference),
+    };
+    const oldContext = {
+        config: base.revision.config,
+        content: base.revision.content,
+        manifest,
+        resolved: {
+            articleMarkdown: base.articleMarkdown,
+            researchNotes: base.revision.config.provided.research ?? null,
+        },
+    };
+    const old = buildRecipes(oldContext);
+    const replacementKeys = new Set(prepared.outputs.map((row) => row.workKey));
+    const proposedManifest = {
+        outputs: [
+            ...manifest.outputs
+                .filter((row) => !replacementKeys.has(row.workKey))
+                .map((row) => inspected.find((one) => one.slot === row.slot && one.assetId === row.assetId) ?? row),
+            ...prepared.outputs,
+        ],
+        pieces: [
+            ...manifest.pieces.filter((row) => !prepared.pieces.some((one) => one.key === row.key)),
+            ...prepared.pieces,
+        ],
+    };
+    const proposedContext = {
+        ...oldContext,
+        manifest: proposedManifest,
+        config,
+        content,
+        resolved: { ...oldContext.resolved, researchNotes: config.provided.research ?? null },
+    };
+    const preliminary = buildRecipes(proposedContext);
+    const articleChanged = old.find((row) => row.key === "article:body")?.fingerprint !==
+        preliminary.find((row) => row.key === "article:body")?.fingerprint;
+    const draftRecipes = config.sources.article === "generate" && !content.articleEdited && articleChanged
+        ? buildRecipes({
+            ...proposedContext,
+            resolved: { ...proposedContext.resolved, articleMarkdown: null },
+        })
+        : preliminary;
+    const changed = new Set(draftRecipes
+        .filter((row) => old.find((value) => value.key === row.key)?.fingerprint !== row.fingerprint)
+        .map((row) => row.key));
+    const desired = buildRecipes({
+        ...proposedContext,
+        manifest: {
+            outputs: proposedManifest.outputs.filter((row) => !changed.has(row.workKey) || prepared.outputs.includes(row)),
+            pieces: proposedManifest.pieces.filter((row) => !changed.has(row.key) || prepared.pieces.includes(row)),
+        },
+        resolved: {
+            ...proposedContext.resolved,
+            articleMarkdown: config.sources.article === "generate" && !content.articleEdited && articleChanged
+                ? null
+                : proposedContext.resolved.articleMarkdown,
+        },
+    });
+    const fingerprints = Object.fromEntries(desired.map((row) => [row.key, row.fingerprint]));
+    const outputs = proposedManifest.outputs.map((row) => {
+        const next = desired.find((value) => value.key === row.workKey);
+        const unchanged = next !== undefined &&
+            old.find((value) => value.key === row.workKey)?.fingerprint === next.fingerprint;
+        const state = next === undefined
+            ? row.state
+            : prepared.outputs.includes(row) || (unchanged && row.state === "ready")
+                ? "ready"
+                : row.fingerprint === next.fingerprint
+                    ? row.state
+                    : next.kind === "provided"
+                        ? "review"
+                        : "outdated";
+        return {
+            ...row,
+            state,
+            fingerprint: state === "ready" && next !== undefined ? next.fingerprint : row.fingerprint,
+        };
+    });
+    return {
+        ok: true,
+        config,
+        content,
+        fingerprints,
+        manifest: { outputs, pieces: proposedManifest.pieces },
+        baseFingerprints: Object.fromEntries(old.map((row) => [row.key, row.fingerprint])),
+        recipes: desired,
+    };
+}
+function normalizeImages(content) {
+    return {
+        ...content,
+        imageDefinitions: Object.fromEntries(Object.entries(content.imageDefinitions).map(([key, value]) => [
+            key,
+            value.source === "provide" ? { ...value, prompt: null, templateKey: null } : value,
+        ])),
+    };
+}
+export function validateRevisionEdit(config, content) {
+    const fields = [];
+    for (const kind of stageKinds)
+        if (!allowedSources[kind].includes(config.sources[kind]))
+            fields.push({
+                field: `sources.${kind}`,
+                message: `The ${kind} stage cannot be set to ${config.sources[kind]}.`,
+            });
+    if (config.sources.images === "provide" &&
+        content.imageOrder.some((key) => content.imageDefinitions[key]?.source === "generate"))
+        fields.push({
+            field: "sources.images",
+            message: "Replace or remove generated images before choosing Provide.",
+        });
+    if (config.sources.images !== "off" && content.imageOrder.length === 0)
+        fields.push({
+            field: "content.imageOrder",
+            message: "Keep at least one image, or turn Images and Video off.",
+        });
+    if (config.sources.images === "off" && config.sources.video !== "off")
+        fields.push({ field: "sources.video", message: "Turn Video off when Images is off." });
+    if (new Set(content.imageOrder).size !== content.imageOrder.length ||
+        content.imageOrder.some((key) => content.imageDefinitions[key] === undefined) ||
+        Object.keys(content.imageDefinitions).some((key) => !content.imageOrder.includes(key)))
+        fields.push({
+            field: "content.imageOrder",
+            message: "Every image must appear exactly once in the image order.",
+        });
+    if (config.title.trim() === "" || config.title.length > 200)
+        fields.push({ field: "title", message: "A title between 1 and 200 characters is required." });
+    if (!Number.isFinite(config.silenceGapSeconds) ||
+        config.silenceGapSeconds < 0 ||
+        config.silenceGapSeconds > silenceGapSecondsMax)
+        fields.push({
+            field: "silenceGapSeconds",
+            message: `Use a gap between 0 and ${silenceGapSecondsMax} seconds.`,
+        });
+    if (config.sources.article === "generate" &&
+        content.articleEdited === true &&
+        !content.articleMarkdown?.trim())
+        fields.push({ field: "content.articleMarkdown", message: "Enter article text." });
+    if (config.sources.article === "provide" &&
+        !(content.articleMarkdown ?? config.provided.article)?.trim())
+        fields.push({ field: "provided.article", message: "Paste the article." });
+    if (config.sources.audio === "off" &&
+        config.subtitles !== undefined &&
+        config.subtitles.mode !== "off")
+        fields.push({ field: "subtitles.mode", message: "Subtitles need narration audio." });
+    return fields;
+}
+function validateCues(content) {
+    const cues = content.subtitleCues?.cues;
+    if (cues === undefined)
+        return [];
+    const fields = [];
+    for (const [index, cue] of cues.entries()) {
+        const path = `content.subtitleCues.cues.${index}`;
+        if (!cue.text.trim())
+            fields.push({ field: `${path}.text`, message: "Enter subtitle text." });
+        if (!Number.isFinite(cue.start) || cue.start < 0 || cue.start < (cues[index - 1]?.end ?? 0))
+            fields.push({ field: `${path}.start`, message: "Cues must be ordered and cannot overlap." });
+        if (!Number.isFinite(cue.end) || cue.end <= cue.start)
+            fields.push({
+                field: `${path}.end`,
+                message: "End must follow start.",
+            });
+    }
+    return fields;
+}

package/dist/slices/rebuild/recipe-text.js

@@ -0,0 +1,170 @@
+import { z } from "zod";
+import { fingerprint } from "../../kernel/runner/work.js";
+import { render } from "../admission/substitute.js";
+import { articleMessages, continuationMessages } from "../article/continuation.js";
+import { plainText } from "../article/plain.js";
+import { segmentMessages } from "../article/segments.js";
+import { splitEndMatter } from "../article/split.js";
+import { plannerMessages, subAgentMessages } from "../research/planner.js";
+import { synthesisMessages } from "../research/synthesis.js";
+import { thumbnailMessages } from "../thumbnail/by-llm.js";
+import { recipe, selectedReference, } from "./recipe-model.js";
+export function renderedPrompt(context, key) {
+    const raw = context.content.promptTemplates[key];
+    return raw === undefined || raw === null
+        ? (context.config.rendered[key] ?? "")
+        : render(raw, context.config.values);
+}
+export function llmInput(context, messages, webSearch = false) {
+    const choice = context.config.llm;
+    const model = context.catalogue?.llm.find((row) => row.provider === choice?.provider &&
+        row.id === choice.model &&
+        row.enabled &&
+        !row.deprecated);
+    return {
+        kind: "llm",
+        version: 1,
+        provider: choice?.provider ?? "",
+        model: choice?.model ?? "",
+        thinking: choice?.thinking ?? null,
+        thinkingConfig: choice?.thinking === undefined ? null : (model?.llm.thinking?.[choice.thinking] ?? null),
+        messages,
+        webSearch,
+    };
+}
+export function selectedText(context, key, field) {
+    const piece = context.manifest.pieces.find((row) => row.key === key &&
+        selectedReference(row) &&
+        row.stageKind === (key.startsWith("entry:") ? "article" : "thumbnail") &&
+        row.piece.state === "done");
+    if (piece?.piece.payload === null || piece === undefined)
+        return null;
+    const parsed = z
+        .object({ text: z.string().optional(), prompt: z.string().optional() })
+        .parse(JSON.parse(piece.piece.payload));
+    return parsed[field]?.trim() ?? null;
+}
+export function textRecipes(context) {
+    const { config, content, resolved } = context;
+    const recipes = [];
+    const brief = { articlePrompt: renderedPrompt(context, "article"), values: config.values };
+    let research;
+    if (config.sources.research === "generate") {
+        const planner = recipe(context, "research:planner", "research", llmInput(context, plannerMessages(brief)));
+        recipes.push(planner);
+        const chapters = resolved.research?.outline.map((title, index) => recipe(context, `research:chapter:${index + 1}`, "research", llmInput(context, subAgentMessages(brief, title, resolved.research?.outline ?? []), true), [planner.key])) ?? [];
+        recipes.push(...chapters);
+        const findings = resolved.research?.findings;
+        research =
+            findings !== undefined && findings.length === chapters.length && chapters.length > 0
+                ? recipe(context, "research:notes", "research", llmInput(context, synthesisMessages(brief, findings)), chapters.map((row) => row.key))
+                : recipe(context, "research:notes", "research", {
+                    kind: "deferred",
+                    version: 1,
+                    operation: "research-synthesis",
+                    template: [planner.fingerprint, chapters.map((row) => row.fingerprint)],
+                }, [planner.key, ...chapters.map((row) => row.key)]);
+        recipes.push(research);
+    }
+    else if (config.sources.research === "provide") {
+        research = recipe(context, "research:notes", "research", {
+            kind: "local",
+            version: 1,
+            operation: "provided-notes",
+            values: config.provided.research ?? "",
+        });
+        recipes.push(research);
+    }
+    const articleMarkdown = config.sources.article === "provide" || content.articleEdited === true
+        ? (content.articleMarkdown ?? config.provided.article ?? "")
+        : resolved.articleMarkdown;
+    const notes = config.sources.research === "off"
+        ? null
+        : config.sources.research === "provide"
+            ? (config.provided.research ?? null)
+            : resolved.researchNotes;
+    const messages = articleMessages({
+        articlePrompt: brief.articlePrompt,
+        ...(notes === null ? {} : { notes }),
+    });
+    const article = config.sources.article === "provide" || content.articleEdited === true
+        ? recipe(context, "article:body", "article", {
+            kind: "local",
+            version: 1,
+            operation: content.articleEdited ? "manual-article" : "provided-article",
+            values: articleMarkdown,
+        })
+        : recipe(context, "article:body", "article", config.sources.research === "generate" && notes === null
+            ? {
+                kind: "deferred",
+                version: 1,
+                operation: "article",
+                template: [llmInputFingerprint(context, messages), research?.fingerprint ?? null],
+            }
+            : llmInput(context, messages), research === undefined ? [] : [research.key]);
+    recipes.push(article);
+    if (resolved.articleContinuation !== undefined &&
+        config.sources.article === "generate" &&
+        !content.articleEdited)
+        recipes.push(recipe(context, "article:continuation", "article", llmInput(context, continuationMessages(messages, resolved.articleContinuation)), [article.key]));
+    const articleText = articleMarkdown === null ? null : plainText(splitEndMatter(articleMarkdown).body);
+    const entries = {};
+    for (const category of ["intro", "outro"]) {
+        const choice = config[category];
+        if (choice === undefined || config.sources.audio !== "generate")
+            continue;
+        const prompt = renderedPrompt(context, category);
+        const key = `entry:${category}:text`;
+        const input = choice.mode === "text"
+            ? { kind: "local", version: 1, operation: "entry-text", values: prompt }
+            : articleText === null
+                ? {
+                    kind: "deferred",
+                    version: 1,
+                    operation: key,
+                    template: [
+                        llmInputFingerprint(context, segmentMessages(prompt, config, "")),
+                        article.fingerprint,
+                    ],
+                }
+                : llmInput(context, segmentMessages(prompt, config, articleText));
+        const value = recipe(context, key, "article", input, choice.mode === "llm" ? [article.key] : []);
+        recipes.push(value);
+        entries[category] = {
+            recipe: value,
+            text: choice.mode === "text" ? prompt : matchingText(context, value, "text"),
+        };
+    }
+    if (config.sources.thumbnail === "prompt_by_llm") {
+        const prompt = renderedPrompt(context, "thumbnailPrompt");
+        const messages = thumbnailMessages({
+            instruction: prompt,
+            title: config.title,
+            values: config.values,
+            format: config.format,
+            article: articleText ?? "",
+        });
+        recipes.push(recipe(context, "thumbnail:prompt", "thumbnail", articleText === null
+            ? {
+                kind: "deferred",
+                version: 1,
+                operation: "thumbnail-prompt",
+                template: [llmInputFingerprint(context, messages), article.fingerprint],
+            }
+            : llmInput(context, messages), [article.key]));
+    }
+    return { recipes, articleText, article, entries };
+}
+function llmInputFingerprint(context, messages) {
+    return fingerprint(JSON.parse(JSON.stringify(llmInput(context, messages))));
+}
+export function matchingText(context, value, field) {
+    const logical = context.catalogue === undefined
+        ? value.fingerprint
+        : recipe({ content: context.content }, value.key, value.stage, value.input.kind === "llm" ? { ...value.input, thinkingConfig: null } : value.input, value.dependsOn).fingerprint;
+    if (!context.manifest.pieces.some((row) => row.key === value.key &&
+        selectedReference(row) &&
+        (row.fingerprint === value.fingerprint || row.fingerprint === logical)))
+        return null;
+    return selectedText(context, value.key, field);
+}