@gentbajko/slopify 0.5.0 → 0.6.1

package/dist/slices/article/run.js

@@ -1,13 +1,14 @@
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
+import { z } from "zod";
+import { messageRoles } from "../../kernel/ports/llm.js";
 import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
 import { projectById } from "../admission/repo.js";
 import { outputPath } from "../storage/layout.js";
 import { outputsOf } from "../storage/repo.js";
 import { storeText } from "../storage/staging.js";
-import { noTokens, plusUsage } from "../telemetry/model.js";
 import { writeArticle } from "./continuation.js";
+import { categories, instructionsText, keepSegment, writeSegment } from "./segments.js";
 import { storeArticleText } from "./store.js";
-const categories = ["intro", "outro"];
 export async function runArticle(deps, context, providers) {
     const { projectId } = context.stage;
     const project = projectById(deps.db, projectId);
@@ -23,34 +24,19 @@ export async function runArticle(deps, context, providers) {
     }
     const notes = researchNotes(deps, projectId);
     const brief = { articlePrompt, ...(notes === undefined ? {} : { notes }) };
-    const written = await writeArticle(providers, choice, brief, (text) => {
-        // The page shows the article as it is written. The idle timeout is restarted by the wrapper
-        // on the same events, not here. ceiling: the partial text of a failed attempt is discarded,
-        // but the deltas already sent cannot be unsent, so a retry mid-stream leaves the page
-        // appending the second telling under the first. What the project keeps is still the
-        // successful attempt's text alone; the upgrade is an event telling the page to start the
-        // article again.
-        context.emit({ type: "article.delta", projectId, text });
-    });
-    // The markdown exactly as the model produced it, and the plain-text narration source.
-    // The end matter is kept out of the narration and stored beside the article instead.
-    const store = { projectId, stageKind: "article" };
-    storeText(deps, { ...store, role: "article_md", text: written.markdown });
-    const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
-    // The article and each entry text are counted as units of their own, so the article's
-    // event goes out as soon as its text is stored, carrying the tokens of the first call
-    // and its continuations.
-    deps.count("stage.completed", {
-        stage: "article",
-        provider: choice.provider,
-        model: choice.model,
-        ...written.tokens,
-    });
-    const sent = [...written.sent];
+    const completed = await articleBody(deps, context, providers, choice, brief);
+    const { narration } = completed;
+    const sent = [...completed.sent];
     for (const [index, category] of categories.entries()) {
+        if (project.config.sources.audio !== "generate")
+            break;
+        if (completed.resumed &&
+            piecesOf(deps.db, context.stage.id, "segment").some((piece) => piece.idx === index + 1 && piece.state === "done"))
+            continue;
         const segment = await writeSegment(providers, choice, project.config, category, narration, sent);
         if (segment !== undefined) {
             keepSegment(deps, context, segment, index + 1);
+            checkpointArticle(deps, context, sent);
             // One event per intro/outro text, named by its segment. A text-mode entry is
             // rendered rather than written, so it made no call, names no provider and reports
             // zero tokens rather than an estimate.
@@ -63,89 +49,72 @@ export async function runArticle(deps, context, providers) {
         }
     }
     // The exact messages sent, the continuations and the entry calls among them.
-    storeText(deps, { ...store, role: "instructions", text: instructionsText(sent) });
+    storeText(deps, {
+        projectId,
+        stageKind: "article",
+        role: "instructions",
+        text: instructionsText(sent),
+    });
     deps.log.write("info", "article.done", {
         projectId,
         stage: "article",
-        detail: `${String(written.sent.length - 1)} continuations, ${String(narration.length)} characters to narrate`,
+        detail: `${String(narration.length)} characters to narrate`,
     });
 }
-// For each picked entry in LLM mode, one call with the filled entry as instruction plus the
-// title, keyword values, and the plain-text article ... Text-mode entries are stored as
-// rendered, with no call.
-async function writeSegment(providers, choice, config, category, article, sent) {
-    const picked = config[category];
-    if (picked === undefined) {
-        return undefined;
-    }
-    const body = config.rendered[category];
-    if (body === undefined) {
-        throw new Error(`the run has no rendered ${category} text`);
-    }
-    const common = { category, name: picked.name, mode: picked.mode };
-    if (picked.mode === "text") {
-        return { ...common, text: body, tokens: noTokens };
+// Once the article is stored, a pause during end matter resumes that end matter
+// instead of buying and replacing the article again. Re-run clears this checkpoint.
+const articleCheckpoint = z.object({
+    sent: z.array(z.object({
+        label: z.string(),
+        messages: z.array(z.object({ role: z.enum(messageRoles), content: z.string() })),
+    })),
+});
+async function articleBody(deps, context, providers, choice, brief) {
+    const { projectId } = context.stage;
+    const saved = piecesOf(deps.db, context.stage.id, "article_written")[0];
+    const outputs = outputsOf(deps.db, projectId);
+    const plain = outputs.find((output) => output.role === "article_txt");
+    const markdown = outputs.find((output) => output.role === "article_md");
+    if (saved?.payload !== null &&
+        saved?.payload !== undefined &&
+        plain &&
+        markdown &&
+        existsSync(outputPath(deps.paths, projectId, plain.path)) &&
+        existsSync(outputPath(deps.paths, projectId, markdown.path))) {
+        return {
+            narration: readFileSync(outputPath(deps.paths, projectId, plain.path), "utf8"),
+            sent: articleCheckpoint.parse(JSON.parse(saved.payload)).sent,
+            resumed: true,
+        };
     }
-    const messages = segmentMessages(body, config, article);
-    sent.push({ label: label(category), messages });
-    const answer = await providers.llm({
+    const written = await writeArticle(providers, choice, brief, (text) => {
+        context.emit({ type: "article.delta", projectId, text });
+    });
+    storeText(deps, { projectId, stageKind: "article", role: "article_md", text: written.markdown });
+    const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
+    checkpointArticle(deps, context, written.sent);
+    deps.count("stage.completed", {
+        stage: "article",
         provider: choice.provider,
         model: choice.model,
-        messages,
-        // An entry that answers with nothing is a failed attempt like any
-        // other, and the wrapper is what retries it.
-        check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
+        ...written.tokens,
     });
-    return { ...common, text: answer.text.trim(), tokens: plusUsage(noTokens, answer.usage) };
+    return { narration, sent: written.sent, resumed: false };
 }
-function segmentMessages(body, config, article) {
-    const values = Object.entries(config.values);
-    return [
-        {
-            role: "user",
-            content: [
-                body,
-                "",
-                `Video title: ${config.title}`,
-                "",
-                "Keyword values for this run:",
-                "",
-                values.length === 0
-                    ? "(none)"
-                    : values.map(([name, value]) => `${name}: ${value}`).join("\n"),
-                "",
-                "The article this video narrates:",
-                "",
-                article,
-            ].join("\n"),
-        },
-    ];
-}
-// The segment rows are keyed by `idx` within the stage, so a stage that runs again after
-// a failure replaces what it wrote rather than colliding with it.
-function keepSegment(deps, context, segment, idx) {
-    // Written out rather than stringified whole: the caller hands in what the call cost as
-    // well, and the narration stage is what reads the piece - it carries the text to speak,
-    // nothing about the model that wrote it.
-    const payload = JSON.stringify({
-        category: segment.category,
-        name: segment.name,
-        mode: segment.mode,
-        text: segment.text,
-    });
-    const existing = piecesOf(deps.db, context.stage.id, "segment").find((piece) => piece.idx === idx);
-    if (existing === undefined) {
+function checkpointArticle(deps, context, sent) {
+    const existing = piecesOf(deps.db, context.stage.id, "article_written")[0];
+    const payload = JSON.stringify({ sent });
+    if (existing)
+        setPiece(deps.db, existing.id, "done", payload);
+    else
         insertPiece(deps.db, {
             id: deps.ids.next(),
             stageId: context.stage.id,
-            kind: "segment",
-            idx,
+            kind: "article_written",
+            idx: 1,
             state: "done",
             payload,
         });
-        return;
-    }
-    setPiece(deps.db, existing.id, "done", payload);
 }
 // Research writes its notes as an output of its own, and a provided research stage stores the
 // pasted text the same way, so one lookup covers both. No row means research was Off or
@@ -157,10 +126,3 @@ function researchNotes(deps, projectId) {
     }
     return readFileSync(outputPath(deps.paths, projectId, notes.path), "utf8");
 }
-function instructionsText(sent) {
-    const parts = sent.map((one) => `=== ${one.label} ===\n\n${one.messages.map((message) => message.content).join("\n\n")}`);
-    return `${parts.join("\n\n")}\n`;
-}
-function label(category) {
-    return category === "intro" ? "Intro" : "Outro";
-}

package/dist/slices/article/segments.js

@@ -0,0 +1,73 @@
+import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
+import { noTokens, plusUsage } from "../telemetry/model.js";
+export const categories = ["intro", "outro"];
+export async function writeSegment(providers, choice, config, category, article, sent) {
+    const picked = config[category];
+    if (picked === undefined)
+        return undefined;
+    const body = config.rendered[category];
+    if (body === undefined)
+        throw new Error(`the run has no rendered ${category} text`);
+    const common = { category, name: picked.name, mode: picked.mode };
+    if (picked.mode === "text")
+        return { ...common, text: body, tokens: noTokens };
+    if (choice === undefined)
+        throw new Error(`the ${category} has no LLM provider or model`);
+    const messages = segmentMessages(body, config, article);
+    sent.push({ label: category === "intro" ? "Intro" : "Outro", messages });
+    const answer = await providers.llm({
+        provider: choice.provider,
+        model: choice.model,
+        messages,
+        check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
+    });
+    return { ...common, text: answer.text.trim(), tokens: plusUsage(noTokens, answer.usage) };
+}
+function segmentMessages(body, config, article) {
+    const values = Object.entries(config.values);
+    return [
+        {
+            role: "user",
+            content: [
+                body,
+                "",
+                `Video title: ${config.title}`,
+                "",
+                "Keyword values for this run:",
+                "",
+                values.length === 0
+                    ? "(none)"
+                    : values.map(([name, value]) => `${name}: ${value}`).join("\n"),
+                "",
+                "The article this video narrates:",
+                "",
+                article,
+            ].join("\n"),
+        },
+    ];
+}
+export function keepSegment(deps, context, segment, idx) {
+    const payload = JSON.stringify({
+        category: segment.category,
+        name: segment.name,
+        mode: segment.mode,
+        text: segment.text,
+    });
+    const existing = piecesOf(deps.db, context.stage.id, "segment").find((piece) => piece.idx === idx);
+    if (existing === undefined) {
+        insertPiece(deps.db, {
+            id: deps.ids.next(),
+            stageId: context.stage.id,
+            kind: "segment",
+            idx,
+            state: "done",
+            payload,
+        });
+    }
+    else
+        setPiece(deps.db, existing.id, "done", payload);
+}
+export function instructionsText(sent) {
+    const parts = sent.map((one) => `=== ${one.label} ===\n\n${one.messages.map((message) => message.content).join("\n\n")}`);
+    return `${parts.join("\n\n")}\n`;
+}

package/dist/slices/cancel/index.js

@@ -1,5 +1,5 @@
 import { derive } from "../../kernel/runner/graph.js";
-import { finishStage, projectExists, stagesOf } from "../admission/repo.js";
+import { finishStage, projectExists, projectPaused, setProjectPaused, stagesOf, } from "../admission/repo.js";
 // Cancel on the project header. Every in-flight call of this project is aborted at once, every
 // `done` output and every finished piece is kept for the resume, and the project sits
 // `canceled` until the user retries a stage.
@@ -10,15 +10,21 @@ export async function cancelProject(deps, projectId) {
         return { ok: false, reason: "no-project" };
     }
     const before = stagesOf(deps.db, projectId);
+    const paused = projectPaused(deps.db, projectId);
+    if (paused) {
+        setProjectPaused(deps.db, projectId, false, deps.clock.now().toISOString());
+        deps.emit(projectId, { type: "project.updated", projectId });
+    }
     const running = before.filter((stage) => stage.state === "running").map((stage) => stage.kind);
-    if (running.length === 0) {
+    if (running.length === 0 && !paused) {
         // A second click is a no-op. Nothing is aborted and no state changes, so
         // the page is simply told what the project already reads.
         return { ok: true, canceled: [], state: derive(before) };
     }
     // Nothing waits for a response. The runner holds the controllers and its own
     // barrier, so a stage that finishes during this does not release its dependents.
-    await deps.abort(projectId);
+    if (running.length > 0)
+        await deps.abort(projectId);
     // The invariant: after cancel completes no stage of the project is `running`.
     // The runner writes that row as each aborted stage unwinds; this is the path where it
     // could not - a failed write is logged there and the run is left mid-flight otherwise.
@@ -41,9 +47,26 @@ export async function cancelProject(deps, projectId) {
             failureReason: canceledByUser,
         });
     }
+    // An abort can race a successful stage completion. Mark one remaining pending
+    // stage canceled so cancellation is explicit, rather than guessing from any
+    // completed stage (independent images can now finish before Article).
+    const stopped = stagesOf(deps.db, projectId);
+    const waiting = stopped.find((stage) => stage.state === "pending");
+    if (!stopped.some((stage) => stage.state === "canceled") && waiting !== undefined) {
+        finishStage(deps.db, waiting.id, "canceled", canceledByUser, deps.clock.now().toISOString());
+        deps.emit(projectId, {
+            type: "stage.state",
+            projectId,
+            stage: waiting.kind,
+            state: "canceled",
+            failureReason: canceledByUser,
+        });
+    }
     const settled = stagesOf(deps.db, projectId);
     const state = derive(settled);
-    if (after.some((stage) => stage.state === "running")) {
+    if (paused ||
+        after.some((stage) => stage.state === "running") ||
+        (waiting !== undefined && !stopped.some((stage) => stage.state === "canceled"))) {
         // Only when the sweep above moved a row: the runner already announced the state it
         // left the project in, and repeating it would tell every open page the same thing
         // twice.

package/dist/slices/control/index.js

@@ -0,0 +1,155 @@
+import { transact } from "../../kernel/db/tx.js";
+import { derive, satisfied } from "../../kernel/runner/graph.js";
+import { finishStage, projectById, resetStage, setProjectPaused, stagesOf, updateProjectConfig, } from "../admission/repo.js";
+import { clearUnfinishedAudio } from "../reruns/index.js";
+import { providers as providerCatalog } from "../settings/model.js";
+import { hasKey, listVoices } from "../settings/repo.js";
+import { withProjectControl } from "./lock.js";
+import { validateLocalProviderChanges, validateProviderChanges } from "./providers.js";
+export const providerCheckTimeoutMs = 15_000;
+async function providerFields(deps, changes) {
+    let timeout;
+    try {
+        return await Promise.race([
+            (async () => validateProviderChanges(changes, {
+                providers: await deps.providers(),
+                voices: listVoices(deps.db),
+                modelsFor: deps.modelsFor,
+            }))(),
+            new Promise((_resolve, reject) => {
+                timeout = setTimeout(() => reject(new Error("Provider catalog timed out")), providerCheckTimeoutMs);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+function changed(deps, id, wake = false) {
+    deps.emit(id, { type: "project.updated", projectId: id });
+    if (wake)
+        deps.runner.tick(id);
+    else
+        deps.emit(id, {
+            type: "project.state",
+            projectId: id,
+            state: derive(stagesOf(deps.db, id), projectById(deps.db, id)?.paused),
+        });
+}
+export function pauseProject(deps, id) {
+    return withProjectControl(deps.db, id, async () => {
+        const project = projectById(deps.db, id);
+        if (project === undefined)
+            return { ok: false, reason: "no-project" };
+        const before = stagesOf(deps.db, id);
+        if (before.every((stage) => satisfied(stage.state)))
+            return { ok: true };
+        const running = before.filter((stage) => stage.state === "running").map((stage) => stage.id);
+        if (project.paused === true && running.length === 0 && !deps.runner.hasInflight?.(id))
+            return { ok: true };
+        transact(deps.db, () => setProjectPaused(deps.db, id, true, deps.clock.now().toISOString()));
+        deps.emit(id, { type: "project.updated", projectId: id });
+        await deps.runner.abortProject(id, "pause");
+        // Covers a failed final row write and runners that report interruption as canceled.
+        for (const stage of stagesOf(deps.db, id)) {
+            if (running.includes(stage.id) && (stage.state === "running" || stage.state === "canceled")) {
+                finishStage(deps.db, stage.id, "pending", null, deps.clock.now().toISOString());
+                deps.emit(id, { type: "stage.state", projectId: id, stage: stage.kind, state: "pending" });
+            }
+        }
+        changed(deps, id);
+        return { ok: true };
+    });
+}
+export function resumeProject(deps, id) {
+    return withProjectControl(deps.db, id, () => {
+        const project = projectById(deps.db, id);
+        if (project === undefined)
+            return { ok: false, reason: "no-project" };
+        const stages = stagesOf(deps.db, id);
+        if (stages.some((stage) => stage.state === "running") || deps.runner.hasInflight?.(id))
+            return { ok: true };
+        if (project.paused !== true && stages.every((stage) => satisfied(stage.state)))
+            return { ok: true };
+        transact(deps.db, () => {
+            for (const stage of stages) {
+                if (stage.state === "failed" || stage.state === "canceled")
+                    resetStage(deps.db, stage.id);
+            }
+            setProjectPaused(deps.db, id, false, deps.clock.now().toISOString());
+        });
+        changed(deps, id, true);
+        return { ok: true };
+    });
+}
+function editable(db, id, runner) {
+    const project = projectById(db, id);
+    if (project === undefined)
+        return { ok: false, reason: "no-project" };
+    const stages = stagesOf(db, id);
+    if (stages.some((stage) => stage.state === "running") || runner.hasInflight?.(id))
+        return { ok: false, reason: "running" };
+    if (project.paused !== true && derive(stages) !== "failed")
+        return { ok: false, reason: "not-editable" };
+    return { ok: true };
+}
+export function changeProviders(deps, id, changes) {
+    return withProjectControl(deps.db, id, async () => {
+        const allowed = editable(deps.db, id, deps.runner);
+        if (!allowed.ok)
+            return allowed;
+        let fields;
+        try {
+            fields = await providerFields(deps, changes);
+        }
+        catch {
+            deps.log.write("warn", "project.providers", {
+                projectId: id,
+                detail: "Provider readiness or model catalog could not be loaded.",
+            });
+            return { ok: false, reason: "catalog-unavailable" };
+        }
+        if (fields.length > 0)
+            return { ok: false, reason: "invalid-providers", fields };
+        // Catalog/CLI reads await. Recheck after them before touching a project another
+        // action may have deleted, and never rebuild its saved prompts or upload choices.
+        const stillAllowed = editable(deps.db, id, deps.runner);
+        if (!stillAllowed.ok)
+            return stillAllowed;
+        const project = projectById(deps.db, id);
+        if (project === undefined)
+            return { ok: false, reason: "no-project" };
+        // Settings can change while a remote catalog is loading. Revalidate its local
+        // key and voice rows synchronously so the saved choice is usable at commit.
+        const localFields = validateLocalProviderChanges(changes, {
+            providers: providerCatalog.map((provider) => ({
+                ...provider,
+                readiness: provider.auth === "cli"
+                    ? { kind: "cli", installed: true }
+                    : { kind: "keyed", hasKey: hasKey(deps.db, provider.id) },
+            })),
+            voices: listVoices(deps.db),
+        });
+        if (localFields.length > 0)
+            return { ok: false, reason: "invalid-providers", fields: localFields };
+        const audioChanged = changes.audio !== undefined &&
+            (changes.audio.provider !== project.config.audio?.provider ||
+                changes.audio.model !== project.config.audio?.model ||
+                changes.audio.voice !== project.config.audio?.voice);
+        const orphaned = transact(deps.db, () => {
+            const files = audioChanged ? clearUnfinishedAudio(deps, id) : [];
+            updateProjectConfig(deps.db, id, {
+                ...project.config,
+                ...(changes.llm === undefined ? {} : { llm: changes.llm }),
+                ...(changes.audio === undefined ? {} : { audio: changes.audio }),
+                ...(changes.images === undefined ? {} : { images: changes.images }),
+            }, deps.clock.now().toISOString());
+            return files;
+        });
+        // The clear helper owns the row changes; files are unlinked only after commit.
+        for (const remove of orphaned)
+            remove();
+        deps.emit(id, { type: "project.updated", projectId: id });
+        return { ok: true };
+    });
+}

package/dist/slices/control/lock.js

@@ -0,0 +1,19 @@
+// Every project mutation uses the same queue. A pause owns it until provider calls
+// have unwound, so a second tab cannot resume or replace configuration mid-abort.
+const queues = new WeakMap();
+export function withProjectControl(db, projectId, work) {
+    let projects = queues.get(db);
+    if (projects === undefined) {
+        projects = new Map();
+        queues.set(db, projects);
+    }
+    const previous = projects.get(projectId);
+    const result = previous === undefined ? Promise.resolve().then(work) : previous.then(work, work);
+    projects.set(projectId, result);
+    const release = () => {
+        if (projects.get(projectId) === result)
+            projects.delete(projectId);
+    };
+    void result.then(release, release);
+    return result;
+}

package/dist/slices/control/providers.js

@@ -0,0 +1,59 @@
+import { z } from "zod";
+const choice = z
+    .object({
+    provider: z.string().trim().min(1).max(200),
+    model: z.string().trim().min(1).max(200),
+})
+    .strict();
+export const providerChangesSchema = z
+    .object({
+    llm: choice.optional(),
+    audio: choice.extend({ voice: z.string().trim().min(1).max(200) }).optional(),
+    images: choice.optional(),
+})
+    .strict()
+    .refine((value) => Object.keys(value).length > 0, "Pick at least one provider to update.");
+export function validateLocalProviderChanges(changes, current) {
+    const fields = [];
+    const families = { llm: "llm", audio: "tts", images: "image" };
+    for (const key of ["llm", "audio", "images"]) {
+        const picked = changes[key];
+        if (picked === undefined)
+            continue;
+        const provider = current.providers.find((one) => one.id === picked.provider && one.family === families[key]);
+        if (provider === undefined) {
+            fields.push({ field: key, message: "Choose a provider for this kind of generation." });
+            continue;
+        }
+        const usable = provider.readiness.kind === "cli" ? provider.readiness.installed : provider.readiness.hasKey;
+        if (!usable) {
+            fields.push({
+                field: key,
+                message: provider.readiness.kind === "cli"
+                    ? "Install and sign in to this CLI first."
+                    : "Save this provider's API key in Settings first.",
+            });
+        }
+    }
+    if (changes.audio !== undefined &&
+        !current.voices.some((voice) => voice.provider === changes.audio?.provider && voice.voiceId === changes.audio.voice)) {
+        fields.push({ field: "audio.voice", message: "Choose a saved voice for this provider." });
+    }
+    return fields;
+}
+export async function validateProviderChanges(changes, current) {
+    const fields = [...validateLocalProviderChanges(changes, current)];
+    if (current.modelsFor === undefined)
+        return fields;
+    const families = { llm: "llm", audio: "tts", images: "image" };
+    for (const key of ["llm", "audio", "images"]) {
+        const picked = changes[key];
+        if (picked === undefined || fields.some((field) => field.field === key))
+            continue;
+        const models = await current.modelsFor(picked.provider, families[key]);
+        if (!models.some((model) => model.id === picked.model)) {
+            fields.push({ field: `${key}.model`, message: "Choose a model supported by this provider." });
+        }
+    }
+    return fields;
+}