@gentbajko/slopify 0.1.0

package/dist/slices/admission/start.js

@@ -0,0 +1,103 @@
+import { transact } from "../../kernel/db/tx.js";
+import { stageKinds } from "../../kernel/pipeline.js";
+import { storeArticleText } from "../article/store.js";
+import { attachStagedFile, dropStagedSource, storeText } from "../storage/staging.js";
+import { insertProject, insertStage } from "./repo.js";
+// logic/01 step 1: Provide → `provided` with its output attached; Off → `skipped`;
+// everything else → `pending`.
+export function initialState(source) {
+    if (source === "provide") {
+        return "provided";
+    }
+    return source === "off" ? "skipped" : "pending";
+}
+// logic/04 step 6 with 04-data-flow Run step 2: the project row, its six stages, and the
+// provided content all land together or not at all. The caller ticks the runner after
+// this returns, never inside it.
+export function startRun(deps, draft, rendered) {
+    const id = deps.ids.next();
+    const at = deps.clock.now().toISOString();
+    const config = { ...draft, rendered };
+    const project = {
+        id,
+        title: draft.title,
+        format: draft.format,
+        config,
+        createdAt: at,
+        updatedAt: at,
+    };
+    const stages = stageKinds.map((kind) => ({
+        id: deps.ids.next(),
+        projectId: id,
+        kind,
+        source: draft.sources[kind],
+        state: initialState(draft.sources[kind]),
+        failureReason: null,
+        attemptCount: 0,
+        progressCurrent: null,
+        progressTotal: null,
+        startedAt: null,
+        finishedAt: null,
+    }));
+    // Filled inside the transaction, emptied after it. Every provided upload is copied into
+    // the project folder rather than moved, so a rollback leaves the staging files exactly
+    // as the form left them and the same Play can simply be pressed again.
+    const moved = [];
+    transact(deps.db, () => {
+        insertProject(deps.db, project);
+        for (const stage of stages) {
+            insertStage(deps.db, stage);
+        }
+        attachProvided(deps, id, draft, moved);
+    });
+    for (const source of moved) {
+        dropStagedSource(deps, source);
+    }
+    return { project, stages };
+}
+function attachProvided(deps, projectId, draft, collected) {
+    const { sources, provided } = draft;
+    if (sources.research === "provide" && provided.research !== undefined) {
+        storeText(deps, {
+            projectId,
+            stageKind: "research",
+            role: "notes",
+            text: provided.research.trim(),
+        });
+    }
+    if (sources.article === "provide" && provided.article !== undefined) {
+        // logic/08 step 1: the end-matter split runs when the article becomes done *or
+        // provided*, so a pasted Sources Consulted list is cut into its own file here exactly
+        // as the article stage cuts a written one, and never reaches the narration.
+        storeArticleText(deps, { projectId, markdown: provided.article.trim() });
+    }
+    if (sources.audio === "provide") {
+        attach(deps, projectId, "audio", provided.audio, "audio_body", collected);
+    }
+    if (sources.thumbnail === "provide") {
+        attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected);
+    }
+    if (sources.images === "provide") {
+        // logic/05 §Q39: slideshow order is the order the user left the list in.
+        for (const [index, stagedFileId] of (provided.images ?? []).entries()) {
+            attach(deps, projectId, "images", stagedFileId, "image", collected, index + 1);
+        }
+    }
+}
+function attach(deps, projectId, kind, stagedFileId, role, collected, index) {
+    if (stagedFileId === undefined) {
+        throw new Error(`the ${kind} stage is provided but carries no staged file`);
+    }
+    const result = attachStagedFile(deps, {
+        stagedFileId,
+        projectId,
+        role,
+        ...(index === undefined ? {} : { index }),
+    });
+    if (!result.ok) {
+        // admit() already refused an unknown or still-copying upload, so reaching here means
+        // the file went away between the check and the write.
+        throw new Error(`the ${kind} stage's file could not be attached: ${result.reason}`);
+    }
+    collected.push(result.stagedSource);
+}

package/dist/slices/admission/substitute.js

@@ -0,0 +1,80 @@
+// logic/03 §Q19-§Q21 fix the grammar: `{{` … `}}`, whitespace immediately inside the
+// braces stripped, the name case-sensitive and free to hold anything but `{`, `}` and a
+// newline (so `{{Middle of Words}}` is one name). There is no escape syntax. Hand-rolled
+// on purpose (standards §Q2): a template library would bring its own grammar.
+export const slotLintKinds = ["unclosed", "empty", "nested"];
+export const fieldGroups = ["common", "text", "image"];
+// A well-formed slot: no brace and no newline between the delimiters.
+const slot = /\{\{([^{}\n]*)\}\}/g;
+const opener = "{{";
+const closer = "}}";
+export function detectSlots(body) {
+    const names = [];
+    const errors = [];
+    let at = body.indexOf(opener);
+    while (at !== -1) {
+        // A name cannot span a line, so the search for the closer stops at the next newline.
+        const newline = body.indexOf("\n", at + opener.length);
+        const limit = newline === -1 ? body.length : newline;
+        const close = body.indexOf(closer, at + opener.length);
+        if (close === -1 || close > limit) {
+            errors.push({ kind: "unclosed", at });
+            at = body.indexOf(opener, at + opener.length);
+            continue;
+        }
+        const inside = body.slice(at + opener.length, close);
+        const name = inside.trim();
+        if (inside.includes("{") || inside.includes("}")) {
+            errors.push({ kind: "nested", at });
+        }
+        else if (name === "") {
+            errors.push({ kind: "empty", at });
+        }
+        else if (!names.includes(name)) {
+            names.push(name);
+        }
+        at = body.indexOf(opener, close + closer.length);
+    }
+    return { names, errors };
+}
+// logic/03 step 3: one field per distinct name; Common when a name is used on both
+// sides, otherwise Text or Image. §Q24 fixes the order as first appearance, which is why
+// both sides arrive as ordered lists of bodies rather than as sets.
+export function collectFields(textBodies, imageBodies) {
+    const text = namesOf(textBodies);
+    const image = namesOf(imageBodies);
+    const fields = [];
+    for (const name of [...text, ...image]) {
+        if (fields.some((field) => field.name === name)) {
+            continue;
+        }
+        const onText = text.includes(name);
+        const onImage = image.includes(name);
+        fields.push({ name, group: onText && onImage ? "common" : onText ? "text" : "image" });
+    }
+    return fields;
+}
+// logic/03 step 5 and §Q21: one pass, so a value that itself contains `{{x}}` lands in
+// the output verbatim and is never looked at again. A name with no value is left as it
+// was written; scenario 04 refuses the run before this can reach a provider.
+export function render(body, values) {
+    return body.replace(slot, (whole, inside) => {
+        // A slot name may be anything but a brace or a newline (§Q19), so "constructor" and
+        // "toString" are ordinary names; a plain lookup would answer them from Object's
+        // prototype and splice a function's source into a prompt bound for a provider.
+        const name = inside.trim();
+        const value = Object.hasOwn(values, name) ? values[name] : undefined;
+        return value === undefined ? whole : value;
+    });
+}
+function namesOf(bodies) {
+    const names = [];
+    for (const body of bodies) {
+        for (const name of detectSlots(body).names) {
+            if (!names.includes(name)) {
+                names.push(name);
+            }
+        }
+    }
+    return names;
+}

package/dist/slices/article/continuation.js

@@ -0,0 +1,81 @@
+import { noTokens, plusUsage } from "../telemetry/model.js";
+// §Q59: "at most 3 continuations; still unfinished after the third is a failed attempt".
+export const continuationLimit = 3;
+// The provider's own word for "I stopped because I ran out of room", which is what every
+// adapter maps its finish reason to (`kernel/ports/llm.ts`).
+const truncatedReason = "length";
+// §Q56: "a fixed 'Research notes' header followed by the notes, then the rendered article
+// prompt; without research, the rendered prompt alone". One user message, and no
+// parameters of the app's own: the provider's defaults are used (§Q61).
+export function articleMessages(brief) {
+    const notes = brief.notes?.trim();
+    const content = notes === undefined || notes === ""
+        ? brief.articlePrompt
+        : `Research notes\n\n${notes}\n\n${brief.articlePrompt}`;
+    return [{ role: "user", content }];
+}
+// The article so far goes back as the assistant turn it was, so the model continues its
+// own answer rather than being asked to write a second article. Nothing is inserted at
+// the seam - the pieces are concatenated exactly as they arrived (§Q57: the stored text
+// is the model's, never edited by the app) - so the instruction has to carry the whole
+// of the rule that keeps the seam invisible.
+export function continuationMessages(base, soFar) {
+    return [
+        ...base,
+        { role: "assistant", content: soFar },
+        {
+            role: "user",
+            content: [
+                "You stopped at your output limit before the article was finished.",
+                "Continue from the exact character you stopped at, in the same voice and format.",
+                "Do not repeat anything you have already written, do not restate the last",
+                "sentence, and do not open with a heading or a preamble unless the article was",
+                "already in the middle of one.",
+            ].join("\n"),
+        },
+    ];
+}
+export async function writeArticle(providers, choice, brief, onDelta) {
+    const base = articleMessages(brief);
+    const sent = [{ label: "Article", messages: base }];
+    const stream = (event) => {
+        if (event.type === "delta") {
+            onDelta(event.text);
+        }
+    };
+    let answer = await providers.llm({ provider: choice.provider, model: choice.model, messages: base, check: written }, stream);
+    const pieces = [answer.text];
+    let tokens = plusUsage(noTokens, answer.usage);
+    for (let n = 1; truncated(answer) && n <= continuationLimit; n += 1) {
+        const messages = continuationMessages(base, pieces.join(""));
+        sent.push({ label: `Continuation ${String(n)}`, messages });
+        answer = await providers.llm({
+            provider: choice.provider,
+            model: choice.model,
+            messages,
+            // The last continuation allowed is the one that has to end the article: a fourth
+            // truncation is a failed attempt, which is the wrapper's to retry (§Q59). The
+            // loop therefore never sees a truncated answer with its budget spent.
+            check: n === continuationLimit ? finished : written,
+        }, stream);
+        pieces.push(answer.text);
+        tokens = plusUsage(tokens, answer.usage);
+    }
+    return { markdown: pieces.join(""), sent, tokens };
+}
+function truncated(answer) {
+    return answer.finishReason === truncatedReason;
+}
+// §Q61: "Empty response → failed attempt".
+function written(answer) {
+    return answer.text.trim() === "" ? "the article answered with nothing" : undefined;
+}
+function finished(answer) {
+    const empty = written(answer);
+    if (empty !== undefined) {
+        return empty;
+    }
+    return truncated(answer)
+        ? `the article was still unfinished after ${String(continuationLimit)} continuations`
+        : undefined;
+}

package/dist/slices/article/plain.js

@@ -0,0 +1,14 @@
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import stripMarkdown from "strip-markdown";
+// `logic/05` §Q37 and `logic/07` step 4: the article is stored twice, as the markdown the
+// model wrote and as the plain text the narration is read from, so no TTS voice ever says
+// a hash or a bracket. remark parses and strip-markdown drops the formatting; the GFM
+// extension is what makes a table a table rather than a paragraph full of pipes, and what
+// takes a footnote marker out of the middle of a sentence.
+export function plainText(markdown) {
+    // Built per call: a shared processor would be a module-level singleton, and building
+    // one costs a few microseconds against a call that just parsed an article.
+    const stripped = remark().use(remarkGfm).use(stripMarkdown).processSync(markdown);
+    return String(stripped);
+}

package/dist/slices/article/run.js

@@ -0,0 +1,166 @@
+import { readFileSync } from "node:fs";
+import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
+import { projectById } from "../admission/repo.js";
+import { outputPath } from "../storage/layout.js";
+import { outputsOf } from "../storage/repo.js";
+import { storeText } from "../storage/staging.js";
+import { noTokens, plusUsage } from "../telemetry/model.js";
+import { writeArticle } from "./continuation.js";
+import { storeArticleText } from "./store.js";
+const categories = ["intro", "outro"];
+export async function runArticle(deps, context, providers) {
+    const { projectId } = context.stage;
+    const project = projectById(deps.db, projectId);
+    if (project === undefined) {
+        throw new Error(`project ${projectId} has no row`);
+    }
+    const choice = project.config.llm;
+    const articlePrompt = project.config.rendered.article;
+    if (choice === undefined || articlePrompt === undefined) {
+        // Admission refuses a run whose article is Generate without both (`logic/04`), so
+        // reaching here is a bug in admission rather than something the user did.
+        throw new Error("the run has no LLM provider or no rendered article prompt");
+    }
+    const notes = researchNotes(deps, projectId);
+    const brief = { articlePrompt, ...(notes === undefined ? {} : { notes }) };
+    const written = await writeArticle(providers, choice, brief, (text) => {
+        // Step 2 and `logic/01` §Q6: the page shows the article as it is written. The idle
+        // timeout is restarted by the wrapper on the same events, not here.
+        // ceiling: §Q60 discards the partial text of a failed attempt, but the deltas already
+        // sent cannot be unsent, so a retry mid-stream leaves the page appending the second
+        // telling under the first. What the project keeps is still the successful attempt's
+        // text alone; the upgrade is an event telling the page to start the article again.
+        context.emit({ type: "article.delta", projectId, text });
+    });
+    // Step 4: the markdown exactly as the model produced it, and the plain-text narration
+    // source. §Q63 keeps the end matter out of the narration and beside the article instead.
+    const store = { projectId, stageKind: "article" };
+    storeText(deps, { ...store, role: "article_md", text: written.markdown });
+    const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
+    // logic/16 step 2 counts the article and each entry text as units of their own, so the
+    // article's own event goes out as soon as its text is stored, carrying the tokens of
+    // the first call and its continuations (logic/07 step 3).
+    deps.count("stage.completed", {
+        stage: "article",
+        provider: choice.provider,
+        model: choice.model,
+        ...written.tokens,
+    });
+    const sent = [...written.sent];
+    for (const [index, category] of categories.entries()) {
+        const segment = await writeSegment(providers, choice, project.config, category, narration, sent);
+        if (segment !== undefined) {
+            keepSegment(deps, context, segment, index + 1);
+            // "each intro/outro text" of logic/16 step 2, named by its segment. A text-mode
+            // entry is rendered rather than written (§Q98), so it made no call and names no
+            // provider; its tokens are the zero step 3 asks for rather than an estimate.
+            deps.count("stage.completed", {
+                stage: "article",
+                segment: category,
+                ...(segment.mode === "llm" ? { provider: choice.provider, model: choice.model } : {}),
+                ...segment.tokens,
+            });
+        }
+    }
+    // §Q57: the exact messages sent, the continuations and the entry calls among them.
+    storeText(deps, { ...store, role: "instructions", text: instructionsText(sent) });
+    deps.log.write("info", "article.done", {
+        projectId,
+        stage: "article",
+        detail: `${String(written.sent.length - 1)} continuations, ${String(narration.length)} characters to narrate`,
+    });
+}
+// Step 5: "for each picked entry in LLM mode, one call with the filled entry as
+// instruction plus the title, keyword values, and the plain-text article ... Text-mode
+// entries are stored as rendered per scenario 03 with no call" (§Q97, §Q98).
+async function writeSegment(providers, choice, config, category, article, sent) {
+    const picked = config[category];
+    if (picked === undefined) {
+        return undefined;
+    }
+    const body = config.rendered[category];
+    if (body === undefined) {
+        throw new Error(`the run has no rendered ${category} text`);
+    }
+    const common = { category, name: picked.name, mode: picked.mode };
+    if (picked.mode === "text") {
+        return { ...common, text: body, tokens: noTokens };
+    }
+    const messages = segmentMessages(body, config, article);
+    sent.push({ label: label(category), messages });
+    const answer = await providers.llm({
+        provider: choice.provider,
+        model: choice.model,
+        messages,
+        // §Q96 and §Q61: an entry that answers with nothing is a failed attempt like any
+        // other, and the wrapper is what retries it.
+        check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
+    });
+    return { ...common, text: answer.text.trim(), tokens: plusUsage(noTokens, answer.usage) };
+}
+function segmentMessages(body, config, article) {
+    const values = Object.entries(config.values);
+    return [
+        {
+            role: "user",
+            content: [
+                body,
+                "",
+                `Video title: ${config.title}`,
+                "",
+                "Keyword values for this run:",
+                "",
+                values.length === 0
+                    ? "(none)"
+                    : values.map(([name, value]) => `${name}: ${value}`).join("\n"),
+                "",
+                "The article this video narrates:",
+                "",
+                article,
+            ].join("\n"),
+        },
+    ];
+}
+// The segment rows are keyed by `idx` within the stage, so a stage that runs again after
+// a failure replaces what it wrote rather than colliding with it.
+function keepSegment(deps, context, segment, idx) {
+    // Written out rather than stringified whole: the caller hands in what the call cost as
+    // well, and the piece is `logic/08`'s to read - it carries the text to narrate, nothing
+    // about the model that wrote it.
+    const payload = JSON.stringify({
+        category: segment.category,
+        name: segment.name,
+        mode: segment.mode,
+        text: segment.text,
+    });
+    const existing = piecesOf(deps.db, context.stage.id, "segment").find((piece) => piece.idx === idx);
+    if (existing === undefined) {
+        insertPiece(deps.db, {
+            id: deps.ids.next(),
+            stageId: context.stage.id,
+            kind: "segment",
+            idx,
+            state: "done",
+            payload,
+        });
+        return;
+    }
+    setPiece(deps.db, existing.id, "done", payload);
+}
+// Research writes its notes as an output of its own (`logic/06` step 4), and a provided
+// research stage stores the pasted text the same way, so one lookup covers both. No row
+// means research was Off or skipped, and the article is written from the prompt alone.
+function researchNotes(deps, projectId) {
+    const notes = outputsOf(deps.db, projectId).find((output) => output.role === "notes");
+    if (notes === undefined) {
+        return undefined;
+    }
+    return readFileSync(outputPath(deps.paths, projectId, notes.path), "utf8");
+}
+function instructionsText(sent) {
+    const parts = sent.map((one) => `=== ${one.label} ===\n\n${one.messages.map((message) => message.content).join("\n\n")}`);
+    return `${parts.join("\n\n")}\n`;
+}
+function label(category) {
+    return category === "intro" ? "Intro" : "Outro";
+}

package/dist/slices/article/split.js

@@ -0,0 +1,51 @@
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import { plainText } from "./plain.js";
+// §Q63 names the two sections. The comparison is on the heading's own text, so "## Sources
+// Consulted", "# SOURCES CONSULTED" and "**Sources Consulted:**" are one heading and
+// "## Sources Consulted and Further Reading" is not.
+const endHeadings = {
+    "sources consulted": "sources",
+    "pronunciation glossary": "glossary",
+};
+export function splitEndMatter(markdown) {
+    const marks = endMatterMarks(markdown);
+    const first = marks[0];
+    if (first === undefined) {
+        return { body: markdown, sources: "", glossary: "" };
+    }
+    const parts = { sources: "", glossary: "" };
+    for (const [index, mark] of marks.entries()) {
+        // Everything up to the next end heading belongs to this one, so a chapter the model
+        // wrote below its sources list travels with the sources rather than being narrated.
+        parts[mark.part] += markdown.slice(mark.at, marks[index + 1]?.at ?? markdown.length);
+    }
+    return { body: markdown.slice(0, first.at), sources: parts.sources, glossary: parts.glossary };
+}
+function endMatterMarks(markdown) {
+    const tree = remark().use(remarkGfm).parse(markdown);
+    const marks = [];
+    for (const node of tree.children) {
+        // A paragraph counts as well as a heading: a model that answers with a bold line
+        // rather than a `#` still meant it as the section's title.
+        if (node.type !== "heading" && node.type !== "paragraph") {
+            continue;
+        }
+        const at = node.position?.start.offset;
+        const to = node.position?.end.offset;
+        if (at === undefined || to === undefined) {
+            continue;
+        }
+        const part = partOf(markdown.slice(at, to));
+        if (part !== undefined) {
+            marks.push({ part, at });
+        }
+    }
+    return marks;
+}
+function partOf(source) {
+    // The heading's own markup is dropped by the same conversion the narration source uses,
+    // so nothing here has to know what a heading looks like.
+    const heading = plainText(source).trim().toLowerCase().replace(/:$/, "");
+    return endHeadings[heading];
+}

package/dist/slices/article/store.js

@@ -0,0 +1,21 @@
+import { storeText } from "../storage/staging.js";
+import { plainText } from "./plain.js";
+import { splitEndMatter } from "./split.js";
+// The narration source it stored, so a caller that needs the article as it will be spoken
+// - `logic/07` step 5 writes the intro and outro from it - does not read the file back.
+export function storeArticleText(deps, input) {
+    const end = splitEndMatter(input.markdown);
+    const store = { projectId: input.projectId, stageKind: "article" };
+    // `logic/05` §Q37's invariant: "the narration source of an article is always plain
+    // text", so no voice ever says a hash or a bracket, whoever wrote the markdown.
+    const narration = plainText(end.body);
+    storeText(deps, { ...store, role: "article_txt", text: narration });
+    // §Q63: no such heading means no file, not an empty one.
+    if (end.sources.trim() !== "") {
+        storeText(deps, { ...store, role: "sources", text: end.sources });
+    }
+    if (end.glossary.trim() !== "") {
+        storeText(deps, { ...store, role: "glossary", text: end.glossary });
+    }
+    return narration;
+}

package/dist/slices/cancel/index.js

@@ -0,0 +1,61 @@
+import { derive } from "../../kernel/runner/graph.js";
+import { finishStage, projectExists, stagesOf } from "../admission/repo.js";
+// `logic/13`: Cancel on the project header. Every in-flight call of this project is
+// aborted at once, every `done` output and every finished piece is kept for the resume,
+// and the project sits `canceled` until the user retries a stage.
+// §Q111 and `logic/01`'s transition table, in the words the stage row carries.
+export const canceledByUser = "canceled by user";
+export async function cancelProject(deps, projectId) {
+    if (!projectExists(deps.db, projectId)) {
+        return { ok: false, reason: "no-project" };
+    }
+    const before = stagesOf(deps.db, projectId);
+    const running = before.filter((stage) => stage.state === "running").map((stage) => stage.kind);
+    if (running.length === 0) {
+        // Step 4: "a second click is a no-op". Nothing is aborted and no state changes, so
+        // the page is simply told what the project already reads.
+        return { ok: true, canceled: [], state: derive(before) };
+    }
+    // Step 1: nothing waits for a response. The runner holds the controllers and its own
+    // barrier, so a stage that finishes during this does not release its dependents.
+    await deps.abort(projectId);
+    // Step 3's invariant: "after cancel completes no stage of the project is `running`".
+    // The runner writes that row as each aborted stage unwinds; this is the path where it
+    // could not - a failed write is logged there and the run is left mid-flight otherwise.
+    const after = stagesOf(deps.db, projectId);
+    for (const stage of after) {
+        if (stage.state !== "running") {
+            continue;
+        }
+        deps.log.write("warn", "cancel.sweep", {
+            projectId,
+            stage: stage.kind,
+            detail: "the stage was still running after its calls stopped",
+        });
+        finishStage(deps.db, stage.id, "canceled", canceledByUser, deps.clock.now().toISOString());
+        deps.emit(projectId, {
+            type: "stage.state",
+            projectId,
+            stage: stage.kind,
+            state: "canceled",
+            failureReason: canceledByUser,
+        });
+    }
+    const settled = stagesOf(deps.db, projectId);
+    const state = derive(settled);
+    if (after.some((stage) => stage.state === "running")) {
+        // Only when the sweep above moved a row: the runner already announced the state it
+        // left the project in, and repeating it would tell every open page the same thing
+        // twice.
+        deps.emit(projectId, { type: "project.state", projectId, state });
+    }
+    return {
+        ok: true,
+        // §Q113: a stage whose output was stored in the same instant as the cancel stays
+        // `done`, so what was stopped is read back rather than assumed from what was running.
+        canceled: settled
+            .filter((stage) => stage.state === "canceled" && running.includes(stage.kind))
+            .map((stage) => stage.kind),
+        state,
+    };
+}