@gentbajko/slopify 3.25.9 → 4.0.0

package/dist/slices/template-studio/bakeoff.js

@@ -0,0 +1,333 @@
+import { z } from "zod";
+import { transact } from "../../kernel/db/tx.js";
+import { projectExists } from "../admission/repo.js";
+import { keepThinking } from "../model-upkeep/switch.js";
+import { templateById } from "../project-templates/repo.js";
+import { currentRevisionId } from "../revisions/repo.js";
+import { getRevisionView, isReadyOutput } from "../revisions/view.js";
+import { runCostOf } from "../run-cost/panel.js";
+import { bakeoffOf, bakeoffSections, clonesOf, deleteBakeoffRow, insertBakeoff, } from "./bakeoff-repo.js";
+import { budgetRoom, budgetWords, fitsBudget } from "./budget.js";
+import { removeHiddenProject } from "./hidden.js";
+import { enqueueJob, queuedJob, runningJob } from "./jobs.js";
+import { sectionLabel } from "./model.js";
+import { quoteSample } from "./quote.js";
+import { sampleOf, sampleVersions } from "./repo.js";
+import { exceedsAcknowledged } from "./save.js";
+import { sectionOfWork } from "./sections.js";
+// Compare models: one section's sample made again on one or two other models, each on a hidden
+// copy of the template's sample (`onboarding/copy.ts`), so its cost and speed sit beside the
+// model the template uses now. Only that section's work runs on each copy. Nothing about the
+// template changes: "Use this model" is a normal Save, and the copies are discarded after.
+export const bakeoffModelSchema = z
+    .object({
+    provider: z.string().trim().min(1).max(100),
+    model: z.string().trim().min(1).max(200),
+})
+    .strict();
+export const bakeoffQuoteSchema = z
+    .object({
+    section: z.enum(bakeoffSections),
+    // The models to compare with the one the template uses now: one or two.
+    models: z.array(bakeoffModelSchema).min(1).max(2),
+})
+    .strict();
+export const bakeoffStartSchema = bakeoffQuoteSchema
+    .extend({
+    // The total high estimate the quote showed, and that steps with no price are accepted.
+    acknowledgedHigh: z.number().finite().min(0).max(10_000),
+    acknowledgedUnknown: z.boolean().optional(),
+})
+    .strict();
+// The stage each section's model works in, which its time is read from.
+export const bakeoffStage = {
+    script: "article",
+    voice: "audio",
+    pictures: "images",
+};
+// The model a section is made with now, or undefined when no model makes it.
+export function sectionModel(document, section) {
+    const { form } = document;
+    const pick = (choice) => choice.provider === "" || choice.model === ""
+        ? undefined
+        : { provider: choice.provider, model: choice.model };
+    switch (section) {
+        case "script":
+            return form.sources.article === "generate" ? pick(form.llm) : undefined;
+        case "voice":
+            return form.sources.audio === "generate" ? pick(form.audio) : undefined;
+        case "pictures":
+            return form.sources.images === "generate" ? pick(form.images) : undefined;
+    }
+}
+// The template's setup with the section made with `model` and nothing else changed.
+export function withModel(document, section, model, catalogue) {
+    const { form } = document;
+    switch (section) {
+        case "script": {
+            const { thinking, ...rest } = form.llm;
+            return {
+                ...document,
+                form: {
+                    ...form,
+                    llm: {
+                        ...rest,
+                        provider: model.provider,
+                        model: model.model,
+                        ...keepThinking(catalogue, model.provider, model.model, thinking),
+                    },
+                },
+            };
+        }
+        case "voice":
+            return { ...document, form: { ...form, audio: { ...form.audio, model: model.model } } };
+        case "pictures":
+            return {
+                ...document,
+                form: { ...form, images: { ...form.images, provider: model.provider, model: model.model } },
+            };
+    }
+}
+const refuse = (reason, message) => ({
+    ok: false,
+    reason,
+    message,
+});
+const missingTemplate = "This template no longer exists; it may have been deleted. Go to Library → Templates to see your templates.";
+// What the bake-off is made on: the template's newest version, with a current paid sample.
+function basis(deps, templateId, section, models) {
+    const template = templateById(deps.db, templateId);
+    if (template === undefined)
+        return refuse("not-found", missingTemplate);
+    const label = sectionLabel[section];
+    const current = sectionModel(template.document, section);
+    if (current === undefined)
+        return refuse("invalid-input", `No model makes ${label} in this template, so there is nothing to compare. Choose a model in ${label} and press Save first.`);
+    const sample = sampleOf(deps.db, templateId);
+    const made = sampleVersions(deps.db, templateId).find((one) => one.templateVersion === template.version);
+    if (sample === undefined || !projectExists(deps.db, sample.projectId) || made?.state !== "done")
+        return refuse("not-ready", `Compare models runs on the template's current sample, and version ${String(template.version)} has none yet. Press Save (or Make sample) in the studio, wait for it, then compare.`);
+    const seen = new Set();
+    for (const model of models) {
+        const key = `${model.provider}:${model.model}`;
+        if (model.provider === current.provider && model.model === current.model)
+            return refuse("invalid-input", `${model.model} is the model ${label} uses now; its figures come from the current sample. Pick another model to compare it with.`);
+        if (seen.has(key))
+            return refuse("invalid-input", `${model.model} is picked twice. Pick each model once.`);
+        seen.add(key);
+        if (section === "voice" && model.provider !== current.provider)
+            return refuse("invalid-input", "A voice is chosen per provider, so voice models are compared within the template's voice provider. Pick models of that provider, or change the voice in Voice first.");
+    }
+    return { ok: true, document: template.document, version: template.version, current };
+}
+export async function quoteBakeoff(deps, templateId, input) {
+    const base = basis(deps, templateId, input.section, input.models);
+    if (!base.ok)
+        return base;
+    const catalogue = deps.review.catalogue.read();
+    const models = [];
+    for (const model of input.models) {
+        const quoted = await quoteSample(deps, templateId, {
+            document: withModel(base.document, input.section, model, catalogue),
+            sections: [input.section],
+        });
+        if (!quoted.ok)
+            return quoted;
+        models.push({
+            ...model,
+            high: quoted.value.high,
+            unknown: quoted.value.unknown,
+            onPlan: quoted.value.onPlan,
+            usesPlan: quoted.value.usesPlan,
+            problem: quoted.value.problem,
+        });
+    }
+    const high = models.reduce((sum, one) => sum + one.high, 0);
+    const usesPlan = models.some((one) => one.usesPlan);
+    const budget = budgetRoom(deps.db, deps.clock.now(), deps.timeZone);
+    return {
+        ok: true,
+        value: {
+            version: base.version,
+            section: input.section,
+            current: base.current,
+            models,
+            high,
+            unknown: models.reduce((sum, one) => sum + one.unknown, 0),
+            usesPlan,
+            budget,
+            fits: fitsBudget(budget, high, usesPlan),
+            problem: models.find((one) => one.problem !== null)?.problem ?? null,
+        },
+    };
+}
+// Compare: priced again first; a higher total, an unknown price not accepted or a total past
+// today's budget is refused with the new quote, and nothing is spent.
+export async function startBakeoff(deps, templateId, input) {
+    const existing = bakeoffOf(deps.db, templateId);
+    if (existing !== undefined && (existing.state === "queued" || existing.state === "making"))
+        return refuse("conflict", "A bake-off of this template is still being made. Wait for it to finish, or discard it, then compare again.");
+    if (queuedJob(deps.db, templateId) !== undefined ||
+        runningJob(deps.db)?.templateId === templateId)
+        return refuse("conflict", "A sample of this template is being made. Compare models once it is done: the bake-off runs on it.");
+    const quoted = await quoteBakeoff(deps, templateId, input);
+    if (!quoted.ok)
+        return quoted;
+    const quote = quoted.value;
+    if (quote.problem !== null)
+        return { ...refuse("invalid-input", quote.problem), quote };
+    const money = (value) => `$${value.toFixed(2)}`;
+    if (exceedsAcknowledged(quote.high, input.acknowledgedHigh))
+        return {
+            ...refuse("cost-changed", `Comparing now costs about ${money(quote.high)}, more than the ${money(input.acknowledgedHigh)} you saw. Press Compare again to accept the new price.`),
+            quote,
+        };
+    if (quote.unknown > 0 && input.acknowledgedUnknown !== true)
+        return {
+            ...refuse("cost-changed", `${String(quote.unknown)} ${quote.unknown === 1 ? "step has" : "steps have"} no known price. Press Compare again to accept that.`),
+            quote,
+        };
+    if (!quote.fits)
+        return {
+            ...refuse("over-budget", `${budgetWords(quote.budget, quote, "Comparing")}, or compare tomorrow.`),
+            quote,
+        };
+    if (existing !== undefined) {
+        const cleared = discardBakeoff(deps, templateId);
+        if (!cleared.ok)
+            return cleared;
+    }
+    const id = deps.uuid();
+    const at = deps.clock.now().toISOString();
+    transact(deps.db, () => {
+        insertBakeoff(deps.db, {
+            id,
+            templateId,
+            templateVersion: quote.version,
+            section: input.section,
+            // The person's figure: the quarter-and-five-cents leeway is applied once, at make time.
+            ackHigh: input.acknowledgedHigh,
+            ackUnknown: quote.unknown > 0,
+            createdAt: at,
+        }, quote.models.map((model) => ({
+            provider: model.provider,
+            model: model.model,
+            quotedHigh: model.high,
+        })));
+        enqueueJob(deps.db, {
+            templateId,
+            templateVersion: quote.version,
+            ackHigh: input.acknowledgedHigh,
+            kind: "bakeoff",
+            requestedAt: at,
+        });
+    });
+    return { ok: true, value: { id, version: quote.version } };
+}
+// Discard: every copy goes, with its files. What the copies were charged today still counts
+// against the sample budget. Refused while a copy still has work going.
+export function discardBakeoff(deps, templateId) {
+    const row = bakeoffOf(deps.db, templateId);
+    if (row === undefined)
+        return { ok: true, value: { removed: 0 } };
+    if (row.state === "making" ||
+        (row.state === "queued" && runningJob(deps.db)?.templateId === templateId))
+        return refuse("conflict", "The bake-off is still being made, and work already going is never stopped. Discard it once it finishes.");
+    let removed = 0;
+    for (const clone of clonesOf(deps.db, row.id)) {
+        if (clone.projectId === null || !projectExists(deps.db, clone.projectId))
+            continue;
+        const gone = removeHiddenProject(deps, clone.projectId);
+        if (!gone.ok)
+            return refuse("conflict", gone.reason === "running"
+                ? "A model's copy still has work going. Discard the bake-off once it finishes."
+                : "A model's copy couldn't be removed from disk. Close any program using Slopify's files, then press Discard again.");
+        removed += 1;
+    }
+    deleteBakeoffRow(deps.db, row.id);
+    // A queued bake-off job has nothing left to make.
+    if (row.state === "queued")
+        deps.db
+            .prepare("DELETE FROM template_sample_jobs WHERE template_id=? AND state='queued' AND kind='bakeoff'")
+            .run(templateId);
+    return { ok: true, value: { removed } };
+}
+const openingChars = 1200;
+function madeOn(deps, projectId, section) {
+    const revisionId = currentRevisionId(deps.db, projectId);
+    if (revisionId === undefined)
+        return { outputs: [], article: null };
+    const view = getRevisionView({ ...deps, ids: { next: () => "" }, clock: { now: () => new Date(), sleep: async () => { } } }, projectId, revisionId);
+    if (view === undefined)
+        return { outputs: [], article: null };
+    const outputs = view.outputs.flatMap((row) => isReadyOutput(row) && sectionOfWork(row.workKey) === section
+        ? [
+            {
+                role: row.output.role,
+                url: `/files/${encodeURIComponent(projectId)}/revisions/${encodeURIComponent(revisionId)}/${encodeURIComponent(row.recordId)}`,
+                name: row.output.path.split("/").pop() ?? row.output.path,
+            },
+        ]
+        : []);
+    return {
+        outputs: section === "script" ? [] : outputs,
+        article: section === "script" && view.articleMarkdown !== null
+            ? view.articleMarkdown.slice(0, openingChars)
+            : null,
+    };
+}
+const secondsOf = (from, to) => from === null || to === null ? null : Math.max(0, (Date.parse(to) - Date.parse(from)) / 1000);
+// The bake-off as the studio shows it: the model in use now (from the template's current
+// sample), then each other model's copy, with what it made, cost and took.
+export function bakeoffView(deps, templateId) {
+    const row = bakeoffOf(deps.db, templateId);
+    if (row === undefined)
+        return null;
+    const head = templateById(deps.db, templateId);
+    const version = templateById(deps.db, templateId, row.templateVersion);
+    const current = version === undefined ? undefined : sectionModel(version.document, row.section);
+    const sample = sampleOf(deps.db, templateId);
+    const entries = [];
+    if (current !== undefined && sample !== undefined && projectExists(deps.db, sample.projectId)) {
+        const cost = runCostOf(deps.db, sample.projectId);
+        const used = cost.byModel.filter((one) => one.provider === current.provider && one.model === current.model);
+        const wall = cost.byStage.find((one) => one.stage === bakeoffStage[row.section])?.wallMs ?? null;
+        entries.push({
+            ...current,
+            slot: null,
+            state: "current",
+            quotedHigh: null,
+            cost: used.length === 0 ? null : used.reduce((sum, one) => sum + one.cost, 0),
+            seconds: wall === null ? null : wall / 1000,
+            failure: null,
+            ...madeOn(deps, sample.projectId, row.section),
+        });
+    }
+    for (const clone of clonesOf(deps.db, row.id)) {
+        const exists = clone.projectId !== null && projectExists(deps.db, clone.projectId);
+        entries.push({
+            provider: clone.provider,
+            model: clone.model,
+            slot: clone.slot,
+            state: clone.state,
+            quotedHigh: clone.quotedHigh,
+            cost: exists && clone.projectId !== null && clone.state !== "queued"
+                ? runCostOf(deps.db, clone.projectId).cost
+                : null,
+            seconds: secondsOf(clone.startedAt, clone.finishedAt),
+            failure: clone.failure,
+            ...(exists && clone.projectId !== null && clone.state === "done"
+                ? madeOn(deps, clone.projectId, row.section)
+                : { outputs: [], article: null }),
+        });
+    }
+    return {
+        id: row.id,
+        section: row.section,
+        version: row.templateVersion,
+        stale: head !== undefined && head.version !== row.templateVersion,
+        state: row.state,
+        sentence: row.sentence,
+        entries,
+    };
+}

package/dist/slices/template-studio/budget.js

@@ -0,0 +1,103 @@
+import { z } from "zod";
+import { dailySampleBudget, dailySampleBudgetKey } from "../designs/budget.js";
+import { readJsonSetting, writeSetting } from "../settings/repo.js";
+// Settings → Template studio: the most the studio's samples may spend in one day, in dollars,
+// and whether they may use a CLI plan at all. Plan usage has no dollar price, so the budget
+// can't hold it back: samples that run on a CLI plan wait until the person turns that on. By
+// default ($2 a day, plans off) samples spend at most $2 a day and never a plan; at $0 nothing
+// spends money. A sample that would go over either waits, saying so, until the next day, a
+// higher budget or plans allowed. The designed pieces' samples have their own daily budget
+// (`designs/budget.ts`), set here too, and follow the same plan switch.
+export const dailyBudgetKey = "templateStudio.dailyBudget";
+export const samplesUsePlansKey = "templateStudio.samplesUsePlans";
+export const defaultDailyBudget = 2;
+export const dailyBudgetMax = 1000;
+// Each field left out keeps what is stored.
+export const studioSettingsSchema = z
+    .object({
+    dailyBudget: z.number().finite().min(0).max(dailyBudgetMax).optional(),
+    samplesUsePlans: z.boolean().optional(),
+    designDailyBudget: z.number().finite().min(0).max(dailyBudgetMax).optional(),
+})
+    .strict()
+    .readonly();
+export function readStudioSettings(db) {
+    return {
+        dailyBudget: readJsonSetting(db, dailyBudgetKey, z.number().finite().min(0).max(dailyBudgetMax), defaultDailyBudget),
+        samplesUsePlans: readJsonSetting(db, samplesUsePlansKey, z.boolean(), false),
+        designDailyBudget: dailySampleBudget(db),
+    };
+}
+export function writeStudioSettings(db, settings) {
+    const cents = (dollars) => JSON.stringify(Math.round(dollars * 100) / 100);
+    if (settings.dailyBudget !== undefined)
+        writeSetting(db, dailyBudgetKey, cents(settings.dailyBudget));
+    if (settings.designDailyBudget !== undefined)
+        writeSetting(db, dailySampleBudgetKey, cents(settings.designDailyBudget));
+    if (settings.samplesUsePlans !== undefined)
+        writeSetting(db, samplesUsePlansKey, JSON.stringify(settings.samplesUsePlans));
+    return readStudioSettings(db);
+}
+// The instant today began on this computer's clock, in `timeZone`.
+export function localMidnight(now, timeZone) {
+    const day = Temporal.Instant.fromEpochMilliseconds(now.valueOf())
+        .toZonedDateTimeISO(timeZone)
+        .startOfDay();
+    return new Date(day.epochMilliseconds);
+}
+export function systemTimeZone() {
+    return Intl.DateTimeFormat().resolvedOptions().timeZone;
+}
+// What the studio's samples were charged since local midnight: the priced calls of every
+// template's sample project and every bake-off copy, billed per use (a CLI plan's calls add
+// nothing), and what such a project spent before it was removed. Other projects' calls never
+// count.
+export function spentToday(db, now, timeZone = systemTimeZone()) {
+    const since = localMidnight(now, timeZone).toISOString();
+    const row = db
+        .prepare(`SELECT COALESCE((SELECT SUM(u.cost) FROM provider_usage u
+         WHERE u.project_id IN (SELECT project_id FROM template_hidden_projects)
+           AND u.on_plan = 0 AND u.created_at >= ?), 0)
+       + COALESCE((SELECT SUM(cost) FROM template_studio_spend WHERE created_at >= ?), 0) AS spent`)
+        .get(since, since);
+    return z.object({ spent: z.number() }).parse(row).spent;
+}
+// Today's sample calls billed per use whose price no one knows: they add nothing to `spentToday`,
+// so they are counted and said beside it rather than read as free.
+export function unpricedToday(db, now, timeZone = systemTimeZone()) {
+    const row = db
+        .prepare(`SELECT count(*) AS n FROM provider_usage u
+       WHERE u.project_id IN (SELECT project_id FROM template_hidden_projects)
+         AND u.on_plan = 0 AND u.cost IS NULL AND u.created_at >= ?`)
+        .get(localMidnight(now, timeZone).toISOString());
+    return z.object({ n: z.number() }).parse(row).n;
+}
+export function budgetRoom(db, now, timeZone = systemTimeZone()) {
+    const { dailyBudget, samplesUsePlans } = readStudioSettings(db);
+    const spent = spentToday(db, now, timeZone);
+    return {
+        budget: dailyBudget,
+        spent,
+        unpriced: unpricedToday(db, now, timeZone),
+        left: Math.max(0, dailyBudget - spent),
+        plans: samplesUsePlans,
+    };
+}
+// Whether a sample of about `high` dollars, `usesPlan` when a step runs on a CLI plan, may
+// start on what is left of today's budget.
+export function fitsBudget(room, high, usesPlan = false) {
+    if (usesPlan && !room.plans)
+        return false;
+    return high <= room.left + 1e-9;
+}
+const money = (value) => `$${value.toFixed(2)}`;
+// Why a sample that doesn't fit (`fitsBudget`) waits, and what lets it start, as one sentence
+// (no full stop) about `subject`: "This sample", "Comparing".
+export function budgetWords(room, cost, subject) {
+    if (cost.usesPlan && !room.plans)
+        return `${subject} runs on your CLI plan, and samples may not use CLI plans. Turn on Samples may use my CLI plans in Settings → Template studio`;
+    const unpriced = room.unpriced === 0
+        ? ""
+        : ` (samples spent ${money(room.spent)} today plus ${String(room.unpriced)} ${room.unpriced === 1 ? "call" : "calls"} with no known price)`;
+    return `${subject} costs about ${money(cost.high)} and ${money(room.left)} of today's ${money(room.budget)} sample budget is left${unpriced}. Raise the budget in Settings → Template studio`;
+}