@gentbajko/slopify 3.25.8 → 4.0.0

package/dist/slices/schedules/schema.js

@@ -85,13 +85,28 @@ export const scheduleReleaseSchema = z
     .strict()
     .readonly();
 const releases = z.array(scheduleReleaseSchema).max(7).readonly();
+// How generated topics are ranked (`score.ts`): not at all, by what the library already
+// covers, or also by public search interest (topic titles go to Google, DuckDuckGo and
+// Wikipedia). Off is what every schedule saved before it has.
+export const topicScorings = ["off", "library", "search"];
 // Enough for a year of daily runs, pasted as one list.
 export const queueMax = 500;
+// What a schedule makes: projects from a project template (every schedule before designs), or
+// designed pieces from a design template, one per row ahead of its date.
+export const scheduleTemplateKinds = ["project", "design"];
+// A project template's id is a UUID (checked by the service); a design template's may be any
+// id its creator chose.
+const templateId = z
+    .string()
+    .min(1)
+    .max(64)
+    .regex(/^[A-Za-z0-9_-]+$/);
 export const scheduleCreateSchema = z
     .object({
     id,
     name: z.string().trim().min(1).max(200),
-    templateId: id,
+    templateKind: z.enum(scheduleTemplateKinds).default("project"),
+    templateId,
     templateVersion: z.number().int().positive(),
     cadence: cadenceSchema,
     timezone: z
@@ -112,6 +127,9 @@ export const scheduleCreateSchema = z
     brief: z.string().trim().max(briefMax).nullable().default(null),
     topicGeneration: topicGenerationSchema.default(topicGenerationOff),
     releases: releases.default([]),
+    // Absent: off for a new schedule, unchanged on a save, so a caller that doesn't know it
+    // never turns it off.
+    topicScoring: z.enum(topicScorings).optional(),
 })
     .strict()
     .readonly();
@@ -120,6 +138,12 @@ export const scheduleUpdateSchema = scheduleCreateSchema
     .extend({ baseVersion: z.number().int().positive(), mutationId: id })
     .strict()
     .readonly();
+// The bodies the schedule routes take: a save names its schedule in the path, and pause,
+// resume, cancel and delete carry only the version they were read at.
+export const scheduleSaveBodySchema = scheduleUpdateSchema.unwrap().omit({ id: true });
+export const scheduleVersionBodySchema = z
+    .object({ baseVersion: z.number().int().positive() })
+    .strict();
 export const scheduleDeleteSchema = z
     .object({ id, baseVersion: z.number().int().positive() })
     .strict()
@@ -128,11 +152,21 @@ export const scheduleControlSchema = z
     .object({ id, baseVersion: z.number().int().positive() })
     .strict()
     .readonly();
+export const scheduleTemplateGateSchema = z
+    .object({
+    head: z.number().int().positive(),
+    runs: z.number().int().positive().nullable(),
+    gated: z.boolean(),
+    gate: z.enum(["legacy", "ready", "carried", "unchecked"]),
+})
+    .strict()
+    .readonly();
 export const scheduleSummarySchema = z
     .object({
     id,
     name: z.string(),
-    templateId: id,
+    templateKind: z.enum(scheduleTemplateKinds).default("project"),
+    templateId,
     templateVersion: z.number().int().positive(),
     cadence: cadenceSchema,
     timezone: z.string(),
@@ -145,6 +179,7 @@ export const scheduleSummarySchema = z
     brief: z.string().nullable().default(null),
     topicGeneration: topicGenerationSchema.default(topicGenerationOff),
     releases: releases.default([]),
+    topicScoring: z.enum(topicScorings).default("off"),
     // The generation's own state: how many topics wait for approval, whether an LLM is being
     // asked now, and the last failure's plain reason (cleared by the next success).
     topics: topicState.default(topicStateIdle),
@@ -154,6 +189,11 @@ export const scheduleSummarySchema = z
     createdAt: z.string(),
     updatedAt: z.string(),
     deletedAt: z.string().datetime({ offset: true }).nullable(),
+    // Which version of its template the schedule runs (`runs`): the newest one until a version
+    // was first marked ready (`gated`), then the newest legacy, ready or carried one (null when
+    // none is); the template's newest (`head`) and the gate of the one that runs. Absent from a
+    // Slopify before the template studio.
+    templateGate: scheduleTemplateGateSchema.optional(),
 })
     .strict()
     .readonly();
@@ -165,6 +205,8 @@ export const scheduleRunSchema = z
     status: z.enum(["running", "succeeded", "failed", "skipped"]),
     requestId: id.nullable(),
     projectIds: z.array(z.string()).readonly(),
+    // The designed pieces a design schedule's run made. Older servers leave it out.
+    designIds: z.array(z.string()).readonly().default([]),
     estimate: z.unknown().nullable(),
     startedAt: z.string().datetime({ offset: true }),
     endedAt: z.string().datetime({ offset: true }).nullable(),
@@ -217,7 +259,7 @@ export const calendarRunSchema = z
     scheduleName: z.string(),
     scheduleVersion: z.number().int().positive(),
     paused: z.boolean(),
-    templateId: id,
+    templateId,
     templateVersion: z.number().int().positive(),
     templateName: z.string().nullable(),
     // The topic's position in the schedule's queue, for reordering; null without one.
@@ -231,6 +273,15 @@ export const calendarRunSchema = z
     // The project prepared ahead for this run (Prepare), which its day continues. Older servers
     // leave it out.
     prepared: z.string().nullable().default(null),
+    // No version of its template is marked ready (or saved before the template studio), so
+    // this run is skipped unless one is by then. Older servers leave it out.
+    notReady: z.boolean().default(false),
+    // A design schedule's run makes a designed piece on its day, never a video: it has nothing
+    // to prepare ahead and fills no video slot. Older servers leave it out.
+    templateKind: z.enum(scheduleTemplateKinds).default("project"),
+    // The schedule's time zone: its runs are shown on that zone's clock. Older servers leave
+    // it out, and the browser's is used.
+    timezone: z.string().optional(),
 })
     .strict()
     .readonly();

package/dist/slices/schedules/score.js

@@ -0,0 +1,107 @@
+// Topic scoring: how new a generated topic is next to what the library already covers, and,
+// when the person opted in, how much public search interest it has. Pure, so the web bundle
+// can import it too. Never reads a channel's analytics.
+import { normaliseTopic, topicWords } from "./similar.js";
+// A suggestion names the topic when it holds every word of it.
+export function readingOf(topic, result) {
+    const words = topicWords(topic);
+    const matching = result.suggestions?.slice(0, 10).filter((suggestion) => {
+        const said = topicWords(suggestion);
+        return words.size > 0 && [...words].every((word) => said.has(word));
+    }) ?? null;
+    return {
+        suggestHits: matching === null ? null : matching.length,
+        examples: matching?.filter((one) => normaliseTopic(one) !== normaliseTopic(topic)).slice(0, 2) ?? [],
+        wikiViews: result.wikiViews,
+    };
+}
+// The key a topic's score is kept by: its normalised title, so a recased or repunctuated
+// topic keeps its score.
+export function topicKey(title) {
+    return normaliseTopic(title);
+}
+function overlap(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let common = 0;
+    for (const word of a)
+        if (b.has(word))
+            common += 1;
+    // A known title that holds every word of the topic covers it, however long the title is.
+    return common === a.size ? 1 : common / (a.size + b.size - common);
+}
+export function noveltyOf(topic, known) {
+    const words = topicWords(topic);
+    let nearest;
+    for (const title of [...known.topics, ...known.projects]) {
+        const share = overlap(words, topicWords(title));
+        if (share > 0 && (nearest === undefined || share > nearest.overlap))
+            nearest = { title, overlap: share };
+    }
+    return nearest === undefined
+        ? { value: 1 }
+        : { value: Math.max(0, 1 - nearest.overlap), nearest };
+}
+// ceiling: ten million views a month (log10 7) is as popular as a topic gets here.
+const viewsCeiling = 7;
+export function interestOf(reading) {
+    if (reading === null)
+        return null;
+    const views = Math.min(1, Math.log10((reading.wikiViews ?? 0) + 1) / viewsCeiling);
+    const hits = Math.min(10, reading.suggestHits ?? 0) / 10;
+    return 0.6 * views + 0.4 * hits;
+}
+export function scoreOf(interest, novelty) {
+    return Math.round(100 * (interest === null ? novelty : 0.6 * interest + 0.4 * novelty));
+}
+function views(count) {
+    if (count >= 1_000_000)
+        return `${(count / 1_000_000).toFixed(1).replace(/\.0$/, "")}M`;
+    if (count >= 1000)
+        return `${String(Math.round(count / 1000))}k`;
+    return String(count);
+}
+// Why a topic scored as it did, one line per signal, as the score chip shows them.
+export function reasonsOf(novelty, reading, search) {
+    const lines = [];
+    if (search === "read" && reading !== null) {
+        if (reading.suggestHits !== null) {
+            const example = reading.examples[0];
+            lines.push(`Search: ${String(reading.suggestHits)} of 10 suggestions match${example === undefined ? "" : `, e.g. “${example}”`}`);
+        }
+        lines.push(reading.wikiViews === null
+            ? "Wikipedia: no article found"
+            : `Wikipedia: ${views(reading.wikiViews)} views last month`);
+    }
+    if (search === "unreachable")
+        lines.push("Search interest unknown: no connection to Google, DuckDuckGo or Wikipedia");
+    lines.push(novelty.nearest === undefined
+        ? "Library: new; nothing like it made or queued"
+        : novelty.value === 0
+            ? `Library: already covered by “${novelty.nearest.title}”`
+            : `Library: new; nearest “${novelty.nearest.title}” (${String(Math.round(novelty.nearest.overlap * 100))}% overlap)`);
+    return lines;
+}
+export function topicScore(novelty, reading, search) {
+    const interest = search === "read" ? interestOf(reading) : null;
+    return {
+        score: scoreOf(interest, novelty.value),
+        interest,
+        novelty: novelty.value,
+        reasons: reasonsOf(novelty, reading, search),
+    };
+}
+// The queue once scored: topics the person added or moved keep their place at the front in
+// their own order, then the scored ones, best first. Stable: equal scores keep their order.
+export function orderQueue(items, scores) {
+    const kept = [];
+    const scored = [];
+    for (const item of items) {
+        const found = scores.get(topicKey(item.title));
+        if (found === undefined || found.pinned)
+            kept.push(item);
+        else
+            scored.push({ item, score: found.score });
+    }
+    return [...kept, ...scored.toSorted((a, b) => b.score - a.score).map((one) => one.item)];
+}

package/dist/slices/schedules/service.js

@@ -1,12 +1,24 @@
 import { z } from "zod";
 import { transact } from "../../kernel/db/tx.js";
 import { requestHash } from "../play-drafts/repo.js";
-import { nextOccurrence } from "./calendar.js";
+import { checkDesign } from "./design-check.js";
+import { nextRunOf } from "./next-run.js";
 import { insertSchedule, runsForSchedule, scheduleById, scheduleRows, setScheduleStatus, tombstoneScheduleRow, updateScheduleRow, } from "./repo.js";
 import { scheduleControlSchema, scheduleCreateSchema, scheduleDeleteSchema, scheduleUpdateSchema, topicStateIdle, } from "./schema.js";
-import { templateKeywords, topicRowProblems } from "./topic-list.js";
+import { scheduleKeywords, topicRowProblems } from "./topic-list.js";
 export function listSchedules(deps) {
-    return scheduleRows(deps.db);
+    const rows = scheduleRows(deps.db);
+    const gateOf = deps.templateGate;
+    if (gateOf === undefined)
+        return rows;
+    return rows.map((schedule) => {
+        // The template studio's gate is a project template's; a design schedule's template is
+        // a design template, readied by the designs.
+        if (schedule.templateKind !== "project")
+            return schedule;
+        const templateGate = gateOf(schedule.templateId);
+        return templateGate === undefined ? schedule : { ...schedule, templateGate };
+    });
 }
 export function listScheduleRuns(deps, id) {
     if (!z.uuid().safeParse(id).success)
@@ -35,18 +47,16 @@ export function createSchedule(deps, input) {
                 ? { ok: true, value: scheduleById(deps.db, parsed.data.id) }
                 : { ok: false, reason: "conflict" };
         }
-        const checked = checkTemplate(deps, parsed.data.templateId);
+        const checked = checkKind(deps, parsed.data, []);
         if (!checked.ok)
             return checked;
-        const topics = checkTopics(deps, parsed.data, []);
-        if (!topics.ok)
-            return topics;
         const now = deps.clock.now();
-        const next = nextOccurrence(parsed.data.cadence, parsed.data.timezone, now);
-        if (next === null)
+        const next = nextRunOf(deps.db, parsed.data, now);
+        // A design schedule with no row still to come waits for rows; any other has to run.
+        if (next === null && parsed.data.templateKind === "project")
             return { ok: false, reason: "not-due" };
         // The version it was saved against, for the record; runs use the newest.
-        const value = summary({ ...parsed.data, templateVersion: checked.value }, next.toISOString(), now.toISOString());
+        const value = summary({ ...parsed.data, templateVersion: checked.value }, next?.toISOString() ?? null, now.toISOString());
         insertSchedule(deps.db, value, hash);
         return { ok: true, value };
     });
@@ -78,22 +88,25 @@ export function updateSchedule(deps, input) {
             return { ok: false, reason: "conflict" };
         if (previous.status === "canceled" || previous.status === "completed")
             return { ok: false, reason: "conflict" };
-        const checked = checkTemplate(deps, parsed.data.templateId);
+        if (parsed.data.templateKind !== previous.templateKind)
+            return {
+                ok: false,
+                reason: "invalid-input",
+                message: "A schedule can't switch between making videos and making designed pieces. Make a new schedule for the other kind.",
+            };
+        const checked = checkKind(deps, parsed.data, previous.items);
         if (!checked.ok)
             return checked;
-        const topics = checkTopics(deps, parsed.data, previous.items);
-        if (!topics.ok)
-            return topics;
         const now = deps.clock.now();
         const next = previous.status === "active"
-            ? nextOccurrence(parsed.data.cadence, parsed.data.timezone, now)
-            : previous.nextRunAt === null
+            ? nextRunOf(deps.db, parsed.data, now)
+            : previous.nextRunAt === null && previous.templateKind === "project"
                 ? null
-                : nextOccurrence(parsed.data.cadence, parsed.data.timezone, now);
+                : nextRunOf(deps.db, parsed.data, now);
         // The version it was saved against, for the record; runs use the newest.
         const value = summary({ ...parsed.data, templateVersion: checked.value }, next?.toISOString() ?? null, now.toISOString(), previous.status, previous.version + 1, 
         // Saving clears a recorded generation failure, so the next tick tries again.
-        { ...previous.topics, failedAt: null, error: null });
+        { ...previous.topics, failedAt: null, error: null }, previous.topicScoring);
         if (!updateScheduleRow(deps.db, value, parsed.data.mutationId, hash, parsed.data.baseVersion))
             return { ok: false, reason: "conflict" };
         return { ok: true, value };
@@ -113,8 +126,9 @@ export function resumeSchedule(deps, input) {
         return { ok: false, reason: "conflict" };
     if (previous.version !== parsed.data.baseVersion || previous.status !== "paused")
         return { ok: false, reason: "conflict" };
-    const next = nextOccurrence(previous.cadence, previous.timezone, deps.clock.now());
-    const status = next === null ? "completed" : "active";
+    const next = nextRunOf(deps.db, previous, deps.clock.now());
+    // A design schedule with no row to come stays active, waiting for rows.
+    const status = next === null && previous.templateKind === "project" ? "completed" : "active";
     if (!setScheduleStatus(deps.db, previous.id, previous.version, status, next?.toISOString() ?? null, deps.clock.now().toISOString()))
         return { ok: false, reason: "conflict" };
     return { ok: true, value: scheduleById(deps.db, previous.id) };
@@ -157,6 +171,24 @@ function control(deps, input, status) {
         return { ok: false, reason: "conflict" };
     return { ok: true, value: scheduleById(deps.db, previous.id) };
 }
+// Answers the template's newest version when a schedule may use it, and checks its queue
+// against that template: a project template's keywords, or a design template's fields and
+// dates.
+function checkKind(deps, input, kept) {
+    if (input.templateKind === "design")
+        return checkDesign(deps, input);
+    // Exactly as before design schedules: a project template's id is a UUID, and its schedule
+    // runs once, daily or weekly.
+    if (!z.uuid().safeParse(input.templateId).success || input.cadence.kind === "by-date")
+        return { ok: false, reason: "invalid-input" };
+    const checked = checkTemplate(deps, input.templateId);
+    if (!checked.ok)
+        return checked;
+    const topics = checkTopics(deps, input, kept);
+    if (!topics.ok)
+        return topics;
+    return checked;
+}
 // Answers the template's newest version when a schedule may use it.
 function checkTemplate(deps, id) {
     const template = deps.template(id);
@@ -178,10 +210,9 @@ function checkTemplate(deps, id) {
 // the template doesn't use would silently do nothing. Topics saved before, unchanged, are left
 // alone, so a queue from before this check still saves.
 function checkTopics(deps, input, kept) {
-    const template = deps.template(input.templateId);
-    if (!template)
+    const keywords = scheduleKeywords(deps, input.templateId);
+    if (keywords === undefined)
         return { ok: false, reason: "missing-template" };
-    const keywords = templateKeywords(template.document.form);
     const named = keywords.length === 0 ? "none" : keywords.map((one) => `“${one}”`).join(", ");
     const problems = [];
     if (input.topicKeyword !== null && !keywords.includes(input.topicKeyword))
@@ -200,10 +231,11 @@ function checkTopics(deps, input, kept) {
         message: `The schedule wasn't saved. ${shown.join(" ")}${more > 0 ? ` And ${String(more)} more like these.` : ""}`,
     };
 }
-function summary(input, nextRunAt, now, status = "active", version = 1, topics = topicStateIdle) {
+function summary(input, nextRunAt, now, status = "active", version = 1, topics = topicStateIdle, topicScoring = "off") {
     return {
         id: input.id,
         name: input.name,
+        templateKind: input.templateKind,
         templateId: input.templateId,
         templateVersion: input.templateVersion,
         cadence: input.cadence,
@@ -217,6 +249,7 @@ function summary(input, nextRunAt, now, status = "active", version = 1, topics =
         brief: input.brief === null || input.brief === "" ? null : input.brief,
         topicGeneration: input.topicGeneration,
         releases: input.releases,
+        topicScoring: input.topicScoring ?? topicScoring,
         topics,
         status,
         version,

package/dist/slices/schedules/topic-list.js

@@ -161,3 +161,18 @@ export function formatTopicList(rows, topicKeyword) {
         : { [topicKeyword]: row.title, ...row.values });
     return stringify(items, { lineWidth: 0 });
 }
+// The keywords a schedule's topics may fill: the version it runs, and the newest one, which a
+// schedule may be set up for before it is marked ready (a keyword the run doesn't use is
+// filled and left unused).
+export function scheduleKeywords(deps, templateId) {
+    const runs = deps.template(templateId);
+    if (runs === undefined)
+        return undefined;
+    const head = deps.templateHead?.(templateId);
+    return [
+        ...new Set([
+            ...templateKeywords(runs.document.form),
+            ...(head === undefined ? [] : templateKeywords(head.document.form)),
+        ]),
+    ];
+}

package/dist/slices/schedules/topic-scores.js

@@ -0,0 +1,104 @@
+import { z } from "zod";
+import { noveltyOf, orderQueue, readingOf, topicKey, topicScore, } from "./score.js";
+// ceiling: three topics asked at once, and the whole scoring of one generation stops after
+// 15 seconds; what wasn't read by then is scored by novelty alone, never held up.
+const concurrency = 3;
+export const scoringBudgetMs = 15_000;
+const scoreRow = z.object({
+    topic_key: z.string(),
+    title: z.string(),
+    score: z.number(),
+    reasons: z.string(),
+    pinned: z.number(),
+});
+export function scoresOf(db, scheduleId) {
+    return new Map(db
+        .prepare("SELECT topic_key,title,score,reasons,pinned FROM topic_scores WHERE schedule_id=?")
+        .all(scheduleId)
+        .map((raw) => {
+        const row = scoreRow.parse(raw);
+        return [
+            row.topic_key,
+            {
+                title: row.title,
+                score: row.score,
+                reasons: z.array(z.string()).parse(JSON.parse(row.reasons)),
+                pinned: row.pinned === 1,
+            },
+        ];
+    }));
+}
+// A topic scored again keeps whether the person pinned it. `pinNew` scores topics that had no
+// score (ones the person added) without moving them.
+export function saveScores(db, scheduleId, scores, at, pinNew = false) {
+    const upsert = db.prepare(`INSERT INTO topic_scores (schedule_id,topic_key,title,score,interest,novelty,reasons,pinned,scored_at)
+     VALUES (?,?,?,?,?,?,?,${pinNew ? 1 : 0},?)
+     ON CONFLICT(schedule_id,topic_key) DO UPDATE SET title=excluded.title,score=excluded.score,
+       interest=excluded.interest,novelty=excluded.novelty,reasons=excluded.reasons,
+       scored_at=excluded.scored_at`);
+    for (const [title, score] of scores)
+        upsert.run(scheduleId, topicKey(title), title, score.score, score.interest, score.novelty, JSON.stringify(score.reasons), at);
+}
+// A topic the person moved stays where they put it.
+export function pinTopic(db, scheduleId, title) {
+    db.prepare("UPDATE topic_scores SET pinned=1 WHERE schedule_id=? AND topic_key=?").run(scheduleId, topicKey(title));
+}
+// The queue in score order (`orderQueue`), from the scores kept for it.
+export function rankedItems(db, scheduleId, items) {
+    return orderQueue(items, scoresOf(db, scheduleId));
+}
+// Scores each title: its novelty against the library, and with "search" its public interest.
+// "library" never asks anything outside this computer. Never throws: a topic search didn't
+// answer for in time is scored by novelty and says so.
+export async function scoreTopics(deps, scoring, titles, known, language, signal) {
+    const readings = new Map();
+    const interest = deps.interest;
+    if (scoring === "search" && interest !== undefined) {
+        const budget = AbortSignal.any([
+            AbortSignal.timeout(scoringBudgetMs),
+            ...(signal === undefined ? [] : [signal]),
+        ]);
+        const queue = [...titles];
+        const worker = async () => {
+            for (let title = queue.shift(); title !== undefined; title = queue.shift()) {
+                if (budget.aborted)
+                    return;
+                try {
+                    readings.set(title, await interest(title, language, budget));
+                }
+                catch {
+                    // The port never throws; a broken adapter is treated as no connection.
+                }
+            }
+        };
+        await Promise.race([
+            Promise.all(Array.from({ length: concurrency }, worker)),
+            new Promise((resolve) => budget.addEventListener("abort", () => resolve(), { once: true })),
+        ]);
+    }
+    return new Map(titles.map((title) => {
+        // A queued topic is in the library's list itself; it is compared with the others.
+        const novelty = noveltyOf(title, {
+            topics: known.topics.filter((other) => topicKey(other) !== topicKey(title)),
+            projects: known.projects,
+        });
+        if (scoring === "library")
+            return [title, topicScore(novelty, null, "off")];
+        const result = readings.get(title);
+        const reading = result?.ok === true ? readingOf(title, result) : null;
+        return [
+            title,
+            topicScore(novelty, reading, reading === null ? "unreachable" : "read"),
+        ];
+    }));
+}
+// The scores kept for these titles, by key, as the schedule page and agents read them.
+export function scoresFor(db, scheduleId, titles) {
+    const kept = scoresOf(db, scheduleId);
+    return Object.fromEntries(titles.flatMap((title) => {
+        const found = kept.get(topicKey(title));
+        return found === undefined
+            ? []
+            : [[topicKey(title), { score: found.score, reasons: found.reasons }]];
+    }));
+}