@gentbajko/slopify 2.4.0 → 3.0.0

package/dist/kernel/runner/graph.js

@@ -1,33 +1,39 @@
 // Images use saved prompts and can run immediately. A thumbnail's source and an
 // audio-only export narrow these dependencies through dependenciesOf(). The document is
 // laid out from the article alone, with the thumbnail as its cover when there is one, so it
-// runs beside narration and images rather than after them.
+// runs beside narration and images rather than after them. The video never reads the
+// thumbnail, so a thumbnail that fails does not hold the video back.
 export const deps = {
     research: [],
     article: ["research"],
     audio: ["article"],
     images: [],
     thumbnail: ["article"],
-    video: ["article", "audio", "images", "thumbnail"],
+    video: ["article", "audio", "images"],
     document: ["article", "thumbnail"],
 };
 // `provided` and `skipped` release a dependency exactly as `done` does.
 export function satisfied(state) {
     return state === "done" || state === "provided" || state === "skipped";
 }
+// What a run is for, in order: the first of these the run asks for is the one that has to be
+// there for the run to count as finished. Anything else failing leaves it done with
+// problems rather than failed.
+const headline = ["video", "audio", "article"];
 // Stage state is derived; an explicit persisted pause takes precedence while calls
-// drain. Completed independent images do not imply that a run was canceled.
+// drain. Completed independent images do not imply that a run was canceled. A stage
+// waiting out a rate limit is still part of a run in progress.
 export function derive(stages, paused = false) {
     if (paused)
         return "paused";
-    if (stages.some((stage) => stage.state === "running")) {
+    if (stages.some((stage) => stage.state === "running" || waiting(stage))) {
         return "running";
     }
     if (stages.some((stage) => stage.state === "canceled")) {
         return "canceled";
     }
     if (stages.some((stage) => stage.state === "failed")) {
-        return "failed";
+        return headlineMade(stages) ? "partial" : "failed";
     }
     if (stages.length > 0 && stages.every((stage) => satisfied(stage.state))) {
         return "done";
@@ -35,6 +41,19 @@ export function derive(stages, paused = false) {
     // Includes a run created but not yet claimed and unfinished work after restart.
     return "pending";
 }
+function waiting(stage) {
+    return stage.state === "pending" && stage.retryAt !== undefined && stage.retryAt !== null;
+}
+function headlineMade(stages) {
+    for (const kind of headline) {
+        const stage = stages.find((one) => one.kind === kind);
+        if (stage === undefined || stage.state === "skipped")
+            continue;
+        // A file the user supplied is not something this run made.
+        return stage.state === "done";
+    }
+    return false;
+}
 export function dependenciesOf(kind, sources) {
     if (kind === "thumbnail" && sources.thumbnail === "from_prompt")
         return [];

package/dist/kernel/runner/index.js

@@ -1,6 +1,7 @@
 import { causedBy } from "../errors.js";
 import { derive, deps as graph, satisfied } from "./graph.js";
 import { progressGate } from "./progress.js";
+import { faultOf } from "./retry-policy.js";
 const pauseReason = new Error("paused by user");
 export function createRunner(deps) {
     const inflight = new Map();
@@ -15,7 +16,7 @@ export function createRunner(deps) {
     const stopped = new Set();
     let count = 0;
     let shuttingDown = false;
-    const emitStage = (stage, state, reason) => {
+    const emitStage = (stage, state, reason, detail = {}) => {
         deps.emit(stage.projectId, {
             type: "stage.state",
             revisionId: stage.work.revisionId,
@@ -24,6 +25,8 @@ export function createRunner(deps) {
             stage: stage.kind,
             state,
             ...(reason === null ? {} : { failureReason: reason }),
+            ...(detail.retryAt === undefined ? {} : { retryAt: detail.retryAt }),
+            ...(detail.failureKind === undefined ? {} : { failureKind: detail.failureKind }),
         });
     };
     // The global tally counts projects with a stage in flight, read off the in-flight set
@@ -46,9 +49,9 @@ export function createRunner(deps) {
         announced.set(projectId, state);
         deps.emit(projectId, { type: "project.state", projectId, state });
     };
-    const conclude = (stage, state, reason) => {
+    const conclude = (stage, state, reason, kind) => {
         try {
-            deps.stages.finish(stage.work, state, reason);
+            deps.stages.finish(stage.work, state, reason, kind ?? null);
         }
         catch (error) {
             // The row stays `running` when the write fails. The event still goes out, and the
@@ -59,7 +62,31 @@ export function createRunner(deps) {
                 detail: reasonOf(error),
             });
         }
-        emitStage(stage, state, reason);
+        emitStage(stage, state, reason, { failureKind: kind });
+    };
+    // False when the failure is to be shown now.
+    const deferred = (stage, fault, reason) => {
+        let at;
+        try {
+            at = deps.stages.waitToRetry?.(stage.work, fault, reason);
+        }
+        catch (error) {
+            deps.log.write("error", "stage.retry", {
+                projectId: stage.projectId,
+                stage: stage.kind,
+                detail: reasonOf(error),
+            });
+            return false;
+        }
+        if (at === undefined)
+            return false;
+        deps.log.write("warn", "stage.retry", {
+            projectId: stage.projectId,
+            stage: stage.kind,
+            detail: `waiting until ${at} to run again after ${fault?.kind ?? "a failure"}: ${reason}`,
+        });
+        emitStage(stage, "pending", reason, { retryAt: at, failureKind: fault?.kind });
+        return true;
     };
     async function execute(stage, controller) {
         const tag = (event) => ({
@@ -103,12 +130,15 @@ export function createRunner(deps) {
                 conclude(stage, paused ? "pending" : "canceled", paused ? null : "canceled by user");
                 return;
             }
+            const fault = faultOf(error);
+            if (deferred(stage, fault, reasonOf(error)))
+                return;
             deps.log.write("error", "stage.failed", {
                 projectId: stage.projectId,
                 stage: stage.kind,
                 detail: causedBy(error),
             });
-            conclude(stage, "failed", reasonOf(error));
+            conclude(stage, "failed", reasonOf(error), fault?.kind);
         }
     }
     function start(stage) {

package/dist/kernel/runner/meter.js

@@ -0,0 +1 @@
+export {};

package/dist/kernel/runner/providers.js

@@ -1,5 +1,6 @@
 import { randomUUID } from "node:crypto";
-import { providerError } from "../ports/model.js";
+import { isProviderError, providerError } from "../ports/model.js";
+import { planAccountOf } from "../ports/plan-limits.js";
 import { attempt } from "./attempt.js";
 export function stageProviders(deps, context, pieceId) {
     const ctx = {
@@ -15,7 +16,44 @@ export function stageProviders(deps, context, pieceId) {
         ...(pieceId === undefined ? {} : { pieceId }),
         signal: context.signal,
     };
-    const schedule = (provider, work) => deps.queue ? deps.queue.run(provider, context.signal, work) : work();
+    const queued = (provider, work) => deps.queue ? deps.queue.run(provider, context.signal, work) : work();
+    const waiter = { projectId: context.stage.projectId, stage: context.stage.kind };
+    // A CLI whose plan allowance is used up is waited for here, outside the queue, so the wait
+    // holds no provider slot; the call is then made afresh with a full set of attempts.
+    const schedule = async (provider, work) => {
+        const account = planAccountOf(provider);
+        const gate = deps.limits;
+        if (account === undefined || gate === undefined)
+            return queued(provider, work);
+        for (;;) {
+            await gate.ready(account, waiter, context.signal);
+            try {
+                return await queued(provider, work);
+            }
+            catch (error) {
+                const hit = isProviderError(error) ? error.fault.planLimit : undefined;
+                if (hit === undefined || context.signal.aborted)
+                    throw error;
+                gate.exhausted(hit);
+            }
+        }
+    };
+    // Metering never fails a call: the answer is already paid for.
+    const meter = (call) => {
+        if (deps.meter === undefined)
+            return;
+        try {
+            deps.meter.record({ ...call, projectId: context.stage.projectId, stage: context.stage.kind });
+        }
+        catch (error) {
+            deps.log.write("warn", "usage.record", {
+                projectId: context.stage.projectId,
+                stage: context.stage.kind,
+                detail: error instanceof Error ? error.message : String(error),
+            });
+        }
+    };
+    const since = (started) => Math.max(0, deps.clock.now().getTime() - started);
     return {
         llm: (call, onEvent) => {
             // Resolved once; the adapter reads the stored key per request, so a key replaced
@@ -32,11 +70,17 @@ export function stageProviders(deps, context, pieceId) {
                 text,
                 ...(reset === undefined ? {} : { reset }),
             });
-            return schedule(call.provider, () => attempt(ctx, async (signal, progress) => {
+            const started = deps.clock.now().getTime();
+            let limits;
+            const answered = schedule(call.provider, () => attempt(ctx, async (signal, progress) => {
                 preview("", true);
                 let text = "";
+                // Typed-ahead text the panel shows past the committed answer; the next delta
+                // replaces it, so the panel never shows a sentence twice.
+                let typed = false;
                 let usage = null;
                 let finishReason = null;
+                limits = undefined;
                 for await (const event of port.complete({
                     model: call.model,
                     ...(call.thinkingConfig === undefined ? {} : { thinkingConfig: call.thinkingConfig }),
@@ -44,6 +88,7 @@ export function stageProviders(deps, context, pieceId) {
                     messages: call.messages,
                     ...(call.documents === undefined ? {} : { documents: call.documents }),
                     ...(call.webSearch === undefined ? {} : { webSearch: call.webSearch }),
+                    ...(call.images === undefined ? {} : { images: call.images }),
                     signal,
                 })) {
                     signal.throwIfAborted();
@@ -51,13 +96,22 @@ export function stageProviders(deps, context, pieceId) {
                     progress();
                     if (event.type === "delta") {
                         text += event.text;
+                        if (typed)
+                            preview(text, true);
+                        else
+                            preview(event.text);
+                        typed = false;
+                    }
+                    else if (event.type === "partial") {
+                        typed = true;
                         preview(event.text);
                     }
                     else if (event.type === "done") {
                         usage = event.usage;
                         finishReason = event.finishReason;
+                        limits = event.limits;
                     }
-                    if (event.type !== "activity")
+                    if (event.type === "delta" || event.type === "done")
                         onEvent?.(event);
                 }
                 const answer = { text, usage, finishReason };
@@ -68,6 +122,28 @@ export function stageProviders(deps, context, pieceId) {
                 }
                 return answer;
             }, { kind: "llm", streaming: port.capabilities.streams }));
+            return answered.then((result) => {
+                if (result.ok) {
+                    const usage = result.value.usage;
+                    meter({
+                        kind: "llm",
+                        provider: call.provider,
+                        model: usage?.model ?? call.model,
+                        ...(usage === null
+                            ? {}
+                            : {
+                                tokensIn: usage.inputTokens,
+                                tokensOut: usage.outputTokens,
+                                ...(usage.cachedInputTokens === undefined
+                                    ? {}
+                                    : { cachedTokens: usage.cachedInputTokens }),
+                            }),
+                        wallMs: since(started),
+                        ...(limits === undefined ? {} : { limits }),
+                    });
+                }
+                return result;
+            });
         },
         tts: (call, observe) => {
             const port = deps.registry.tts(call.provider);
@@ -95,13 +171,20 @@ export function stageProviders(deps, context, pieceId) {
                     });
                 }
             };
-            return schedule(call.provider, () => attempt({ ...ctx, continuation }, async (signal, progress) => {
+            const started = deps.clock.now().getTime();
+            const spoken = schedule(call.provider, () => attempt({ ...ctx, continuation }, async (signal, progress) => {
                 notify({ type: "start" });
                 try {
+                    if (call.dialogue !== undefined && port.capabilities.dialogue !== true)
+                        throw providerError({
+                            kind: "unsupported",
+                            message: `${call.provider} can't speak several voices in one request. Turn off Native multi-speaker under Speakers (Play → Audio, or Edit project → Providers), then Try again.`,
+                        });
                     const spoken = await port.synthesize({
                         model: call.model,
                         voiceId: call.voiceId,
                         text: call.text,
+                        ...(call.dialogue === undefined ? {} : { dialogue: call.dialogue }),
                         signal,
                         onActivity: progress,
                         continuation,
@@ -138,17 +221,76 @@ export function stageProviders(deps, context, pieceId) {
                     throw error;
                 }
             }, { kind: "tts", streaming: port.capabilities.streams }));
+            return spoken.then((result) => {
+                // Narration is billed by the characters sent, which is the text of this call.
+                if (result.ok)
+                    meter({
+                        kind: "tts",
+                        provider: call.provider,
+                        model: call.model ?? "",
+                        characters: call.text.length,
+                        wallMs: since(started),
+                    });
+                return result;
+            });
         },
         image: (call) => {
             const port = deps.registry.image(call.provider);
-            return schedule(call.provider, () => attempt(ctx, (signal) => port.generate({
+            const callId = randomUUID();
+            // A long job (the Codex CLI reviewing and redrawing its image) says how far it has got
+            // on the stage's live panel; the other providers report nothing and show no panel.
+            const onProgress = (text) => context.emit({
+                type: "llm.preview",
+                ...(pieceId === undefined ? {} : { workPieceId: pieceId }),
+                projectId: context.stage.projectId,
+                stage: context.stage.kind,
+                callId,
+                label: call.previewLabel ?? "Image",
+                text,
+                reset: true,
+            });
+            const started = deps.clock.now().getTime();
+            const drawn = schedule(call.provider, () => attempt(ctx, (signal) => port.generate({
                 model: call.model,
                 prompt: call.prompt,
                 aspect: call.aspect,
+                ...(call.thinking === undefined ? {} : { thinking: call.thinking }),
+                ...(call.reference === undefined ? {} : { reference: call.reference }),
+                ...(call.cast === undefined ? {} : { cast: call.cast }),
+                onProgress,
                 signal,
             }), 
-            // One request, one answer: the 300 s runs over the whole call.
-            { kind: "image" }));
+            // One request, one answer: the limit runs over the whole call - 300 s, or the
+            // provider's own when it needs longer.
+            {
+                kind: "image",
+                ...(port.timeoutMs === undefined ? {} : { timeoutMs: port.timeoutMs }),
+            }));
+            return drawn.then((result) => {
+                if (result.ok) {
+                    const { usage, limits } = result.value;
+                    meter({
+                        kind: "image",
+                        provider: call.provider,
+                        model: call.model,
+                        images: 1,
+                        size: call.aspect,
+                        ...(call.thinking === undefined ? {} : { quality: call.thinking }),
+                        ...(usage === undefined
+                            ? {}
+                            : {
+                                tokensIn: usage.inputTokens,
+                                tokensOut: usage.outputTokens,
+                                ...(usage.cachedInputTokens === undefined
+                                    ? {}
+                                    : { cachedTokens: usage.cachedInputTokens }),
+                            }),
+                        wallMs: since(started),
+                        ...(limits === undefined ? {} : { limits }),
+                    });
+                }
+                return result;
+            });
         },
         animate: (call) => {
             const port = deps.registry.image(call.provider);
@@ -158,7 +300,8 @@ export function stageProviders(deps, context, pieceId) {
                     kind: "unsupported",
                     message: `${call.provider} can't turn images into video clips. Choose fal.ai or Replicate as the image provider in Edit project → Providers, or turn Animate images off in Edit project → Inputs → Look.`,
                 }));
-            return schedule(call.provider, () => attempt(ctx, (signal) => animate({
+            const started = deps.clock.now().getTime();
+            const animated = schedule(call.provider, () => attempt(ctx, (signal) => animate({
                 model: call.model,
                 prompt: call.prompt,
                 image: call.image,
@@ -168,6 +311,18 @@ export function stageProviders(deps, context, pieceId) {
             }), 
             // Queued and slow: the 900 s runs over the whole call, polling included.
             { kind: "video" }));
+            return animated.then((result) => {
+                if (result.ok)
+                    meter({
+                        kind: "video",
+                        provider: call.provider,
+                        model: call.model,
+                        seconds: call.seconds,
+                        size: call.aspect,
+                        wallMs: since(started),
+                    });
+                return result;
+            });
         },
         forPiece: (piece) => stageProviders(deps, context, piece),
     };

package/dist/kernel/runner/retry-policy.js

@@ -0,0 +1,43 @@
+import { isProviderError } from "../ports/model.js";
+// The second tier of the retry policy. `attempt.ts` retries a failed call a few times within
+// seconds; when those run out on a failure that time can fix, the step is put back to wait
+// and runs again later, so a rate limit, a timeout or a dropped connection costs the user
+// nothing but time. The wait is written down, not held in memory: a restart keeps it.
+// The failures worth waiting out. A refusal, a rejected or missing key, an unsupported
+// request and a lost submission are the provider's final answer (or need a person), so
+// they are shown at once. `other` covers a request the provider rejected as well as an
+// unreadable answer; the quick in-call retries are all it gets.
+export const waitableKinds = ["rate_limit", "timeout", "dropped"];
+// Four waits after the first failed run: about 2, 4, 8 and 16 minutes, 30 in all.
+export const autoRetryLimit = 4;
+export const autoRetryBaseMs = 2 * 60_000;
+// ±25 %, so a burst of steps that hit the same limit together do not all return together.
+export const autoRetryJitter = 0.25;
+// A provider asking for longer than this (a daily quota) is reported rather than waited on.
+export const retryAfterCeilingMs = 60 * 60_000;
+// A slice may wrap the wrapper's error in its own sentence ("Image 3: ..."); the kind is on
+// the cause.
+export function faultOf(error) {
+    let current = error;
+    for (let depth = 0; depth < 6 && current !== undefined && current !== null; depth += 1) {
+        if (isProviderError(current))
+            return current.fault;
+        current = current instanceof Error ? current.cause : undefined;
+    }
+    return undefined;
+}
+// `retries` is how many waits the step already had. Undefined: report the failure now.
+export function autoRetryDelay(fault, retries, random) {
+    if (fault === undefined || !waitableKinds.includes(fault.kind))
+        return undefined;
+    if (retries >= autoRetryLimit)
+        return undefined;
+    const nominal = autoRetryBaseMs * 2 ** retries;
+    const jittered = Math.round(nominal * (1 - autoRetryJitter + 2 * autoRetryJitter * random()));
+    if (fault.retryAfterMs === undefined)
+        return jittered;
+    if (fault.retryAfterMs > retryAfterCeilingMs)
+        return undefined;
+    // The provider's own wait is the floor; ours still spreads the burst above it.
+    return Math.max(fault.retryAfterMs, jittered);
+}

package/dist/kernel/runner/standalone.js

@@ -0,0 +1,134 @@
+import { isProviderError } from "../ports/model.js";
+import { attempt } from "./attempt.js";
+export async function standaloneLlm(deps, call) {
+    const port = deps.registry.llm(call.provider);
+    const started = deps.clock.now().getTime();
+    const answer = await wrapped(deps, call, async (signal, progress) => {
+        let text = "";
+        let usage = null;
+        let limits;
+        for await (const event of port.complete({
+            model: call.model,
+            ...(call.thinking === undefined ? {} : { thinking: call.thinking }),
+            messages: call.messages,
+            signal,
+        })) {
+            signal.throwIfAborted();
+            // Every event is a sign of life, so the idle clock restarts here.
+            progress();
+            if (event.type === "delta")
+                text += event.text;
+            else if (event.type === "done") {
+                usage = event.usage;
+                limits = event.limits;
+            }
+        }
+        return { text, usage, ...(limits === undefined ? {} : { limits }) };
+    }, { kind: "llm", streaming: port.capabilities.streams });
+    meter(deps, call, {
+        kind: "llm",
+        provider: call.provider,
+        model: answer.usage?.model ?? call.model,
+        ...tokens(answer.usage ?? undefined),
+        wallMs: Math.max(0, deps.clock.now().getTime() - started),
+        ...(answer.limits === undefined ? {} : { limits: answer.limits }),
+    });
+    return answer;
+}
+export async function standaloneImage(deps, call) {
+    const port = deps.registry.image(call.provider);
+    const started = deps.clock.now().getTime();
+    const image = await wrapped(deps, call, (signal) => port.generate({
+        model: call.model,
+        prompt: call.prompt,
+        aspect: call.aspect,
+        signal,
+    }), { kind: "image", ...(port.timeoutMs === undefined ? {} : { timeoutMs: port.timeoutMs }) });
+    meter(deps, call, {
+        kind: "image",
+        provider: call.provider,
+        model: call.model,
+        images: 1,
+        size: call.aspect,
+        ...tokens(image.usage),
+        wallMs: Math.max(0, deps.clock.now().getTime() - started),
+        ...(image.limits === undefined ? {} : { limits: image.limits }),
+    });
+    return image;
+}
+async function wrapped(deps, use, call, opts) {
+    // The wrapper names its log lines by project and stage; these name what the call was for.
+    const label = `${use.purpose}:${use.owner.kind}:${use.owner.id}`;
+    let next = 0;
+    const attempts = {
+        start: () => {
+            next += 1;
+            return String(next);
+        },
+        end: () => { },
+    };
+    const stage = opts.kind === "image" ? "images" : "article";
+    try {
+        const result = await attempt({
+            work: {
+                projectId: label,
+                revisionId: label,
+                workId: label,
+                stageId: label,
+                kind: stage,
+                fingerprint: label,
+            },
+            maySubmit: () => true,
+            clock: deps.clock,
+            log: deps.log,
+            attempts,
+            projectId: label,
+            stage,
+            stageId: label,
+            signal: use.signal ?? new AbortController().signal,
+        }, call, opts);
+        if (!result.ok)
+            throw new Error("The provider did not take the request.");
+        return result.value;
+    }
+    catch (error) {
+        throw standaloneError(error);
+    }
+}
+// The wrapper's own advice points at a stage's buttons ("use Try again"); a standalone call
+// has none, and each caller says itself where to go next, so that advice is dropped.
+function standaloneError(error) {
+    if (!isProviderError(error))
+        return error;
+    const kept = error.message
+        .replace(/,? then use (?:Retry stage|Try again)(?:, or choose another model in the Providers section of Edit project)?/g, "")
+        .split(/(?<=\.)\s+/)
+        .filter((sentence) => !/Retry stage|Try again|Edit project/.test(sentence))
+        .join(" ")
+        .trim();
+    if (kept === error.message || kept === "")
+        return error;
+    return Object.assign(new Error(kept, { cause: error }), { fault: error.fault });
+}
+function tokens(usage) {
+    if (usage === undefined)
+        return {};
+    return {
+        tokensIn: usage.inputTokens,
+        tokensOut: usage.outputTokens,
+        ...(usage.cachedInputTokens === undefined ? {} : { cachedTokens: usage.cachedInputTokens }),
+    };
+}
+// Metering never fails a call: the answer is already paid for.
+function meter(deps, use, call) {
+    if (deps.meter === undefined)
+        return;
+    try {
+        deps.meter.record({ ...call, owner: use.owner, purpose: use.purpose });
+    }
+    catch (error) {
+        deps.log.write("warn", "usage.record", {
+            detail: `${use.purpose} for ${use.owner.kind} ${use.owner.id}: ${error instanceof Error ? error.message : String(error)}`,
+        });
+    }
+}

package/dist/kernel/runner/work-authority.js

@@ -29,9 +29,35 @@ export function ownsCurrentWork(db, work, pieceId) {
         .prepare(`SELECT 1 FROM revision_work_reservations r JOIN project_heads h ON h.project_id=r.project_id AND h.revision_id=r.revision_id JOIN project_revisions v ON v.id=r.revision_id JOIN json_each(v.fingerprints) f ON f.key=COALESCE(r.logical_key,r.work_key) AND f.value=COALESCE(r.desired_fingerprint,r.fingerprint) WHERE r.project_id=? AND r.work_id=? AND r.piece_id IS ? AND (r.piece_id IS NULL OR EXISTS(SELECT 1 FROM revision_work_pieces p WHERE p.id=r.piece_id AND p.work_id=r.work_id AND p.work_key=r.work_key AND p.fingerprint=r.fingerprint))`)
         .get(work.projectId, work.workId, pieceId) !== undefined);
 }
-export function finishWork(db, work, state, reason) {
+export function finishWork(db, work, state, reason, kind = null) {
     if (!workExists(db, work))
         return;
     const normalized = state === "provided" || state === "skipped" ? "done" : state;
-    db.prepare(`UPDATE revision_work SET state=?,failure_reason=?,dispatch_state=CASE WHEN ?='pending' THEN 'held' ELSE dispatch_state END WHERE id=?`).run(normalized, reason, normalized, work.workId);
+    db.prepare(`UPDATE revision_work SET state=?,failure_reason=?,failure_kind=?,retry_at=NULL,dispatch_state=CASE WHEN ?='pending' THEN 'held' ELSE dispatch_state END WHERE id=?`).run(normalized, reason, normalized === "failed" ? kind : null, normalized, work.workId);
+}
+// How many times the automatic retry already put this step back to wait.
+export function autoRetriesOf(db, work) {
+    const row = db.prepare("SELECT auto_retries FROM revision_work WHERE id=?").get(work.workId);
+    return typeof row?.auto_retries === "number" ? row.auto_retries : 0;
+}
+// A running step that failed on something time can fix goes back to `pending` with the time
+// it may start again. It stays dispatchable, unlike a paused one, and its unfinished pieces
+// may be sent again: the failure that ended them was not a lost submission.
+export function deferWork(db, work, at, reason, kind) {
+    if (!workExists(db, work))
+        return false;
+    const moved = db
+        .prepare(`UPDATE revision_work SET state='pending',retry_at=?,auto_retries=auto_retries+1,failure_reason=?,failure_kind=? WHERE id=? AND state='running' AND dispatch_state='allowed'`)
+        .run(at, reason, kind, work.workId).changes === 1;
+    if (moved)
+        db.prepare("UPDATE revision_work_pieces SET state='pending' WHERE work_id=? AND state IN ('failed','running')").run(work.workId);
+    return moved;
+}
+// The projects with a wait that has run out, their wait cleared so the next tick may claim
+// them. A paused project keeps its wait: Resume ticks it, and nothing else should.
+export function dueRetries(db, now) {
+    const rows = db
+        .prepare(`UPDATE revision_work SET retry_at=NULL WHERE state='pending' AND retry_at IS NOT NULL AND retry_at<=? AND NOT EXISTS(SELECT 1 FROM project_controls c WHERE c.project_id=revision_work.project_id AND c.paused=1) RETURNING project_id`)
+        .all(now);
+    return [...new Set(rows.map((row) => String(row.project_id)))];
 }