cloving 1.2.0 → 1.4.0

package/dist/src/daemon/retrying-turn.js

@@ -1,39 +1,17 @@
-import { join } from "node:path";
-import { mkdir } from "node:fs/promises";
 import { setTimeout as delay } from "node:timers/promises";
-import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
-import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
-import { analysisCommand } from "./analysis-launch.js";
-import { capacityRefusal, startTurn, } from "./claude-turn.js";
+import { startTurn, } from "./claude-turn.js";
+import { turnHead, retryEvidence, indexFailure } from "./retry-worktree.js";
 import { readRecord, writeRecord } from "./operation-journal.js";
-const waits = [60_000, 180_000];
+import { resumeWorktree } from "./resume-worktree.js";
+import { readResume, resumedCommand, resumePrompt } from "./claude-recovery.js";
+import { attemptCommand, combined, kindOf, } from "./turn-attempts.js";
+import { prepareAttempt, attemptJournal, recordedEnd, } from "./turn-attempt-journal.js";
+const waits = { capacity: [60_000, 180_000], network: [600_000, 2_400_000] };
+const networkContinuation = "An earlier attempt stopped because the network was not available. Continue from the current state of the work tree. Check each earlier commit, push, review comment and message. Do not repeat a write that is done. Stop and report a write whose result is not known.";
 const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
 function written(folder, entry) {
     return writeRecord(folder, { ...entry, permissions: "private" });
 }
-function total(values) {
-    if (values.includes(null))
-        return null;
-    return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
-}
-function combined(attempts) {
-    const { ends } = attempts;
-    const last = ends.at(-1) ?? { exitCode: null, result: undefined };
-    const result = last.result ?? ends.find((end) => end.result)?.result;
-    if (result === undefined)
-        return last;
-    return {
-        ...last,
-        result: {
-            ...result,
-            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
-            ...(last.result === undefined ? { failed: true, text: "" } : {}),
-            usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
-            costUsd: total(ends.map((end) => end.result?.costUsd ?? null)),
-            turns: total(ends.map((end) => end.result?.turns ?? null)),
-        },
-    };
-}
 function controlled(attempts, start) {
     const turn = startTurn(start);
     const stop = () => {
@@ -47,87 +25,51 @@ function controlled(attempts, start) {
         }),
     };
 }
-function attemptCommand(attempts, attempt) {
-    const { start, options } = attempts;
-    const original = start.mode === "analysis" ? analysisCommand() : start.command;
-    const fallback = start.mode === "analysis"
-        ? undefined
-        : options.fallbackModels?.[attempt - 3];
-    const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
-    const model = commandModel(command);
-    return {
-        command,
-        model,
-        ...(fallback !== undefined && model === null
-            ? { fallbackReason: "fallback-model-unavailable" }
-            : {}),
-    };
+async function promptOf(attempts) {
+    const { start, ends, stopping, turnStartCommit } = attempts;
+    if (ends.length === 0)
+        return start.prompt;
+    if (!ends.some((end) => kindOf(attempts, end) === "network"))
+        return continuation + start.prompt;
+    const evidence = await retryEvidence({ directory: start.directory, signal: stopping.signal }, turnStartCommit);
+    return `${networkContinuation}\n\nCommits since the start of this turn:\n${evidence}\n\n${start.prompt}`;
 }
-function attemptUsage(attempts, completion) {
-    const { end, attempt } = completion;
-    const model = end.result?.model ?? attemptCommand(attempts, attempt).model;
-    const usage = end.result?.usage ?? usageOf(null);
-    return {
-        attempt,
-        model,
-        usage,
-        costUsd: end.result?.costUsd ?? null,
-        estimatedCostUsd: estimatedCostUsd(usage, {
-            worker: attempts.start.mode === "analysis"
-                ? "claude"
-                : (attempts.start.worker ?? "claude"),
-            price: priceForModel(attempts.options.tokenPrices ?? {}, model),
-        }),
-    };
+function launchCommand(attempts, prepared) {
+    const { entry, selected } = prepared;
+    if (entry.resumedAfter === undefined)
+        return selected.command;
+    return (resumedCommand({
+        worker: attempts.start.worker,
+        command: selected.command,
+        sessionId: attempts.options.resume?.sessionId ?? "",
+    }) ?? selected.command);
 }
-async function launched(attempts) {
-    const { start, options, ends } = attempts;
-    const attempt = ends.length + 1;
-    const selected = attemptCommand(attempts, attempt);
-    const folder = start.mode === "analysis" || attempt === 1
-        ? start.folder
-        : join(start.folder, `attempt-${String(attempt)}`);
-    const entry = {
-        ...options.metadata,
-        model: selected.model,
-        ...(selected.fallbackReason === undefined
-            ? {}
-            : { fallbackReason: selected.fallbackReason }),
-        attempt,
-        startedAt: attempt === 1 ? attempts.startedAt : new Date().toISOString(),
-    };
-    await written(start.folder, {
-        name: "started.json",
-        value: {
-            ...entry,
-            startedAt: attempts.startedAt,
-            phase: "starting",
-            processId: null,
-        },
-    });
-    await mkdir(folder, { recursive: true, mode: 0o700 });
-    if (attempts.stopping.signal.aborted)
+async function launched(attempts, prompt = attempts.start.prompt) {
+    const prepared = await prepareAttempt(attempts);
+    if (prepared === undefined || attempts.stopping.signal.aborted)
         return undefined;
+    const { start } = attempts;
+    const { entry, folder } = prepared;
+    const journal = attemptJournal(attempts, entry);
     const turn = controlled(attempts, {
         ...start,
-        command: selected.command,
+        command: launchCommand(attempts, prepared),
+        progress: journal.progress,
         folder,
-        ...(attempt === 1 ? {} : { activityFolder: start.folder }),
-        prompt: start.mode === "analysis" || attempt === 1
-            ? start.prompt
-            : continuation + start.prompt,
+        ...(entry.attempt === 1 ? {} : { activityFolder: start.folder }),
+        prompt: entry.resumedAfter === undefined ? prompt : resumePrompt,
     });
     try {
-        const value = { ...entry, phase: "running", processId: turn.processId };
-        await written(start.folder, {
-            name: "started.json",
-            value: { ...value, startedAt: attempts.startedAt },
-        });
-        await written(start.folder, {
-            name: `attempt-${String(attempt)}.json`,
-            value,
-        });
-        return turn;
+        await journal.running(turn.processId);
+        return {
+            ...turn,
+            ended: turn.ended.then((end) => ({
+                ...end,
+                ...(entry.resumedAfter === undefined
+                    ? {}
+                    : { resumedAfter: entry.resumedAfter }),
+            })),
+        };
     }
     catch (error) {
         turn.stop(0);
@@ -135,39 +77,13 @@ async function launched(attempts) {
         throw error;
     }
 }
-async function recordedEnd(attempts, completion) {
-    const { start, ends, options } = attempts;
-    const { processId, end } = completion;
-    const attempt = ends.length;
-    const selected = attemptCommand(attempts, attempt);
-    const name = `attempt-${String(attempt)}.json`;
-    const started = await readRecord(start.folder, name);
-    const startedAt = Reflect.get(Object(started), "startedAt");
-    if (typeof startedAt !== "string")
-        throw new Error("attempt-start-time-missing");
-    await written(start.folder, {
-        name,
-        value: {
-            ...options.metadata,
-            ...attemptUsage(attempts, { end, attempt }),
-            ...(selected.fallbackReason === undefined
-                ? {}
-                : { fallbackReason: selected.fallbackReason }),
-            attempt,
-            processId,
-            startedAt,
-            endedAt: new Date().toISOString(),
-            exitCode: end.exitCode,
-            capacity: capacityRefusal(end.result),
-        },
-    });
-}
 async function waited(attempts, milliseconds) {
     const { start, options, stopping, ends } = attempts;
     await written(start.folder, {
         name: "started.json",
         value: {
             ...options.metadata,
+            turnStartCommit: attempts.turnStartCommit,
             startedAt: attempts.startedAt,
             attempt: ends.length,
             phase: "waiting",
@@ -183,6 +99,27 @@ async function waited(attempts, milliseconds) {
             throw error;
     });
 }
+async function preparedRetry(attempts) {
+    const prompt = await promptOf(attempts);
+    const failure = await indexFailure({
+        directory: attempts.start.directory,
+        signal: attempts.stopping.signal,
+    });
+    if (cancelled(attempts))
+        return { ...combined(attempts), stopped: true };
+    if (failure !== null) {
+        await written(attempts.start.folder, {
+            name: "retry-failure.json",
+            value: failure,
+        });
+        return {
+            ...combined(attempts),
+            reason: failure.reason,
+            detail: `${failure.fix}${failure.lockPath === undefined ? "" : ` Lock: ${failure.lockPath}`}`,
+        };
+    }
+    return prompt;
+}
 async function finished(attempts, first) {
     const { ends } = attempts;
     let turn = first;
@@ -190,36 +127,103 @@ async function finished(attempts, first) {
         const end = await turn.ended;
         ends.push(end);
         await recordedEnd(attempts, { processId: turn.processId, end });
-        const milliseconds = waits[ends.length - 1];
-        if (cancelled(attempts) ||
-            !capacityRefusal(end.result) ||
-            milliseconds === undefined)
+        if (interruptedChild(attempts, { processId: turn.processId, end }))
+            return resumedChild(attempts);
+        const milliseconds = retryWait(attempts, end);
+        if (cancelled(attempts) || milliseconds === undefined)
             break;
         await waited(attempts, milliseconds);
         if (cancelled(attempts))
             return { ...combined(attempts), stopped: true };
-        turn = await launched(attempts);
+        const prepared = await preparedRetry(attempts);
+        if (typeof prepared !== "string")
+            return prepared;
+        turn = await launched(attempts, prepared);
     }
+    return attemptOutcome(attempts);
+}
+function interruptedChild(attempts, completion) {
+    const { start } = attempts;
+    const { processId, end } = completion;
+    return (!cancelled(attempts) &&
+        (start.worker ?? "claude") === "claude" &&
+        start.output !== "text" &&
+        processId !== null &&
+        end.exitCode !== 0 &&
+        end.result === undefined);
+}
+async function resumedChild(attempts) {
+    const started = await readRecord(attempts.start.folder, "started.json");
+    const resume = await readResume(attempts.start.folder, started);
+    if (resume === undefined)
+        return { ...combined(attempts), reason: "claude-turn-interrupted" };
+    const locked = await resumeWorktree({
+        directory: attempts.start.directory,
+        folder: attempts.start.folder,
+        signal: attempts.stopping.signal,
+        stopMilliseconds: attempts.grace.milliseconds,
+    });
+    if (locked !== null)
+        return { ...combined(attempts), reason: locked.reason };
+    const prepared = await preparedRetry(attempts);
+    if (typeof prepared !== "string")
+        return prepared;
+    const recovered = {
+        ...attempts,
+        options: { ...attempts.options, resume },
+        ends: [...resume.ends],
+    };
+    const turn = await launched(recovered);
+    return turn === undefined
+        ? attemptOutcome(recovered)
+        : finished(recovered, turn);
+}
+function retryWait(attempts, end) {
+    const kind = kindOf(attempts, end);
+    if (kind === null)
+        return undefined;
+    if (attemptCommand(attempts, attempts.ends.length + 1).budgetUsd === 0) {
+        attempts.failure = "claude-turn-budget-exhausted";
+        return undefined;
+    }
+    return waits[kind][attempts.ends.filter((end) => end.resumedAfter === undefined).length - 1];
+}
+function attemptOutcome(attempts) {
     return {
         ...combined(attempts),
-        ...(cancelled(attempts) || turn === undefined ? { stopped: true } : {}),
+        ...(attempts.failure === undefined ? {} : { reason: attempts.failure }),
+        ...(cancelled(attempts) ? { stopped: true } : {}),
     };
 }
 function cancelled(attempts) {
     return attempts.stopping.signal.aborted;
 }
 export async function startRetriedTurn(start, options) {
+    const stopping = options.stopping ?? new AbortController();
+    const turnStartCommit = options.resume === undefined
+        ? await turnHead({
+            directory: start.directory,
+            signal: stopping.signal,
+        })
+        : options.resume.turnStartCommit;
     const attempts = {
+        turnStartCommit,
         start,
-        startedAt: new Date().toISOString(),
+        startedAt: options.resume?.startedAt ?? new Date().toISOString(),
         options,
-        stopping: new AbortController(),
-        ends: [],
-        grace: { milliseconds: 2_000 },
+        stopping,
+        ends: [...(options.resume?.ends ?? [])],
+        grace: { milliseconds: options.stopMilliseconds ?? 15_000 },
     };
     const turn = await launched(attempts);
     if (turn === undefined)
-        throw new Error("turn-start-stopped");
+        return {
+            processId: null,
+            ended: Promise.resolve(attemptOutcome(attempts)),
+            stop: () => {
+                stopping.abort();
+            },
+        };
     return {
         processId: turn.processId,
         ended: finished(attempts, turn),

package/dist/src/daemon/schedule-control.d.ts

@@ -0,0 +1,26 @@
+import type { SchedulePort, ScheduleProgress } from "../schedule-progress.ts";
+import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { scheduleJournal } from "./schedule-journal.ts";
+type Journal = Awaited<ReturnType<typeof scheduleJournal>>;
+interface ScheduleState {
+    readonly journal: Journal;
+    readonly declarations: readonly DeclaredWorkflow[];
+    readonly clock: () => number;
+    readonly recoverPending: (entry: {
+        readonly workflowId: string;
+        readonly progress: ScheduleProgress;
+    }) => Promise<ScheduleProgress>;
+}
+export declare function scheduleConnection(): {
+    readonly request: SchedulePort;
+    readonly connect: (port: SchedulePort) => void;
+};
+export declare function scheduleControl(settings: ScheduleState & {
+    readonly signal: AbortSignal;
+    readonly tick: () => Promise<void>;
+}): {
+    readonly request: SchedulePort;
+    readonly tick: () => Promise<void>;
+    readonly drain: () => Promise<void>;
+};
+export {};

package/dist/src/daemon/schedule-control.js

@@ -0,0 +1,110 @@
+import { scheduleStops } from "../schedule-progress.js";
+import { runIdentity } from "../service/submission-handler.js";
+import { failed, invalidRequest, result } from "../service/replies.js";
+function summary(workflowId, progress) {
+    return {
+        workflowId,
+        everySeconds: progress.everySeconds,
+        state: progress.state,
+        failuresInRow: progress.failuresInRow,
+        failedRunIds: progress.failedRunIds,
+        lastSlot: progress.lastSlot,
+        nextAt: progress.state === "active" ? progress.nextAt : null,
+        activeRunId: progress.activeCommandId === undefined
+            ? null
+            : runIdentity(progress.activeCommandId),
+        stoppedAt: progress.stoppedAt ?? null,
+        stoppedSlot: progress.stoppedSlot ?? null,
+        stops: scheduleStops(progress),
+    };
+}
+export function scheduleConnection() {
+    let port;
+    return {
+        request: (parameters) => port?.(parameters) ??
+            Promise.resolve(failed("schedules-unavailable", {})),
+        connect: (connected) => {
+            port = connected;
+        },
+    };
+}
+async function change(settings, command) {
+    const { workflowId, action, now } = command;
+    const { journal } = settings;
+    let prior = journal.entries.get(workflowId);
+    if (prior === undefined)
+        return failed("unknown-schedule", { workflowId });
+    const state = action === "pause" ? "paused" : "active";
+    if (prior.state === state)
+        return result("schedule", { schedule: summary(workflowId, prior) });
+    if (prior.pending !== undefined) {
+        prior = await settings.recoverPending({ workflowId, progress: prior });
+        journal.entries.set(workflowId, prior);
+        await journal.save();
+    }
+    const rest = { ...prior };
+    delete rest.stoppedAt;
+    delete rest.stoppedSlot;
+    const progress = action === "pause"
+        ? { ...prior, state, stops: scheduleStops(prior) }
+        : {
+            ...rest,
+            state,
+            failuresInRow: 0,
+            failedRunIds: [],
+            stops: scheduleStops(prior),
+            nextAt: now + prior.everySeconds * 1000,
+        };
+    journal.entries.set(workflowId, progress);
+    try {
+        await journal.save();
+    }
+    catch (error) {
+        journal.entries.set(workflowId, prior);
+        throw error;
+    }
+    return result("schedule", { schedule: summary(workflowId, progress) });
+}
+function request(settings, parameters) {
+    const { action, workflowId } = parameters;
+    const scheduled = settings.declarations.filter((entry) => entry.schedule !== undefined);
+    if (action === "list")
+        return Promise.resolve(result("schedules", {
+            stops: Array.from(settings.journal.entries, ([workflowId, progress]) => ({
+                workflowId,
+                stops: scheduleStops(progress),
+            })),
+            schedules: scheduled.flatMap((entry) => {
+                const progress = settings.journal.entries.get(entry.workflowId);
+                return progress === undefined
+                    ? []
+                    : [summary(entry.workflowId, progress)];
+            }),
+        }));
+    if ((action !== "pause" && action !== "resume") ||
+        typeof workflowId !== "string")
+        return Promise.resolve(invalidRequest("Use list, pause or resume with a workflow ID."));
+    if (!scheduled.some((entry) => entry.workflowId === workflowId))
+        return Promise.resolve(failed("unknown-schedule", { workflowId }));
+    return change(settings, {
+        workflowId,
+        action,
+        now: settings.clock(),
+    });
+}
+export function scheduleControl(settings) {
+    let running = Promise.resolve();
+    const serialized = (operation) => {
+        const next = running.then(async () => {
+            settings.signal.throwIfAborted();
+            return operation();
+        });
+        running = next.then(() => undefined, () => undefined);
+        return next;
+    };
+    return {
+        request: (parameters) => serialized(() => request(settings, parameters)).catch(() => failed("schedules-unavailable", {})),
+        tick: () => serialized(settings.tick),
+        drain: () => running,
+    };
+}

package/dist/src/daemon/schedule-journal.d.ts

@@ -0,0 +1,6 @@
+import type { ScheduleProgress } from "../schedule-progress.ts";
+export type { PendingSlot, ScheduleProgress } from "../schedule-progress.ts";
+export declare function scheduleJournal(directory: string): Promise<{
+    readonly entries: Map<string, ScheduleProgress>;
+    readonly save: () => Promise<void>;
+}>;

package/dist/src/daemon/schedule-journal.js

@@ -0,0 +1,120 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { isCommandId, isRecordKey } from "../authority/commands.js";
+function pendingSlot(value) {
+    const fields = record(value);
+    if (fields?.["kind"] === "run")
+        return record(fields["input"]) !== undefined;
+    return fields?.["kind"] === "skip" && isRecordKey(fields["runId"]);
+}
+function whole(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function timing(fields) {
+    const { everySeconds, nextAt, lastSlot } = fields;
+    return (whole(everySeconds) &&
+        everySeconds >= 60 &&
+        everySeconds <= 86400 &&
+        whole(nextAt) &&
+        whole(lastSlot));
+}
+function isProgress(value) {
+    const fields = record(value);
+    if (fields === undefined)
+        return false;
+    const { activeCommandId } = fields;
+    return (timing(fields) &&
+        control(fields) &&
+        history(fields) &&
+        pendingProgress(fields) &&
+        (activeCommandId === undefined || isCommandId(activeCommandId)));
+}
+function pendingProgress(fields) {
+    const { pending, state } = fields;
+    if (pending === undefined)
+        return true;
+    return (state === undefined || state === "active") && pendingSlot(pending);
+}
+function stopped(fields) {
+    const { state, stoppedAt, stoppedSlot } = fields;
+    if (stoppedAt === undefined && stoppedSlot === undefined)
+        return state !== "stopped";
+    return (whole(stoppedAt) &&
+        whole(stoppedSlot) &&
+        stoppedSlot <= Number(fields["lastSlot"]));
+}
+function failures(fields) {
+    const { failuresInRow, failedRunIds } = fields;
+    return (whole(failuresInRow) &&
+        failuresInRow <= 3 &&
+        Array.isArray(failedRunIds) &&
+        failedRunIds.length === failuresInRow &&
+        failedRunIds.every(isRecordKey) &&
+        new Set(failedRunIds).size === failuresInRow);
+}
+function history(fields) {
+    const stops = fields["stops"];
+    if (stops === undefined)
+        return true;
+    if (!Array.isArray(stops))
+        return false;
+    let previousSlot = -1;
+    for (const item of stops) {
+        const stop = record(item);
+        if (stop === undefined)
+            return false;
+        const slot = stop["stoppedSlot"];
+        if (!whole(slot) || !whole(stop["stoppedAt"]))
+            return false;
+        if (slot <= previousSlot || slot > Number(fields["lastSlot"]))
+            return false;
+        if (!failures({ ...stop, failuresInRow: 3 }))
+            return false;
+        previousSlot = slot;
+    }
+    return true;
+}
+function control(fields) {
+    const { state, failuresInRow, failedRunIds } = fields;
+    if (state === undefined &&
+        failuresInRow === undefined &&
+        failedRunIds === undefined)
+        return (fields["stoppedAt"] === undefined && fields["stoppedSlot"] === undefined);
+    return (["active", "paused", "stopped"].includes(String(state)) &&
+        failures(fields) &&
+        stopped(fields) &&
+        (state !== "stopped" || failuresInRow === 3));
+}
+export async function scheduleJournal(directory) {
+    const file = join(directory, "schedules.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "{}";
+        throw error;
+    });
+    const parsed = record(JSON.parse(text));
+    if (parsed === undefined)
+        throw new Error("invalid-schedule-state");
+    const entries = new Map();
+    for (const [workflowId, value] of Object.entries(parsed)) {
+        if (!isProgress(value))
+            throw new Error("invalid-schedule-state");
+        entries.set(workflowId, {
+            ...value,
+            state: value.state ?? "active",
+            failuresInRow: value.failuresInRow ?? 0,
+            failedRunIds: value.failedRunIds ?? [],
+        });
+    }
+    return {
+        entries,
+        save: async () => {
+            await writeFile(`${file}.tmp`, JSON.stringify(Object.fromEntries(entries)) + "\n", {
+                mode: 0o600,
+                flush: true,
+            });
+            await rename(`${file}.tmp`, file);
+        },
+    };
+}

package/dist/src/daemon/text-output.d.ts

@@ -0,0 +1,18 @@
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import type { WriteStream } from "node:fs";
+export declare function textOutput(child: ChildProcessWithoutNullStreams, options: {
+    readonly output: WriteStream;
+    readonly limit: number;
+}): (exitCode: number | null) => Promise<{
+    text: string;
+    failed: boolean;
+    model: null;
+    costUsd: null;
+    usage: import("../worker-usage.ts").WorkerUsage;
+    sessionId: null;
+    turns: null;
+    reviewFeedback?: string;
+    reviewVerdict: import("../review-subject.ts").ReviewVerdict | null;
+    ticketStatus: import("../ticket-status.ts").TicketStatus;
+    ticketStatusTooLarge?: true;
+}>;