cloving 1.3.0 → 1.5.0

package/dist/src/daemon/retrying-turn.js

@@ -1,41 +1,17 @@
-import { join } from "node:path";
-import { mkdir } from "node:fs/promises";
 import { setTimeout as delay } from "node:timers/promises";
-import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
-import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
-import { capacityRefusal, retryKind, startTurn, } from "./claude-turn.js";
+import { startTurn, } from "./claude-turn.js";
 import { turnHead, retryEvidence, indexFailure } from "./retry-worktree.js";
 import { readRecord, writeRecord } from "./operation-journal.js";
-import { nativeHarness } from "../harness-defaults.js";
+import { resumeWorktree } from "./resume-worktree.js";
+import { readResume, resumedCommand, resumePrompt } from "./claude-recovery.js";
+import { attemptCommand, combined, kindOf, } from "./turn-attempts.js";
+import { prepareAttempt, attemptJournal, recordedEnd, } from "./turn-attempt-journal.js";
 const waits = { capacity: [60_000, 180_000], network: [600_000, 2_400_000] };
 const networkContinuation = "An earlier attempt stopped because the network was not available. Continue from the current state of the work tree. Check each earlier commit, push, review comment and message. Do not repeat a write that is done. Stop and report a write whose result is not known.";
 const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
 function written(folder, entry) {
     return writeRecord(folder, { ...entry, permissions: "private" });
 }
-function total(values) {
-    if (values.includes(null))
-        return null;
-    return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
-}
-function combined(attempts) {
-    const { ends } = attempts;
-    const last = ends.at(-1) ?? { exitCode: null, result: undefined };
-    const result = last.result ?? ends.find((end) => end.result)?.result;
-    if (result === undefined)
-        return last;
-    return {
-        ...last,
-        result: {
-            ...result,
-            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
-            ...(last.result === undefined ? { failed: true, text: "" } : {}),
-            usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
-            costUsd: total(ends.map((end) => end.result?.costUsd ?? null)),
-            turns: total(ends.map((end) => end.result?.turns ?? null)),
-        },
-    };
-}
 function controlled(attempts, start) {
     const turn = startTurn(start);
     const stop = () => {
@@ -49,66 +25,6 @@ function controlled(attempts, start) {
         }),
     };
 }
-function attemptCommand(attempts, attempt) {
-    const { start, options } = attempts;
-    const original = start.command;
-    const previous = attempts.ends[attempt - 2];
-    const fallback = previous !== undefined && kindOf(attempts, previous) === "capacity"
-        ? options.fallbackModels?.[attempt - 3]
-        : undefined;
-    const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
-    const model = start.output === "text" ? null : commandModel(command);
-    return {
-        command,
-        model,
-        ...(fallback !== undefined && model === null
-            ? { fallbackReason: "fallback-model-unavailable" }
-            : {}),
-    };
-}
-function attemptUsage(attempts, completion) {
-    const { end, attempt } = completion;
-    const result = end.result ?? {
-        model: null,
-        usage: usageOf(null),
-        costUsd: null,
-        pricingModel: undefined,
-    };
-    const model = result.model ?? attemptCommand(attempts, attempt).model;
-    const usage = result.usage;
-    return {
-        attempt,
-        retryKind: kindOf(attempts, end),
-        model,
-        ...(end.result?.models === undefined ? {} : { models: end.result.models }),
-        usage,
-        costUsd: result.costUsd,
-        estimatedCostUsd: attemptEstimate(attempts, { end, model, usage }),
-    };
-}
-function attemptEstimate(attempts, result) {
-    const worker = attempts.start.worker ?? "claude";
-    if (!nativeHarness(worker))
-        return null;
-    const models = result.end.result?.models;
-    const prices = attempts.options.tokenPrices ?? {};
-    return estimatedCostUsd(result.usage, worker === "gemini" && models !== undefined
-        ? { worker, models, prices }
-        : {
-            worker,
-            price: priceForModel(prices, result.end.result?.pricingModel === undefined
-                ? result.model
-                : result.end.result.pricingModel),
-        });
-}
-function kindOf(attempts, end) {
-    const { worker, output } = attempts.start;
-    if (output === "text" || !["claude", "codex"].includes(worker ?? "claude"))
-        return null;
-    if (worker !== "codex")
-        return retryKind(end.result);
-    return capacityRefusal(end.result) ? "capacity" : null;
-}
 async function promptOf(attempts) {
     const { start, ends, stopping, turnStartCommit } = attempts;
     if (ends.length === 0)
@@ -118,53 +34,42 @@ async function promptOf(attempts) {
     const evidence = await retryEvidence({ directory: start.directory, signal: stopping.signal }, turnStartCommit);
     return `${networkContinuation}\n\nCommits since the start of this turn:\n${evidence}\n\n${start.prompt}`;
 }
+function launchCommand(attempts, prepared) {
+    const { entry, selected } = prepared;
+    if (entry.resumedAfter === undefined)
+        return selected.command;
+    return (resumedCommand({
+        worker: attempts.start.worker,
+        command: selected.command,
+        sessionId: attempts.options.resume?.sessionId ?? "",
+    }) ?? selected.command);
+}
 async function launched(attempts, prompt = attempts.start.prompt) {
-    const { start, options, ends } = attempts;
-    const attempt = ends.length + 1;
-    const selected = attemptCommand(attempts, attempt);
-    const folder = attempt === 1
-        ? start.folder
-        : join(start.folder, `attempt-${String(attempt)}`);
-    const entry = {
-        ...options.metadata,
-        turnStartCommit: attempts.turnStartCommit,
-        model: selected.model,
-        ...(selected.fallbackReason === undefined
-            ? {}
-            : { fallbackReason: selected.fallbackReason }),
-        attempt,
-        startedAt: attempt === 1 ? attempts.startedAt : new Date().toISOString(),
-    };
-    await written(start.folder, {
-        name: "started.json",
-        value: {
-            ...entry,
-            startedAt: attempts.startedAt,
-            phase: "starting",
-            processId: null,
-        },
-    });
-    await mkdir(folder, { recursive: true, mode: 0o700 });
-    if (attempts.stopping.signal.aborted)
+    const prepared = await prepareAttempt(attempts);
+    if (prepared === undefined || attempts.stopping.signal.aborted)
         return undefined;
+    const { start } = attempts;
+    const { entry, folder } = prepared;
+    const journal = attemptJournal(attempts, entry);
     const turn = controlled(attempts, {
         ...start,
-        command: selected.command,
+        command: launchCommand(attempts, prepared),
+        progress: journal.progress,
         folder,
-        ...(attempt === 1 ? {} : { activityFolder: start.folder }),
-        prompt,
+        ...(entry.attempt === 1 ? {} : { activityFolder: start.folder }),
+        prompt: entry.resumedAfter === undefined ? prompt : resumePrompt,
     });
     try {
-        const value = { ...entry, phase: "running", processId: turn.processId };
-        await written(start.folder, {
-            name: "started.json",
-            value: { ...value, startedAt: attempts.startedAt },
-        });
-        await written(start.folder, {
-            name: `attempt-${String(attempt)}.json`,
-            value,
-        });
-        return turn;
+        await journal.running(turn.processId);
+        return {
+            ...turn,
+            ended: turn.ended.then((end) => ({
+                ...end,
+                ...(entry.resumedAfter === undefined
+                    ? {}
+                    : { resumedAfter: entry.resumedAfter }),
+            })),
+        };
     }
     catch (error) {
         turn.stop(0);
@@ -172,34 +77,6 @@ async function launched(attempts, prompt = attempts.start.prompt) {
         throw error;
     }
 }
-async function recordedEnd(attempts, completion) {
-    const { start, ends, options } = attempts;
-    const { processId, end } = completion;
-    const attempt = ends.length;
-    const selected = attemptCommand(attempts, attempt);
-    const name = `attempt-${String(attempt)}.json`;
-    const started = await readRecord(start.folder, name);
-    const startedAt = Reflect.get(Object(started), "startedAt");
-    if (typeof startedAt !== "string")
-        throw new Error("attempt-start-time-missing");
-    await written(start.folder, {
-        name,
-        value: {
-            ...options.metadata,
-            turnStartCommit: attempts.turnStartCommit,
-            ...attemptUsage(attempts, { end, attempt }),
-            ...(selected.fallbackReason === undefined
-                ? {}
-                : { fallbackReason: selected.fallbackReason }),
-            attempt,
-            processId,
-            startedAt,
-            endedAt: new Date().toISOString(),
-            exitCode: end.exitCode,
-            capacity: capacityRefusal(end.result),
-        },
-    });
-}
 async function waited(attempts, milliseconds) {
     const { start, options, stopping, ends } = attempts;
     await written(start.folder, {
@@ -250,8 +127,9 @@ async function finished(attempts, first) {
         const end = await turn.ended;
         ends.push(end);
         await recordedEnd(attempts, { processId: turn.processId, end });
-        const kind = kindOf(attempts, end);
-        const milliseconds = kind === null ? undefined : waits[kind][ends.length - 1];
+        if (interruptedChild(attempts, { processId: turn.processId, end }))
+            return resumedChild(attempts);
+        const milliseconds = retryWait(attempts, end);
         if (cancelled(attempts) || milliseconds === undefined)
             break;
         await waited(attempts, milliseconds);
@@ -262,9 +140,59 @@ async function finished(attempts, first) {
             return prepared;
         turn = await launched(attempts, prepared);
     }
+    return attemptOutcome(attempts);
+}
+function interruptedChild(attempts, completion) {
+    const { start } = attempts;
+    const { processId, end } = completion;
+    return (!cancelled(attempts) &&
+        (start.worker ?? "claude") === "claude" &&
+        start.output !== "text" &&
+        processId !== null &&
+        end.exitCode !== 0 &&
+        end.result === undefined);
+}
+async function resumedChild(attempts) {
+    const locked = await resumeWorktree({
+        directory: attempts.start.directory,
+        folder: attempts.start.folder,
+        signal: attempts.stopping.signal,
+        stopMilliseconds: attempts.grace.milliseconds,
+    });
+    if (locked !== null)
+        return { ...combined(attempts), reason: locked.reason };
+    const started = await readRecord(attempts.start.folder, "started.json");
+    const resume = await readResume(attempts.start.folder, started);
+    if (resume === undefined)
+        return { ...combined(attempts), reason: "claude-turn-interrupted" };
+    const prepared = await preparedRetry(attempts);
+    if (typeof prepared !== "string")
+        return prepared;
+    const recovered = {
+        ...attempts,
+        options: { ...attempts.options, resume },
+        ends: [...resume.ends],
+    };
+    const turn = await launched(recovered);
+    return turn === undefined
+        ? attemptOutcome(recovered)
+        : finished(recovered, turn);
+}
+function retryWait(attempts, end) {
+    const kind = kindOf(attempts, end);
+    if (kind === null)
+        return undefined;
+    if (attemptCommand(attempts, attempts.ends.length + 1).budgetUsd === 0) {
+        attempts.failure = "claude-turn-budget-exhausted";
+        return undefined;
+    }
+    return waits[kind][attempts.ends.filter((end) => end.resumedAfter === undefined).length - 1];
+}
+function attemptOutcome(attempts) {
     return {
         ...combined(attempts),
-        ...(cancelled(attempts) || turn === undefined ? { stopped: true } : {}),
+        ...(attempts.failure === undefined ? {} : { reason: attempts.failure }),
+        ...(cancelled(attempts) ? { stopped: true } : {}),
     };
 }
 function cancelled(attempts) {
@@ -272,28 +200,26 @@ function cancelled(attempts) {
 }
 export async function startRetriedTurn(start, options) {
     const stopping = options.stopping ?? new AbortController();
-    const turnStartCommit = await turnHead({
-        directory: start.directory,
-        signal: stopping.signal,
-    });
+    const turnStartCommit = options.resume === undefined
+        ? await turnHead({
+            directory: start.directory,
+            signal: stopping.signal,
+        })
+        : options.resume.turnStartCommit;
     const attempts = {
         turnStartCommit,
         start,
-        startedAt: new Date().toISOString(),
+        startedAt: options.resume?.startedAt ?? new Date().toISOString(),
         options,
         stopping,
-        ends: [],
-        grace: { milliseconds: options.stopMilliseconds ?? 2_000 },
+        ends: [...(options.resume?.ends ?? [])],
+        grace: { milliseconds: options.stopMilliseconds ?? 15_000 },
     };
     const turn = await launched(attempts);
     if (turn === undefined)
         return {
             processId: null,
-            ended: Promise.resolve({
-                exitCode: null,
-                result: undefined,
-                stopped: true,
-            }),
+            ended: Promise.resolve(attemptOutcome(attempts)),
             stop: () => {
                 stopping.abort();
             },

package/dist/src/daemon/schedule-control.d.ts

@@ -0,0 +1,26 @@
+import type { SchedulePort, ScheduleProgress } from "../schedule-progress.ts";
+import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { scheduleJournal } from "./schedule-journal.ts";
+type Journal = Awaited<ReturnType<typeof scheduleJournal>>;
+interface ScheduleState {
+    readonly journal: Journal;
+    readonly declarations: readonly DeclaredWorkflow[];
+    readonly clock: () => number;
+    readonly recoverPending: (entry: {
+        readonly workflowId: string;
+        readonly progress: ScheduleProgress;
+    }) => Promise<ScheduleProgress>;
+}
+export declare function scheduleConnection(): {
+    readonly request: SchedulePort;
+    readonly connect: (port: SchedulePort) => void;
+};
+export declare function scheduleControl(settings: ScheduleState & {
+    readonly signal: AbortSignal;
+    readonly tick: () => Promise<void>;
+}): {
+    readonly request: SchedulePort;
+    readonly tick: () => Promise<void>;
+    readonly drain: () => Promise<void>;
+};
+export {};

package/dist/src/daemon/schedule-control.js

@@ -0,0 +1,110 @@
+import { scheduleStops } from "../schedule-progress.js";
+import { runIdentity } from "../service/submission-handler.js";
+import { failed, invalidRequest, result } from "../service/replies.js";
+function summary(workflowId, progress) {
+    return {
+        workflowId,
+        everySeconds: progress.everySeconds,
+        state: progress.state,
+        failuresInRow: progress.failuresInRow,
+        failedRunIds: progress.failedRunIds,
+        lastSlot: progress.lastSlot,
+        nextAt: progress.state === "active" ? progress.nextAt : null,
+        activeRunId: progress.activeCommandId === undefined
+            ? null
+            : runIdentity(progress.activeCommandId),
+        stoppedAt: progress.stoppedAt ?? null,
+        stoppedSlot: progress.stoppedSlot ?? null,
+        stops: scheduleStops(progress),
+    };
+}
+export function scheduleConnection() {
+    let port;
+    return {
+        request: (parameters) => port?.(parameters) ??
+            Promise.resolve(failed("schedules-unavailable", {})),
+        connect: (connected) => {
+            port = connected;
+        },
+    };
+}
+async function change(settings, command) {
+    const { workflowId, action, now } = command;
+    const { journal } = settings;
+    let prior = journal.entries.get(workflowId);
+    if (prior === undefined)
+        return failed("unknown-schedule", { workflowId });
+    const state = action === "pause" ? "paused" : "active";
+    if (prior.state === state)
+        return result("schedule", { schedule: summary(workflowId, prior) });
+    if (prior.pending !== undefined) {
+        prior = await settings.recoverPending({ workflowId, progress: prior });
+        journal.entries.set(workflowId, prior);
+        await journal.save();
+    }
+    const rest = { ...prior };
+    delete rest.stoppedAt;
+    delete rest.stoppedSlot;
+    const progress = action === "pause"
+        ? { ...prior, state, stops: scheduleStops(prior) }
+        : {
+            ...rest,
+            state,
+            failuresInRow: 0,
+            failedRunIds: [],
+            stops: scheduleStops(prior),
+            nextAt: now + prior.everySeconds * 1000,
+        };
+    journal.entries.set(workflowId, progress);
+    try {
+        await journal.save();
+    }
+    catch (error) {
+        journal.entries.set(workflowId, prior);
+        throw error;
+    }
+    return result("schedule", { schedule: summary(workflowId, progress) });
+}
+function request(settings, parameters) {
+    const { action, workflowId } = parameters;
+    const scheduled = settings.declarations.filter((entry) => entry.schedule !== undefined);
+    if (action === "list")
+        return Promise.resolve(result("schedules", {
+            stops: Array.from(settings.journal.entries, ([workflowId, progress]) => ({
+                workflowId,
+                stops: scheduleStops(progress),
+            })),
+            schedules: scheduled.flatMap((entry) => {
+                const progress = settings.journal.entries.get(entry.workflowId);
+                return progress === undefined
+                    ? []
+                    : [summary(entry.workflowId, progress)];
+            }),
+        }));
+    if ((action !== "pause" && action !== "resume") ||
+        typeof workflowId !== "string")
+        return Promise.resolve(invalidRequest("Use list, pause or resume with a workflow ID."));
+    if (!scheduled.some((entry) => entry.workflowId === workflowId))
+        return Promise.resolve(failed("unknown-schedule", { workflowId }));
+    return change(settings, {
+        workflowId,
+        action,
+        now: settings.clock(),
+    });
+}
+export function scheduleControl(settings) {
+    let running = Promise.resolve();
+    const serialized = (operation) => {
+        const next = running.then(async () => {
+            settings.signal.throwIfAborted();
+            return operation();
+        });
+        running = next.then(() => undefined, () => undefined);
+        return next;
+    };
+    return {
+        request: (parameters) => serialized(() => request(settings, parameters)).catch(() => failed("schedules-unavailable", {})),
+        tick: () => serialized(settings.tick),
+        drain: () => running,
+    };
+}

package/dist/src/daemon/schedule-journal.d.ts

@@ -1,18 +1,5 @@
-import { type Fields } from "../protocol.ts";
-export type PendingSlot = {
-    readonly kind: "run";
-    readonly input: Fields;
-} | {
-    readonly kind: "skip";
-    readonly runId: string;
-};
-export interface ScheduleProgress {
-    readonly everySeconds: number;
-    readonly nextAt: number;
-    readonly lastSlot: number;
-    readonly activeCommandId?: string;
-    readonly pending?: PendingSlot;
-}
+import type { ScheduleProgress } from "../schedule-progress.ts";
+export type { PendingSlot, ScheduleProgress } from "../schedule-progress.ts";
 export declare function scheduleJournal(directory: string): Promise<{
     readonly entries: Map<string, ScheduleProgress>;
     readonly save: () => Promise<void>;

package/dist/src/daemon/schedule-journal.js

@@ -23,10 +23,68 @@ function isProgress(value) {
     const fields = record(value);
     if (fields === undefined)
         return false;
-    const { activeCommandId, pending } = fields;
+    const { activeCommandId } = fields;
     return (timing(fields) &&
-        (activeCommandId === undefined || isCommandId(activeCommandId)) &&
-        (pending === undefined || pendingSlot(pending)));
+        control(fields) &&
+        history(fields) &&
+        pendingProgress(fields) &&
+        (activeCommandId === undefined || isCommandId(activeCommandId)));
+}
+function pendingProgress(fields) {
+    const { pending, state } = fields;
+    if (pending === undefined)
+        return true;
+    return (state === undefined || state === "active") && pendingSlot(pending);
+}
+function stopped(fields) {
+    const { state, stoppedAt, stoppedSlot } = fields;
+    if (stoppedAt === undefined && stoppedSlot === undefined)
+        return state !== "stopped";
+    return (whole(stoppedAt) &&
+        whole(stoppedSlot) &&
+        stoppedSlot <= Number(fields["lastSlot"]));
+}
+function failures(fields) {
+    const { failuresInRow, failedRunIds } = fields;
+    return (whole(failuresInRow) &&
+        failuresInRow <= 3 &&
+        Array.isArray(failedRunIds) &&
+        failedRunIds.length === failuresInRow &&
+        failedRunIds.every(isRecordKey) &&
+        new Set(failedRunIds).size === failuresInRow);
+}
+function history(fields) {
+    const stops = fields["stops"];
+    if (stops === undefined)
+        return true;
+    if (!Array.isArray(stops))
+        return false;
+    let previousSlot = -1;
+    for (const item of stops) {
+        const stop = record(item);
+        if (stop === undefined)
+            return false;
+        const slot = stop["stoppedSlot"];
+        if (!whole(slot) || !whole(stop["stoppedAt"]))
+            return false;
+        if (slot <= previousSlot || slot > Number(fields["lastSlot"]))
+            return false;
+        if (!failures({ ...stop, failuresInRow: 3 }))
+            return false;
+        previousSlot = slot;
+    }
+    return true;
+}
+function control(fields) {
+    const { state, failuresInRow, failedRunIds } = fields;
+    if (state === undefined &&
+        failuresInRow === undefined &&
+        failedRunIds === undefined)
+        return (fields["stoppedAt"] === undefined && fields["stoppedSlot"] === undefined);
+    return (["active", "paused", "stopped"].includes(String(state)) &&
+        failures(fields) &&
+        stopped(fields) &&
+        (state !== "stopped" || failuresInRow === 3));
 }
 export async function scheduleJournal(directory) {
     const file = join(directory, "schedules.json");
@@ -42,7 +100,12 @@ export async function scheduleJournal(directory) {
     for (const [workflowId, value] of Object.entries(parsed)) {
         if (!isProgress(value))
             throw new Error("invalid-schedule-state");
-        entries.set(workflowId, value);
+        entries.set(workflowId, {
+            ...value,
+            state: value.state ?? "active",
+            failuresInRow: value.failuresInRow ?? 0,
+            failedRunIds: value.failedRunIds ?? [],
+        });
     }
     return {
         entries,

package/dist/src/daemon/settings-reload.d.ts

@@ -0,0 +1,26 @@
+import { type HarnessSettings } from "../harness-defaults.ts";
+import { type TurnSettings } from "../project-settings.ts";
+import type { SettingChanges } from "../service/workflow-reload.ts";
+import { type TurnProfiles } from "../turn-profile.ts";
+export type SettingDigests = Readonly<Record<string, string>>;
+export interface ReloadedSettings {
+    readonly turnSettings: TurnSettings;
+    readonly settingDigests: SettingDigests;
+}
+export interface TurnSettingsSource {
+    readonly current: () => TurnSettings;
+    readonly replace: (next: TurnSettings) => void;
+}
+export declare function turnSettingsSource(initial: TurnSettings): TurnSettingsSource;
+export declare function reloadedSettings(output: Readonly<Record<string, unknown>> | undefined): ReloadedSettings | undefined;
+export declare function settingChanges(current: SettingDigests, next: SettingDigests): SettingChanges;
+export declare function keptDigests(current: SettingDigests, change: {
+    readonly digests: SettingDigests;
+    readonly reloadedFields: readonly string[];
+}): SettingDigests;
+export declare function profileRefusal(settings: HarnessSettings & {
+    readonly profiles?: TurnProfiles;
+}, turnSettings: TurnSettings): {
+    readonly code: string;
+    readonly detail: string;
+} | undefined;