cloving 1.3.0 → 1.4.0

package/dist/src/daemon/queue-status.js

@@ -8,7 +8,7 @@ function queueBusy(value) {
         throw new Error("invalid-queue-status");
     return busy;
 }
-export async function activeQueueRuns(settings) {
+export async function queueStatus(settings) {
     let count = 0;
     let after = "";
     do {
@@ -20,13 +20,17 @@ export async function activeQueueRuns(settings) {
             throw new Error(reply.error.code);
         const busy = queueBusy(reply.result["queue"]);
         if (busy !== undefined)
-            return busy;
+            return {
+                busy,
+                holding: record(record(reply.result["queue"])?.["spend"])?.["holding"] ===
+                    true,
+            };
         const runs = reply.result["runs"];
         if (!Array.isArray(runs))
             throw new Error("invalid-run-list");
         count += runs.filter((value) => {
             const run = record(value);
-            return (run?.["workflowId"] === (settings.workflowId ?? "develop-issue") &&
+            return (run?.["workflowId"] === settings.workflowId &&
                 !["completed", "failed", "cancelled"].includes(String(run["state"])));
         }).length;
         const next = reply.result["next"];
@@ -34,5 +38,14 @@ export async function activeQueueRuns(settings) {
             throw new Error("invalid-run-page");
         after = next;
     } while (after !== null);
-    return count;
+    return { busy: count, holding: false };
+}
+export function spendNotice() {
+    let held = false;
+    return ({ holding }) => {
+        if (holding && !held)
+            process.stderr.write("Queue holding for spend limit\n");
+        held = holding;
+        return holding;
+    };
 }

package/dist/src/daemon/resume-index-lock.d.ts

@@ -0,0 +1,13 @@
+interface LockIdentity {
+    readonly path: string;
+    readonly dev: number;
+    readonly ino: number;
+    readonly ctimeMs: number;
+    readonly mtimeMs: number;
+}
+export interface OwnedIndexLock extends LockIdentity {
+    readonly processIds: readonly number[];
+}
+export declare function ownedIndexLock(path: string): Promise<OwnedIndexLock | undefined>;
+export declare function removeStoppedIndexLock(lock: OwnedIndexLock, signal: AbortSignal): Promise<boolean>;
+export {};

package/dist/src/daemon/resume-index-lock.js

@@ -0,0 +1,160 @@
+import { execFile } from "node:child_process";
+import { lstat, readdir, readFile, stat, unlink } from "node:fs/promises";
+import { promisify } from "node:util";
+const execute = promisify(execFile);
+function disappeared(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        (error.code === "ENOENT" || error.code === "ESRCH"));
+}
+async function descriptorUsesLock(path, lock) {
+    try {
+        const opened = await stat(path);
+        if (opened.dev !== lock.dev || opened.ino !== lock.ino)
+            return false;
+        const information = await readFile(path.replace("/fd/", "/fdinfo/"), "utf8");
+        const flags = /^flags:\s+([0-7]+)$/m.exec(information)?.[1];
+        const inode = /^ino:\s+(\d+)$/m.exec(information)?.[1];
+        if (flags === undefined || Number(inode) !== lock.ino)
+            return undefined;
+        return (Number.parseInt(flags, 8) & 3) !== 0;
+    }
+    catch (error) {
+        return disappeared(error) ? false : undefined;
+    }
+}
+async function processUsesLock(directory, lock) {
+    try {
+        for (const descriptor of await readdir(`${directory}/fd`)) {
+            const used = await descriptorUsesLock(`${directory}/fd/${descriptor}`, lock);
+            if (used !== false)
+                return used;
+        }
+        return false;
+    }
+    catch (error) {
+        if (disappeared(error))
+            return false;
+        if (!(error instanceof Error) ||
+            !("code" in error) ||
+            (error.code !== "EACCES" && error.code !== "EPERM"))
+            return undefined;
+        return (await unrelatedProcess(directory, lock)) ? false : undefined;
+    }
+}
+async function unrelatedProcess(directory, lock) {
+    if (lock.uid !== process.geteuid?.() || (lock.mode & 0o022) !== 0)
+        return false;
+    try {
+        const status = await readFile(`${directory}/status`, "utf8");
+        const users = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m.exec(status);
+        return (users !== null &&
+            users
+                .slice(1)
+                .map(Number)
+                .every((user) => Number.isSafeInteger(user) && user !== lock.uid));
+    }
+    catch {
+        return false;
+    }
+}
+async function lockOwners(lock) {
+    if (process.platform === "linux") {
+        const processIds = [];
+        for (const entry of await readdir("/proc")) {
+            if (!/^\d+$/.test(entry))
+                continue;
+            const used = await processUsesLock(`/proc/${entry}`, lock);
+            if (used === undefined)
+                return undefined;
+            if (used)
+                processIds.push(Number(entry));
+        }
+        return processIds;
+    }
+    try {
+        const { stdout, stderr } = await execute("lsof", ["-Fpfa", "--", lock.path], {
+            encoding: "utf8",
+            timeout: 5000,
+            maxBuffer: 8 * 1024 * 1024,
+        });
+        return stderr === "" ? writableOwners(stdout) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function writableOwner(record) {
+    const [header, ...descriptors] = record.trim().split(/^f/m);
+    const processId = Number(header?.trim().slice(1));
+    const access = descriptors.map((descriptor) => descriptor.split("\n")[1]);
+    if (!Number.isSafeInteger(processId) ||
+        processId < 1 ||
+        descriptors.length === 0 ||
+        access.some((mode) => mode !== "ar" && mode !== "aw" && mode !== "au"))
+        return undefined;
+    return access.some((mode) => mode === "aw" || mode === "au")
+        ? processId
+        : null;
+}
+function writableOwners(text) {
+    const owners = text
+        .trim()
+        .split(/(?=^p)/m)
+        .map(writableOwner);
+    return owners.includes(undefined)
+        ? undefined
+        : [
+            ...new Set(owners.filter((owner) => typeof owner === "number")),
+        ];
+}
+export async function ownedIndexLock(path) {
+    try {
+        const identity = await lstat(path);
+        if (!identity.isFile())
+            return undefined;
+        const lock = {
+            path,
+            dev: identity.dev,
+            ino: identity.ino,
+            ctimeMs: identity.ctimeMs,
+            mtimeMs: identity.mtimeMs,
+            uid: identity.uid,
+            mode: identity.mode,
+        };
+        const processIds = await lockOwners(lock);
+        return processIds === undefined || processIds.length === 0
+            ? undefined
+            : { ...lock, processIds };
+    }
+    catch {
+        return undefined;
+    }
+}
+function stopped(processId) {
+    try {
+        process.kill(processId, 0);
+        return false;
+    }
+    catch (error) {
+        return error instanceof Error && "code" in error && error.code === "ESRCH";
+    }
+}
+export async function removeStoppedIndexLock(lock, signal) {
+    try {
+        if (!lock.processIds.every(stopped))
+            return false;
+        const after = await lstat(lock.path);
+        if (signal.aborted ||
+            lock.dev !== after.dev ||
+            lock.ino !== after.ino ||
+            lock.ctimeMs !== after.ctimeMs ||
+            lock.mtimeMs !== after.mtimeMs)
+            return false;
+        await unlink(lock.path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/src/daemon/resume-worktree.d.ts

@@ -0,0 +1,7 @@
+import { type IndexFailure } from "./retry-worktree.ts";
+export declare function resumeWorktree(request: {
+    readonly directory: string;
+    readonly folder: string;
+    readonly signal: AbortSignal;
+    readonly stopMilliseconds?: number;
+}): Promise<IndexFailure | null>;

package/dist/src/daemon/resume-worktree.js

@@ -0,0 +1,29 @@
+import { ownedIndexLock, removeStoppedIndexLock } from "./resume-index-lock.js";
+import { stopTreeProcesses } from "./worktree-processes.js";
+import { indexFailure } from "./retry-worktree.js";
+async function inspectLock(failure) {
+    if (failure?.lockPath === undefined)
+        return undefined;
+    return ownedIndexLock(failure.lockPath);
+}
+export async function resumeWorktree(request) {
+    const aborted = () => request.signal.aborted;
+    const before = await indexFailure(request);
+    const lock = await inspectLock(before);
+    if (aborted())
+        return before;
+    const stopped = await stopTreeProcesses(request);
+    const locked = await indexFailure(request);
+    if (!stopped)
+        return (locked ?? {
+            reason: "worktree-index-locked",
+            fix: "Stop the remaining work tree processes before resuming the turn.",
+        });
+    if (locked?.reason !== "worktree-index-locked" || aborted())
+        return locked;
+    if (lock === undefined ||
+        lock.path !== locked.lockPath ||
+        !(await removeStoppedIndexLock(lock, request.signal)))
+        return locked;
+    return indexFailure(request);
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -1,14 +1,4 @@
-import { type TokenPrices } from "../token-prices.ts";
 import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
-export interface RetrySettings {
-    readonly tokenPrices?: TokenPrices;
-    readonly fallbackModels?: readonly string[];
-    readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
-}
-interface RetryOptions extends RetrySettings {
-    readonly stopping?: AbortController;
-    readonly stopMilliseconds?: number;
-    readonly metadata: Readonly<Record<string, unknown>>;
-}
+import { type RetryOptions } from "./turn-attempts.ts";
+export type { RetrySettings } from "./turn-attempts.ts";
 export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;
-export {};

package/dist/src/daemon/retrying-turn.js

@@ -1,41 +1,17 @@
-import { join } from "node:path";
-import { mkdir } from "node:fs/promises";
 import { setTimeout as delay } from "node:timers/promises";
-import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
-import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
-import { capacityRefusal, retryKind, startTurn, } from "./claude-turn.js";
+import { startTurn, } from "./claude-turn.js";
 import { turnHead, retryEvidence, indexFailure } from "./retry-worktree.js";
 import { readRecord, writeRecord } from "./operation-journal.js";
-import { nativeHarness } from "../harness-defaults.js";
+import { resumeWorktree } from "./resume-worktree.js";
+import { readResume, resumedCommand, resumePrompt } from "./claude-recovery.js";
+import { attemptCommand, combined, kindOf, } from "./turn-attempts.js";
+import { prepareAttempt, attemptJournal, recordedEnd, } from "./turn-attempt-journal.js";
 const waits = { capacity: [60_000, 180_000], network: [600_000, 2_400_000] };
 const networkContinuation = "An earlier attempt stopped because the network was not available. Continue from the current state of the work tree. Check each earlier commit, push, review comment and message. Do not repeat a write that is done. Stop and report a write whose result is not known.";
 const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
 function written(folder, entry) {
     return writeRecord(folder, { ...entry, permissions: "private" });
 }
-function total(values) {
-    if (values.includes(null))
-        return null;
-    return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
-}
-function combined(attempts) {
-    const { ends } = attempts;
-    const last = ends.at(-1) ?? { exitCode: null, result: undefined };
-    const result = last.result ?? ends.find((end) => end.result)?.result;
-    if (result === undefined)
-        return last;
-    return {
-        ...last,
-        result: {
-            ...result,
-            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
-            ...(last.result === undefined ? { failed: true, text: "" } : {}),
-            usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
-            costUsd: total(ends.map((end) => end.result?.costUsd ?? null)),
-            turns: total(ends.map((end) => end.result?.turns ?? null)),
-        },
-    };
-}
 function controlled(attempts, start) {
     const turn = startTurn(start);
     const stop = () => {
@@ -49,66 +25,6 @@ function controlled(attempts, start) {
         }),
     };
 }
-function attemptCommand(attempts, attempt) {
-    const { start, options } = attempts;
-    const original = start.command;
-    const previous = attempts.ends[attempt - 2];
-    const fallback = previous !== undefined && kindOf(attempts, previous) === "capacity"
-        ? options.fallbackModels?.[attempt - 3]
-        : undefined;
-    const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
-    const model = start.output === "text" ? null : commandModel(command);
-    return {
-        command,
-        model,
-        ...(fallback !== undefined && model === null
-            ? { fallbackReason: "fallback-model-unavailable" }
-            : {}),
-    };
-}
-function attemptUsage(attempts, completion) {
-    const { end, attempt } = completion;
-    const result = end.result ?? {
-        model: null,
-        usage: usageOf(null),
-        costUsd: null,
-        pricingModel: undefined,
-    };
-    const model = result.model ?? attemptCommand(attempts, attempt).model;
-    const usage = result.usage;
-    return {
-        attempt,
-        retryKind: kindOf(attempts, end),
-        model,
-        ...(end.result?.models === undefined ? {} : { models: end.result.models }),
-        usage,
-        costUsd: result.costUsd,
-        estimatedCostUsd: attemptEstimate(attempts, { end, model, usage }),
-    };
-}
-function attemptEstimate(attempts, result) {
-    const worker = attempts.start.worker ?? "claude";
-    if (!nativeHarness(worker))
-        return null;
-    const models = result.end.result?.models;
-    const prices = attempts.options.tokenPrices ?? {};
-    return estimatedCostUsd(result.usage, worker === "gemini" && models !== undefined
-        ? { worker, models, prices }
-        : {
-            worker,
-            price: priceForModel(prices, result.end.result?.pricingModel === undefined
-                ? result.model
-                : result.end.result.pricingModel),
-        });
-}
-function kindOf(attempts, end) {
-    const { worker, output } = attempts.start;
-    if (output === "text" || !["claude", "codex"].includes(worker ?? "claude"))
-        return null;
-    if (worker !== "codex")
-        return retryKind(end.result);
-    return capacityRefusal(end.result) ? "capacity" : null;
-}
 async function promptOf(attempts) {
     const { start, ends, stopping, turnStartCommit } = attempts;
     if (ends.length === 0)
@@ -118,53 +34,42 @@ async function promptOf(attempts) {
     const evidence = await retryEvidence({ directory: start.directory, signal: stopping.signal }, turnStartCommit);
     return `${networkContinuation}\n\nCommits since the start of this turn:\n${evidence}\n\n${start.prompt}`;
 }
+function launchCommand(attempts, prepared) {
+    const { entry, selected } = prepared;
+    if (entry.resumedAfter === undefined)
+        return selected.command;
+    return (resumedCommand({
+        worker: attempts.start.worker,
+        command: selected.command,
+        sessionId: attempts.options.resume?.sessionId ?? "",
+    }) ?? selected.command);
+}
 async function launched(attempts, prompt = attempts.start.prompt) {
-    const { start, options, ends } = attempts;
-    const attempt = ends.length + 1;
-    const selected = attemptCommand(attempts, attempt);
-    const folder = attempt === 1
-        ? start.folder
-        : join(start.folder, `attempt-${String(attempt)}`);
-    const entry = {
-        ...options.metadata,
-        turnStartCommit: attempts.turnStartCommit,
-        model: selected.model,
-        ...(selected.fallbackReason === undefined
-            ? {}
-            : { fallbackReason: selected.fallbackReason }),
-        attempt,
-        startedAt: attempt === 1 ? attempts.startedAt : new Date().toISOString(),
-    };
-    await written(start.folder, {
-        name: "started.json",
-        value: {
-            ...entry,
-            startedAt: attempts.startedAt,
-            phase: "starting",
-            processId: null,
-        },
-    });
-    await mkdir(folder, { recursive: true, mode: 0o700 });
-    if (attempts.stopping.signal.aborted)
+    const prepared = await prepareAttempt(attempts);
+    if (prepared === undefined || attempts.stopping.signal.aborted)
         return undefined;
+    const { start } = attempts;
+    const { entry, folder } = prepared;
+    const journal = attemptJournal(attempts, entry);
     const turn = controlled(attempts, {
         ...start,
-        command: selected.command,
+        command: launchCommand(attempts, prepared),
+        progress: journal.progress,
         folder,
-        ...(attempt === 1 ? {} : { activityFolder: start.folder }),
-        prompt,
+        ...(entry.attempt === 1 ? {} : { activityFolder: start.folder }),
+        prompt: entry.resumedAfter === undefined ? prompt : resumePrompt,
     });
     try {
-        const value = { ...entry, phase: "running", processId: turn.processId };
-        await written(start.folder, {
-            name: "started.json",
-            value: { ...value, startedAt: attempts.startedAt },
-        });
-        await written(start.folder, {
-            name: `attempt-${String(attempt)}.json`,
-            value,
-        });
-        return turn;
+        await journal.running(turn.processId);
+        return {
+            ...turn,
+            ended: turn.ended.then((end) => ({
+                ...end,
+                ...(entry.resumedAfter === undefined
+                    ? {}
+                    : { resumedAfter: entry.resumedAfter }),
+            })),
+        };
     }
     catch (error) {
         turn.stop(0);
@@ -172,34 +77,6 @@ async function launched(attempts, prompt = attempts.start.prompt) {
         throw error;
     }
 }
-async function recordedEnd(attempts, completion) {
-    const { start, ends, options } = attempts;
-    const { processId, end } = completion;
-    const attempt = ends.length;
-    const selected = attemptCommand(attempts, attempt);
-    const name = `attempt-${String(attempt)}.json`;
-    const started = await readRecord(start.folder, name);
-    const startedAt = Reflect.get(Object(started), "startedAt");
-    if (typeof startedAt !== "string")
-        throw new Error("attempt-start-time-missing");
-    await written(start.folder, {
-        name,
-        value: {
-            ...options.metadata,
-            turnStartCommit: attempts.turnStartCommit,
-            ...attemptUsage(attempts, { end, attempt }),
-            ...(selected.fallbackReason === undefined
-                ? {}
-                : { fallbackReason: selected.fallbackReason }),
-            attempt,
-            processId,
-            startedAt,
-            endedAt: new Date().toISOString(),
-            exitCode: end.exitCode,
-            capacity: capacityRefusal(end.result),
-        },
-    });
-}
 async function waited(attempts, milliseconds) {
     const { start, options, stopping, ends } = attempts;
     await written(start.folder, {
@@ -250,8 +127,9 @@ async function finished(attempts, first) {
         const end = await turn.ended;
         ends.push(end);
         await recordedEnd(attempts, { processId: turn.processId, end });
-        const kind = kindOf(attempts, end);
-        const milliseconds = kind === null ? undefined : waits[kind][ends.length - 1];
+        if (interruptedChild(attempts, { processId: turn.processId, end }))
+            return resumedChild(attempts);
+        const milliseconds = retryWait(attempts, end);
         if (cancelled(attempts) || milliseconds === undefined)
             break;
         await waited(attempts, milliseconds);
@@ -262,9 +140,59 @@ async function finished(attempts, first) {
             return prepared;
         turn = await launched(attempts, prepared);
     }
+    return attemptOutcome(attempts);
+}
+function interruptedChild(attempts, completion) {
+    const { start } = attempts;
+    const { processId, end } = completion;
+    return (!cancelled(attempts) &&
+        (start.worker ?? "claude") === "claude" &&
+        start.output !== "text" &&
+        processId !== null &&
+        end.exitCode !== 0 &&
+        end.result === undefined);
+}
+async function resumedChild(attempts) {
+    const started = await readRecord(attempts.start.folder, "started.json");
+    const resume = await readResume(attempts.start.folder, started);
+    if (resume === undefined)
+        return { ...combined(attempts), reason: "claude-turn-interrupted" };
+    const locked = await resumeWorktree({
+        directory: attempts.start.directory,
+        folder: attempts.start.folder,
+        signal: attempts.stopping.signal,
+        stopMilliseconds: attempts.grace.milliseconds,
+    });
+    if (locked !== null)
+        return { ...combined(attempts), reason: locked.reason };
+    const prepared = await preparedRetry(attempts);
+    if (typeof prepared !== "string")
+        return prepared;
+    const recovered = {
+        ...attempts,
+        options: { ...attempts.options, resume },
+        ends: [...resume.ends],
+    };
+    const turn = await launched(recovered);
+    return turn === undefined
+        ? attemptOutcome(recovered)
+        : finished(recovered, turn);
+}
+function retryWait(attempts, end) {
+    const kind = kindOf(attempts, end);
+    if (kind === null)
+        return undefined;
+    if (attemptCommand(attempts, attempts.ends.length + 1).budgetUsd === 0) {
+        attempts.failure = "claude-turn-budget-exhausted";
+        return undefined;
+    }
+    return waits[kind][attempts.ends.filter((end) => end.resumedAfter === undefined).length - 1];
+}
+function attemptOutcome(attempts) {
     return {
         ...combined(attempts),
-        ...(cancelled(attempts) || turn === undefined ? { stopped: true } : {}),
+        ...(attempts.failure === undefined ? {} : { reason: attempts.failure }),
+        ...(cancelled(attempts) ? { stopped: true } : {}),
     };
 }
 function cancelled(attempts) {
@@ -272,28 +200,26 @@ function cancelled(attempts) {
 }
 export async function startRetriedTurn(start, options) {
     const stopping = options.stopping ?? new AbortController();
-    const turnStartCommit = await turnHead({
-        directory: start.directory,
-        signal: stopping.signal,
-    });
+    const turnStartCommit = options.resume === undefined
+        ? await turnHead({
+            directory: start.directory,
+            signal: stopping.signal,
+        })
+        : options.resume.turnStartCommit;
     const attempts = {
         turnStartCommit,
         start,
-        startedAt: new Date().toISOString(),
+        startedAt: options.resume?.startedAt ?? new Date().toISOString(),
         options,
         stopping,
-        ends: [],
-        grace: { milliseconds: options.stopMilliseconds ?? 2_000 },
+        ends: [...(options.resume?.ends ?? [])],
+        grace: { milliseconds: options.stopMilliseconds ?? 15_000 },
     };
     const turn = await launched(attempts);
     if (turn === undefined)
         return {
             processId: null,
-            ended: Promise.resolve({
-                exitCode: null,
-                result: undefined,
-                stopped: true,
-            }),
+            ended: Promise.resolve(attemptOutcome(attempts)),
             stop: () => {
                 stopping.abort();
             },

package/dist/src/daemon/schedule-control.d.ts

@@ -0,0 +1,26 @@
+import type { SchedulePort, ScheduleProgress } from "../schedule-progress.ts";
+import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { scheduleJournal } from "./schedule-journal.ts";
+type Journal = Awaited<ReturnType<typeof scheduleJournal>>;
+interface ScheduleState {
+    readonly journal: Journal;
+    readonly declarations: readonly DeclaredWorkflow[];
+    readonly clock: () => number;
+    readonly recoverPending: (entry: {
+        readonly workflowId: string;
+        readonly progress: ScheduleProgress;
+    }) => Promise<ScheduleProgress>;
+}
+export declare function scheduleConnection(): {
+    readonly request: SchedulePort;
+    readonly connect: (port: SchedulePort) => void;
+};
+export declare function scheduleControl(settings: ScheduleState & {
+    readonly signal: AbortSignal;
+    readonly tick: () => Promise<void>;
+}): {
+    readonly request: SchedulePort;
+    readonly tick: () => Promise<void>;
+    readonly drain: () => Promise<void>;
+};
+export {};