cloving 1.2.0 → 1.3.0

package/dist/src/daemon/retry-worktree.js

@@ -0,0 +1,139 @@
+import { spawn } from "node:child_process";
+import { lstat } from "node:fs/promises";
+import { resolve } from "node:path";
+function kill(child) {
+    if (child.pid === undefined)
+        return;
+    try {
+        process.kill(-child.pid, "SIGKILL");
+    }
+    catch {
+        child.kill("SIGKILL");
+    }
+}
+export async function retryGit(tree, parts) {
+    if (tree.signal.aborted)
+        return { text: "", code: null };
+    const child = spawn("git", ["-C", tree.directory, ...parts], {
+        stdio: ["ignore", "pipe", "ignore"],
+        detached: true,
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+    });
+    const closed = Promise.withResolvers();
+    let text = "";
+    const stop = () => {
+        kill(child);
+        child.stdout.destroy();
+        closed.resolve(null);
+    };
+    const timer = setTimeout(stop, 30_000);
+    tree.signal.addEventListener("abort", stop, { once: true });
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        text += chunk;
+        if (Buffer.byteLength(text) > 1_048_576)
+            stop();
+    });
+    child.once("error", () => {
+        closed.resolve(null);
+    });
+    child.once("close", closed.resolve);
+    const code = await closed.promise;
+    clearTimeout(timer);
+    tree.signal.removeEventListener("abort", stop);
+    return { text: text.trim(), code };
+}
+export async function turnHead(tree) {
+    const head = await retryGit(tree, ["rev-parse", "--verify", "HEAD"]);
+    return head.code === 0 && /^[a-f0-9]{40,64}$/.test(head.text)
+        ? head.text
+        : null;
+}
+async function remoteState(tree) {
+    const branch = await retryGit(tree, ["symbolic-ref", "--short", "HEAD"]);
+    if (branch.code !== 0 || tree.signal.aborted)
+        return "unknown";
+    const reference = `refs/heads/${branch.text}`;
+    const remote = await retryGit(tree, [
+        "ls-remote",
+        "--exit-code",
+        "--heads",
+        "origin",
+        reference,
+    ]);
+    if (remote.code === 2)
+        return "not pushed";
+    if (remote.code !== 0)
+        return "unknown";
+    const tracking = `refs/remotes/origin/${branch.text}`;
+    const fetched = await retryGit(tree, [
+        "fetch",
+        "--no-tags",
+        "origin",
+        `+${reference}:${tracking}`,
+    ]);
+    return fetched.code === 0 ? tracking : "unknown";
+}
+async function pushState(tree, evidence) {
+    const { commit, remote } = evidence;
+    if (!remote.startsWith("refs/"))
+        return remote;
+    const ancestor = await retryGit(tree, [
+        "merge-base",
+        "--is-ancestor",
+        commit,
+        remote,
+    ]);
+    if (ancestor.code === 1)
+        return "not pushed";
+    return ancestor.code === 0 ? "pushed" : "unknown";
+}
+export async function retryEvidence(tree, startCommit) {
+    const remote = await remoteState(tree);
+    if (startCommit === null || tree.signal.aborted)
+        return "unknown: commits since the start of this turn could not be determined.";
+    const commits = await retryGit(tree, [
+        "log",
+        "--format=%H %s",
+        `${startCommit}..HEAD`,
+        "--",
+    ]);
+    if (commits.code !== 0)
+        return "unknown: commits since the start of this turn could not be listed.";
+    const lines = [];
+    for (const line of commits.text.split("\n").filter(Boolean)) {
+        const commit = line.split(" ")[0] ?? "";
+        const state = await pushState(tree, { commit, remote });
+        lines.push(`${commit} ${state}${line.slice(commit.length)}`);
+    }
+    if (lines.length > 0)
+        return lines.join("\n");
+    return remote === "unknown"
+        ? "unknown: remote push state could not be determined; no local commits since the start of this turn."
+        : "No commits since the start of this turn.";
+}
+export async function indexFailure(tree) {
+    const lookup = await retryGit(tree, [
+        "rev-parse",
+        "--git-path",
+        "index.lock",
+    ]);
+    const unknown = {
+        reason: "worktree-index-unknown",
+        fix: "Check that the work tree and its git index path are accessible before starting another turn.",
+    };
+    if (lookup.code !== 0 || lookup.text === "")
+        return unknown;
+    const lockPath = resolve(tree.directory, lookup.text);
+    try {
+        await lstat(lockPath);
+        return {
+            reason: "worktree-index-locked",
+            lockPath,
+            fix: "Remove the index lock when no git process uses the work tree, then start another turn.",
+        };
+    }
+    catch (error) {
+        return Reflect.get(Object(error), "code") === "ENOENT" ? null : unknown;
+    }
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -6,6 +6,8 @@ export interface RetrySettings {
     readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
 }
 interface RetryOptions extends RetrySettings {
+    readonly stopping?: AbortController;
+    readonly stopMilliseconds?: number;
     readonly metadata: Readonly<Record<string, unknown>>;
 }
 export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;

package/dist/src/daemon/retrying-turn.js

@@ -3,10 +3,12 @@ import { mkdir } from "node:fs/promises";
 import { setTimeout as delay } from "node:timers/promises";
 import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
 import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
-import { analysisCommand } from "./analysis-launch.js";
-import { capacityRefusal, startTurn, } from "./claude-turn.js";
+import { capacityRefusal, retryKind, startTurn, } from "./claude-turn.js";
+import { turnHead, retryEvidence, indexFailure } from "./retry-worktree.js";
 import { readRecord, writeRecord } from "./operation-journal.js";
-const waits = [60_000, 180_000];
+import { nativeHarness } from "../harness-defaults.js";
+const waits = { capacity: [60_000, 180_000], network: [600_000, 2_400_000] };
+const networkContinuation = "An earlier attempt stopped because the network was not available. Continue from the current state of the work tree. Check each earlier commit, push, review comment and message. Do not repeat a write that is done. Stop and report a write whose result is not known.";
 const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
 function written(folder, entry) {
     return writeRecord(folder, { ...entry, permissions: "private" });
@@ -49,12 +51,13 @@ function controlled(attempts, start) {
 }
 function attemptCommand(attempts, attempt) {
     const { start, options } = attempts;
-    const original = start.mode === "analysis" ? analysisCommand() : start.command;
-    const fallback = start.mode === "analysis"
-        ? undefined
-        : options.fallbackModels?.[attempt - 3];
+    const original = start.command;
+    const previous = attempts.ends[attempt - 2];
+    const fallback = previous !== undefined && kindOf(attempts, previous) === "capacity"
+        ? options.fallbackModels?.[attempt - 3]
+        : undefined;
     const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
-    const model = commandModel(command);
+    const model = start.output === "text" ? null : commandModel(command);
     return {
         command,
         model,
@@ -65,30 +68,66 @@ function attemptCommand(attempts, attempt) {
 }
 function attemptUsage(attempts, completion) {
     const { end, attempt } = completion;
-    const model = end.result?.model ?? attemptCommand(attempts, attempt).model;
-    const usage = end.result?.usage ?? usageOf(null);
+    const result = end.result ?? {
+        model: null,
+        usage: usageOf(null),
+        costUsd: null,
+        pricingModel: undefined,
+    };
+    const model = result.model ?? attemptCommand(attempts, attempt).model;
+    const usage = result.usage;
     return {
         attempt,
+        retryKind: kindOf(attempts, end),
         model,
+        ...(end.result?.models === undefined ? {} : { models: end.result.models }),
         usage,
-        costUsd: end.result?.costUsd ?? null,
-        estimatedCostUsd: estimatedCostUsd(usage, {
-            worker: attempts.start.mode === "analysis"
-                ? "claude"
-                : (attempts.start.worker ?? "claude"),
-            price: priceForModel(attempts.options.tokenPrices ?? {}, model),
-        }),
+        costUsd: result.costUsd,
+        estimatedCostUsd: attemptEstimate(attempts, { end, model, usage }),
     };
 }
-async function launched(attempts) {
+function attemptEstimate(attempts, result) {
+    const worker = attempts.start.worker ?? "claude";
+    if (!nativeHarness(worker))
+        return null;
+    const models = result.end.result?.models;
+    const prices = attempts.options.tokenPrices ?? {};
+    return estimatedCostUsd(result.usage, worker === "gemini" && models !== undefined
+        ? { worker, models, prices }
+        : {
+            worker,
+            price: priceForModel(prices, result.end.result?.pricingModel === undefined
+                ? result.model
+                : result.end.result.pricingModel),
+        });
+}
+function kindOf(attempts, end) {
+    const { worker, output } = attempts.start;
+    if (output === "text" || !["claude", "codex"].includes(worker ?? "claude"))
+        return null;
+    if (worker !== "codex")
+        return retryKind(end.result);
+    return capacityRefusal(end.result) ? "capacity" : null;
+}
+async function promptOf(attempts) {
+    const { start, ends, stopping, turnStartCommit } = attempts;
+    if (ends.length === 0)
+        return start.prompt;
+    if (!ends.some((end) => kindOf(attempts, end) === "network"))
+        return continuation + start.prompt;
+    const evidence = await retryEvidence({ directory: start.directory, signal: stopping.signal }, turnStartCommit);
+    return `${networkContinuation}\n\nCommits since the start of this turn:\n${evidence}\n\n${start.prompt}`;
+}
+async function launched(attempts, prompt = attempts.start.prompt) {
     const { start, options, ends } = attempts;
     const attempt = ends.length + 1;
     const selected = attemptCommand(attempts, attempt);
-    const folder = start.mode === "analysis" || attempt === 1
+    const folder = attempt === 1
         ? start.folder
         : join(start.folder, `attempt-${String(attempt)}`);
     const entry = {
         ...options.metadata,
+        turnStartCommit: attempts.turnStartCommit,
         model: selected.model,
         ...(selected.fallbackReason === undefined
             ? {}
@@ -113,9 +152,7 @@ async function launched(attempts) {
         command: selected.command,
         folder,
         ...(attempt === 1 ? {} : { activityFolder: start.folder }),
-        prompt: start.mode === "analysis" || attempt === 1
-            ? start.prompt
-            : continuation + start.prompt,
+        prompt,
     });
     try {
         const value = { ...entry, phase: "running", processId: turn.processId };
@@ -149,6 +186,7 @@ async function recordedEnd(attempts, completion) {
         name,
         value: {
             ...options.metadata,
+            turnStartCommit: attempts.turnStartCommit,
             ...attemptUsage(attempts, { end, attempt }),
             ...(selected.fallbackReason === undefined
                 ? {}
@@ -168,6 +206,7 @@ async function waited(attempts, milliseconds) {
         name: "started.json",
         value: {
             ...options.metadata,
+            turnStartCommit: attempts.turnStartCommit,
             startedAt: attempts.startedAt,
             attempt: ends.length,
             phase: "waiting",
@@ -183,6 +222,27 @@ async function waited(attempts, milliseconds) {
             throw error;
     });
 }
+async function preparedRetry(attempts) {
+    const prompt = await promptOf(attempts);
+    const failure = await indexFailure({
+        directory: attempts.start.directory,
+        signal: attempts.stopping.signal,
+    });
+    if (cancelled(attempts))
+        return { ...combined(attempts), stopped: true };
+    if (failure !== null) {
+        await written(attempts.start.folder, {
+            name: "retry-failure.json",
+            value: failure,
+        });
+        return {
+            ...combined(attempts),
+            reason: failure.reason,
+            detail: `${failure.fix}${failure.lockPath === undefined ? "" : ` Lock: ${failure.lockPath}`}`,
+        };
+    }
+    return prompt;
+}
 async function finished(attempts, first) {
     const { ends } = attempts;
     let turn = first;
@@ -190,15 +250,17 @@ async function finished(attempts, first) {
         const end = await turn.ended;
         ends.push(end);
         await recordedEnd(attempts, { processId: turn.processId, end });
-        const milliseconds = waits[ends.length - 1];
-        if (cancelled(attempts) ||
-            !capacityRefusal(end.result) ||
-            milliseconds === undefined)
+        const kind = kindOf(attempts, end);
+        const milliseconds = kind === null ? undefined : waits[kind][ends.length - 1];
+        if (cancelled(attempts) || milliseconds === undefined)
             break;
         await waited(attempts, milliseconds);
         if (cancelled(attempts))
             return { ...combined(attempts), stopped: true };
-        turn = await launched(attempts);
+        const prepared = await preparedRetry(attempts);
+        if (typeof prepared !== "string")
+            return prepared;
+        turn = await launched(attempts, prepared);
     }
     return {
         ...combined(attempts),
@@ -209,17 +271,33 @@ function cancelled(attempts) {
     return attempts.stopping.signal.aborted;
 }
 export async function startRetriedTurn(start, options) {
+    const stopping = options.stopping ?? new AbortController();
+    const turnStartCommit = await turnHead({
+        directory: start.directory,
+        signal: stopping.signal,
+    });
     const attempts = {
+        turnStartCommit,
         start,
         startedAt: new Date().toISOString(),
         options,
-        stopping: new AbortController(),
+        stopping,
         ends: [],
-        grace: { milliseconds: 2_000 },
+        grace: { milliseconds: options.stopMilliseconds ?? 2_000 },
     };
     const turn = await launched(attempts);
     if (turn === undefined)
-        throw new Error("turn-start-stopped");
+        return {
+            processId: null,
+            ended: Promise.resolve({
+                exitCode: null,
+                result: undefined,
+                stopped: true,
+            }),
+            stop: () => {
+                stopping.abort();
+            },
+        };
     return {
         processId: turn.processId,
         ended: finished(attempts, turn),

package/dist/src/daemon/schedule-journal.d.ts

@@ -0,0 +1,19 @@
+import { type Fields } from "../protocol.ts";
+export type PendingSlot = {
+    readonly kind: "run";
+    readonly input: Fields;
+} | {
+    readonly kind: "skip";
+    readonly runId: string;
+};
+export interface ScheduleProgress {
+    readonly everySeconds: number;
+    readonly nextAt: number;
+    readonly lastSlot: number;
+    readonly activeCommandId?: string;
+    readonly pending?: PendingSlot;
+}
+export declare function scheduleJournal(directory: string): Promise<{
+    readonly entries: Map<string, ScheduleProgress>;
+    readonly save: () => Promise<void>;
+}>;

package/dist/src/daemon/schedule-journal.js

@@ -0,0 +1,57 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { isCommandId, isRecordKey } from "../authority/commands.js";
+function pendingSlot(value) {
+    const fields = record(value);
+    if (fields?.["kind"] === "run")
+        return record(fields["input"]) !== undefined;
+    return fields?.["kind"] === "skip" && isRecordKey(fields["runId"]);
+}
+function whole(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function timing(fields) {
+    const { everySeconds, nextAt, lastSlot } = fields;
+    return (whole(everySeconds) &&
+        everySeconds >= 60 &&
+        everySeconds <= 86400 &&
+        whole(nextAt) &&
+        whole(lastSlot));
+}
+function isProgress(value) {
+    const fields = record(value);
+    if (fields === undefined)
+        return false;
+    const { activeCommandId, pending } = fields;
+    return (timing(fields) &&
+        (activeCommandId === undefined || isCommandId(activeCommandId)) &&
+        (pending === undefined || pendingSlot(pending)));
+}
+export async function scheduleJournal(directory) {
+    const file = join(directory, "schedules.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "{}";
+        throw error;
+    });
+    const parsed = record(JSON.parse(text));
+    if (parsed === undefined)
+        throw new Error("invalid-schedule-state");
+    const entries = new Map();
+    for (const [workflowId, value] of Object.entries(parsed)) {
+        if (!isProgress(value))
+            throw new Error("invalid-schedule-state");
+        entries.set(workflowId, value);
+    }
+    return {
+        entries,
+        save: async () => {
+            await writeFile(`${file}.tmp`, JSON.stringify(Object.fromEntries(entries)) + "\n", {
+                mode: 0o600,
+                flush: true,
+            });
+            await rename(`${file}.tmp`, file);
+        },
+    };
+}

package/dist/src/daemon/text-output.d.ts

@@ -0,0 +1,18 @@
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import type { WriteStream } from "node:fs";
+export declare function textOutput(child: ChildProcessWithoutNullStreams, options: {
+    readonly output: WriteStream;
+    readonly limit: number;
+}): (exitCode: number | null) => Promise<{
+    text: string;
+    failed: boolean;
+    model: null;
+    costUsd: null;
+    usage: import("../worker-usage.ts").WorkerUsage;
+    sessionId: null;
+    turns: null;
+    reviewFeedback?: string;
+    reviewVerdict: import("../review-subject.ts").ReviewVerdict | null;
+    ticketStatus: import("../ticket-status.ts").TicketStatus;
+    ticketStatusTooLarge?: true;
+}>;

package/dist/src/daemon/text-output.js

@@ -0,0 +1,44 @@
+import { usageOf } from "../worker-usage.js";
+import { reviewReportStream } from "../review-subject.js";
+import { ticketStatusStream } from "../ticket-status-stream.js";
+export function textOutput(child, options) {
+    const { output, limit } = options;
+    const status = ticketStatusStream(limit);
+    const verdict = reviewReportStream(limit);
+    let text = "";
+    let error = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+        error = (error + chunk).slice(-limit);
+    });
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        text = (text + chunk).slice(-limit);
+        status.write(chunk);
+        verdict.write(chunk);
+        if (!output.write(chunk)) {
+            child.stdout.pause();
+            output.once("drain", () => {
+                child.stdout.resume();
+            });
+        }
+    });
+    return async (exitCode) => {
+        await new Promise((resolve) => output.end(resolve));
+        const review = verdict.result();
+        return {
+            ...status.result(),
+            reviewVerdict: review.reviewVerdict,
+            ...(review.reviewVerdict === null
+                ? {}
+                : { reviewFeedback: review.feedback }),
+            text: exitCode === 0 ? text : error,
+            failed: exitCode !== 0,
+            model: null,
+            costUsd: null,
+            usage: usageOf(null),
+            sessionId: null,
+            turns: null,
+        };
+    };
+}

package/dist/src/daemon/ticket-registration.d.ts

@@ -1,5 +1,6 @@
-import type { DeclaredWorkflow, WorkflowStage } from "../workflow-declaration.ts";
+import { type DeclaredWorkflow } from "../workflow-declaration.ts";
 import type { Resolver, WorkflowRegistration } from "../service/registry.ts";
+import type { LocalCommands } from "../command-settings.ts";
 export declare const ticketWorkflow: "develop-ticket";
 export declare const promptWorkflow: "develop-prompt";
 export declare const issueWorkflow: "develop-issue";
@@ -18,15 +19,10 @@ export interface TicketSource {
     readonly githubCommand?: readonly string[] | undefined;
     readonly skipTickets?: readonly string[];
 }
-export interface TicketRegistration {
-    readonly workflowId: string;
+export interface TicketRegistration extends Omit<WorkflowRegistration, "manifest" | "availability" | "admission"> {
     readonly functionName: string;
-    readonly resolver: Resolver;
-    readonly submission: WorkflowRegistration["submission"];
     readonly effectLimit: number;
     readonly capabilities?: readonly string[];
-    readonly stages?: readonly WorkflowStage[];
-    readonly handoffs?: readonly string[];
 }
 export declare function ticketResolver(source: TicketSource): Resolver;
 export declare function issueResolver(source: TicketSource): Resolver;
@@ -34,4 +30,6 @@ export declare function promptResolver(repository: string): Resolver;
 export declare function ticketRegistrationTable(source: TicketSource, declarations?: readonly DeclaredWorkflow[]): readonly TicketRegistration[];
 export declare function ticketRegistrations(manifests: Readonly<Record<string, string | undefined>>, options: {
     readonly table: readonly TicketRegistration[];
+    readonly bundleDigest?: string;
+    readonly commands?: LocalCommands;
 }): Promise<TicketRegistrations>;