cloving 1.1.0 → 1.3.0

package/dist/src/daemon/retrying-turn.js

@@ -1,10 +1,14 @@
 import { join } from "node:path";
 import { mkdir } from "node:fs/promises";
 import { setTimeout as delay } from "node:timers/promises";
-import { priceOf, sumUsage, usageOf } from "../worker-usage.js";
-import { capacityRefusal, startTurn, } from "./claude-turn.js";
+import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
+import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
+import { capacityRefusal, retryKind, startTurn, } from "./claude-turn.js";
+import { turnHead, retryEvidence, indexFailure } from "./retry-worktree.js";
 import { readRecord, writeRecord } from "./operation-journal.js";
-const waits = [60_000, 180_000];
+import { nativeHarness } from "../harness-defaults.js";
+const waits = { capacity: [60_000, 180_000], network: [600_000, 2_400_000] };
+const networkContinuation = "An earlier attempt stopped because the network was not available. Continue from the current state of the work tree. Check each earlier commit, push, review comment and message. Do not repeat a write that is done. Stop and report a write whose result is not known.";
 const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
 function written(folder, entry) {
     return writeRecord(folder, { ...entry, permissions: "private" });
@@ -14,10 +18,9 @@ function total(values) {
         return null;
     return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
 }
-function combined(ends) {
+function combined(attempts) {
+    const { ends } = attempts;
     const last = ends.at(-1) ?? { exitCode: null, result: undefined };
-    if (ends.length < 2)
-        return last;
     const result = last.result ?? ends.find((end) => end.result)?.result;
     if (result === undefined)
         return last;
@@ -25,6 +28,7 @@ function combined(ends) {
         ...last,
         result: {
             ...result,
+            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
             ...(last.result === undefined ? { failed: true, text: "" } : {}),
             usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
             costUsd: total(ends.map((end) => end.result?.costUsd ?? null)),
@@ -45,14 +49,89 @@ function controlled(attempts, start) {
         }),
     };
 }
-async function launched(attempts) {
+function attemptCommand(attempts, attempt) {
+    const { start, options } = attempts;
+    const original = start.command;
+    const previous = attempts.ends[attempt - 2];
+    const fallback = previous !== undefined && kindOf(attempts, previous) === "capacity"
+        ? options.fallbackModels?.[attempt - 3]
+        : undefined;
+    const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
+    const model = start.output === "text" ? null : commandModel(command);
+    return {
+        command,
+        model,
+        ...(fallback !== undefined && model === null
+            ? { fallbackReason: "fallback-model-unavailable" }
+            : {}),
+    };
+}
+function attemptUsage(attempts, completion) {
+    const { end, attempt } = completion;
+    const result = end.result ?? {
+        model: null,
+        usage: usageOf(null),
+        costUsd: null,
+        pricingModel: undefined,
+    };
+    const model = result.model ?? attemptCommand(attempts, attempt).model;
+    const usage = result.usage;
+    return {
+        attempt,
+        retryKind: kindOf(attempts, end),
+        model,
+        ...(end.result?.models === undefined ? {} : { models: end.result.models }),
+        usage,
+        costUsd: result.costUsd,
+        estimatedCostUsd: attemptEstimate(attempts, { end, model, usage }),
+    };
+}
+function attemptEstimate(attempts, result) {
+    const worker = attempts.start.worker ?? "claude";
+    if (!nativeHarness(worker))
+        return null;
+    const models = result.end.result?.models;
+    const prices = attempts.options.tokenPrices ?? {};
+    return estimatedCostUsd(result.usage, worker === "gemini" && models !== undefined
+        ? { worker, models, prices }
+        : {
+            worker,
+            price: priceForModel(prices, result.end.result?.pricingModel === undefined
+                ? result.model
+                : result.end.result.pricingModel),
+        });
+}
+function kindOf(attempts, end) {
+    const { worker, output } = attempts.start;
+    if (output === "text" || !["claude", "codex"].includes(worker ?? "claude"))
+        return null;
+    if (worker !== "codex")
+        return retryKind(end.result);
+    return capacityRefusal(end.result) ? "capacity" : null;
+}
+async function promptOf(attempts) {
+    const { start, ends, stopping, turnStartCommit } = attempts;
+    if (ends.length === 0)
+        return start.prompt;
+    if (!ends.some((end) => kindOf(attempts, end) === "network"))
+        return continuation + start.prompt;
+    const evidence = await retryEvidence({ directory: start.directory, signal: stopping.signal }, turnStartCommit);
+    return `${networkContinuation}\n\nCommits since the start of this turn:\n${evidence}\n\n${start.prompt}`;
+}
+async function launched(attempts, prompt = attempts.start.prompt) {
     const { start, options, ends } = attempts;
     const attempt = ends.length + 1;
-    const folder = start.mode === "analysis" || attempt === 1
+    const selected = attemptCommand(attempts, attempt);
+    const folder = attempt === 1
         ? start.folder
         : join(start.folder, `attempt-${String(attempt)}`);
     const entry = {
         ...options.metadata,
+        turnStartCommit: attempts.turnStartCommit,
+        model: selected.model,
+        ...(selected.fallbackReason === undefined
+            ? {}
+            : { fallbackReason: selected.fallbackReason }),
         attempt,
         startedAt: attempt === 1 ? attempts.startedAt : new Date().toISOString(),
     };
@@ -70,9 +149,10 @@ async function launched(attempts) {
         return undefined;
     const turn = controlled(attempts, {
         ...start,
+        command: selected.command,
         folder,
         ...(attempt === 1 ? {} : { activityFolder: start.folder }),
-        prompt: attempt === 1 ? start.prompt : continuation + start.prompt,
+        prompt,
     });
     try {
         const value = { ...entry, phase: "running", processId: turn.processId };
@@ -96,6 +176,7 @@ async function recordedEnd(attempts, completion) {
     const { start, ends, options } = attempts;
     const { processId, end } = completion;
     const attempt = ends.length;
+    const selected = attemptCommand(attempts, attempt);
     const name = `attempt-${String(attempt)}.json`;
     const started = await readRecord(start.folder, name);
     const startedAt = Reflect.get(Object(started), "startedAt");
@@ -105,14 +186,17 @@ async function recordedEnd(attempts, completion) {
         name,
         value: {
             ...options.metadata,
+            turnStartCommit: attempts.turnStartCommit,
+            ...attemptUsage(attempts, { end, attempt }),
+            ...(selected.fallbackReason === undefined
+                ? {}
+                : { fallbackReason: selected.fallbackReason }),
             attempt,
             processId,
             startedAt,
             endedAt: new Date().toISOString(),
             exitCode: end.exitCode,
             capacity: capacityRefusal(end.result),
-            usage: end.result?.usage ?? usageOf(null),
-            costUsd: end.result?.costUsd ?? null,
         },
     });
 }
@@ -122,6 +206,7 @@ async function waited(attempts, milliseconds) {
         name: "started.json",
         value: {
             ...options.metadata,
+            turnStartCommit: attempts.turnStartCommit,
             startedAt: attempts.startedAt,
             attempt: ends.length,
             phase: "waiting",
@@ -137,6 +222,27 @@ async function waited(attempts, milliseconds) {
             throw error;
     });
 }
+async function preparedRetry(attempts) {
+    const prompt = await promptOf(attempts);
+    const failure = await indexFailure({
+        directory: attempts.start.directory,
+        signal: attempts.stopping.signal,
+    });
+    if (cancelled(attempts))
+        return { ...combined(attempts), stopped: true };
+    if (failure !== null) {
+        await written(attempts.start.folder, {
+            name: "retry-failure.json",
+            value: failure,
+        });
+        return {
+            ...combined(attempts),
+            reason: failure.reason,
+            detail: `${failure.fix}${failure.lockPath === undefined ? "" : ` Lock: ${failure.lockPath}`}`,
+        };
+    }
+    return prompt;
+}
 async function finished(attempts, first) {
     const { ends } = attempts;
     let turn = first;
@@ -144,18 +250,20 @@ async function finished(attempts, first) {
         const end = await turn.ended;
         ends.push(end);
         await recordedEnd(attempts, { processId: turn.processId, end });
-        const milliseconds = waits[ends.length - 1];
-        if (cancelled(attempts) ||
-            !capacityRefusal(end.result) ||
-            milliseconds === undefined)
+        const kind = kindOf(attempts, end);
+        const milliseconds = kind === null ? undefined : waits[kind][ends.length - 1];
+        if (cancelled(attempts) || milliseconds === undefined)
             break;
         await waited(attempts, milliseconds);
         if (cancelled(attempts))
-            return { ...combined(ends), stopped: true };
-        turn = await launched(attempts);
+            return { ...combined(attempts), stopped: true };
+        const prepared = await preparedRetry(attempts);
+        if (typeof prepared !== "string")
+            return prepared;
+        turn = await launched(attempts, prepared);
     }
     return {
-        ...combined(ends),
+        ...combined(attempts),
         ...(cancelled(attempts) || turn === undefined ? { stopped: true } : {}),
     };
 }
@@ -163,17 +271,33 @@ function cancelled(attempts) {
     return attempts.stopping.signal.aborted;
 }
 export async function startRetriedTurn(start, options) {
+    const stopping = options.stopping ?? new AbortController();
+    const turnStartCommit = await turnHead({
+        directory: start.directory,
+        signal: stopping.signal,
+    });
     const attempts = {
+        turnStartCommit,
         start,
         startedAt: new Date().toISOString(),
         options,
-        stopping: new AbortController(),
+        stopping,
         ends: [],
-        grace: { milliseconds: 2_000 },
+        grace: { milliseconds: options.stopMilliseconds ?? 2_000 },
     };
     const turn = await launched(attempts);
     if (turn === undefined)
-        throw new Error("turn-start-stopped");
+        return {
+            processId: null,
+            ended: Promise.resolve({
+                exitCode: null,
+                result: undefined,
+                stopped: true,
+            }),
+            stop: () => {
+                stopping.abort();
+            },
+        };
     return {
         processId: turn.processId,
         ended: finished(attempts, turn),

package/dist/src/daemon/schedule-journal.d.ts

@@ -0,0 +1,19 @@
+import { type Fields } from "../protocol.ts";
+export type PendingSlot = {
+    readonly kind: "run";
+    readonly input: Fields;
+} | {
+    readonly kind: "skip";
+    readonly runId: string;
+};
+export interface ScheduleProgress {
+    readonly everySeconds: number;
+    readonly nextAt: number;
+    readonly lastSlot: number;
+    readonly activeCommandId?: string;
+    readonly pending?: PendingSlot;
+}
+export declare function scheduleJournal(directory: string): Promise<{
+    readonly entries: Map<string, ScheduleProgress>;
+    readonly save: () => Promise<void>;
+}>;

package/dist/src/daemon/schedule-journal.js

@@ -0,0 +1,57 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { isCommandId, isRecordKey } from "../authority/commands.js";
+function pendingSlot(value) {
+    const fields = record(value);
+    if (fields?.["kind"] === "run")
+        return record(fields["input"]) !== undefined;
+    return fields?.["kind"] === "skip" && isRecordKey(fields["runId"]);
+}
+function whole(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function timing(fields) {
+    const { everySeconds, nextAt, lastSlot } = fields;
+    return (whole(everySeconds) &&
+        everySeconds >= 60 &&
+        everySeconds <= 86400 &&
+        whole(nextAt) &&
+        whole(lastSlot));
+}
+function isProgress(value) {
+    const fields = record(value);
+    if (fields === undefined)
+        return false;
+    const { activeCommandId, pending } = fields;
+    return (timing(fields) &&
+        (activeCommandId === undefined || isCommandId(activeCommandId)) &&
+        (pending === undefined || pendingSlot(pending)));
+}
+export async function scheduleJournal(directory) {
+    const file = join(directory, "schedules.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "{}";
+        throw error;
+    });
+    const parsed = record(JSON.parse(text));
+    if (parsed === undefined)
+        throw new Error("invalid-schedule-state");
+    const entries = new Map();
+    for (const [workflowId, value] of Object.entries(parsed)) {
+        if (!isProgress(value))
+            throw new Error("invalid-schedule-state");
+        entries.set(workflowId, value);
+    }
+    return {
+        entries,
+        save: async () => {
+            await writeFile(`${file}.tmp`, JSON.stringify(Object.fromEntries(entries)) + "\n", {
+                mode: 0o600,
+                flush: true,
+            });
+            await rename(`${file}.tmp`, file);
+        },
+    };
+}

package/dist/src/daemon/text-output.d.ts

@@ -0,0 +1,18 @@
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import type { WriteStream } from "node:fs";
+export declare function textOutput(child: ChildProcessWithoutNullStreams, options: {
+    readonly output: WriteStream;
+    readonly limit: number;
+}): (exitCode: number | null) => Promise<{
+    text: string;
+    failed: boolean;
+    model: null;
+    costUsd: null;
+    usage: import("../worker-usage.ts").WorkerUsage;
+    sessionId: null;
+    turns: null;
+    reviewFeedback?: string;
+    reviewVerdict: import("../review-subject.ts").ReviewVerdict | null;
+    ticketStatus: import("../ticket-status.ts").TicketStatus;
+    ticketStatusTooLarge?: true;
+}>;

package/dist/src/daemon/text-output.js

@@ -0,0 +1,44 @@
+import { usageOf } from "../worker-usage.js";
+import { reviewReportStream } from "../review-subject.js";
+import { ticketStatusStream } from "../ticket-status-stream.js";
+export function textOutput(child, options) {
+    const { output, limit } = options;
+    const status = ticketStatusStream(limit);
+    const verdict = reviewReportStream(limit);
+    let text = "";
+    let error = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+        error = (error + chunk).slice(-limit);
+    });
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        text = (text + chunk).slice(-limit);
+        status.write(chunk);
+        verdict.write(chunk);
+        if (!output.write(chunk)) {
+            child.stdout.pause();
+            output.once("drain", () => {
+                child.stdout.resume();
+            });
+        }
+    });
+    return async (exitCode) => {
+        await new Promise((resolve) => output.end(resolve));
+        const review = verdict.result();
+        return {
+            ...status.result(),
+            reviewVerdict: review.reviewVerdict,
+            ...(review.reviewVerdict === null
+                ? {}
+                : { reviewFeedback: review.feedback }),
+            text: exitCode === 0 ? text : error,
+            failed: exitCode !== 0,
+            model: null,
+            costUsd: null,
+            usage: usageOf(null),
+            sessionId: null,
+            turns: null,
+        };
+    };
+}

package/dist/src/daemon/ticket-dependency-waits.d.ts

@@ -0,0 +1,18 @@
+import { type Fields } from "../protocol.ts";
+interface Source {
+    readonly repository: string;
+    readonly command: readonly string[] | undefined;
+    readonly githubCommand?: readonly string[] | undefined;
+}
+export declare class TicketDependencyReadError extends Error {
+}
+export declare function ticketDependencyWaits(source: Source): {
+    readonly check: (candidate: {
+        readonly ticketId: string;
+        readonly ticket: string;
+        readonly signal: AbortSignal;
+    }) => Promise<readonly string[]>;
+    readonly refresh: (signal: AbortSignal) => Promise<readonly string[]>;
+    readonly skipped: () => readonly Fields[];
+};
+export {};

package/dist/src/daemon/ticket-dependency-waits.js

@@ -0,0 +1,60 @@
+import { hardDependencies } from "../ticket-dependencies.js";
+import { issueIdentity } from "../issue-identity.js";
+import { record } from "../protocol.js";
+import { calls } from "./github-calls.js";
+import { ticketText } from "./ticket-source.js";
+export class TicketDependencyReadError extends Error {
+}
+async function openDependencies(source, candidate) {
+    const github = calls({ repository: source.repository, gh: source.githubCommand ?? ["gh"] }, candidate.signal);
+    const issue = issueIdentity(candidate.ticket);
+    const repository = issue === null ? "{owner}/{repo}" : `${issue.owner}/${issue.repository}`;
+    const open = [];
+    for (const id of hardDependencies(candidate.ticket)) {
+        const response = JSON.parse(await github.gh(["api", `repos/${repository}/issues/${id}`]).catch(() => {
+            throw new TicketDependencyReadError("github-call-failed");
+        }));
+        const state = record(response)?.["state"];
+        if (state !== "open" && state !== "closed")
+            throw new Error("invalid-dependency-state");
+        if (state === "open")
+            open.push(id);
+    }
+    return open;
+}
+export function ticketDependencyWaits(source) {
+    const waiting = new Map();
+    let nextRefresh;
+    const check = async (candidate) => {
+        const open = await openDependencies(source, candidate);
+        if (open.length === 0)
+            waiting.delete(candidate.ticketId);
+        else
+            waiting.set(candidate.ticketId, open);
+        return open;
+    };
+    return {
+        check,
+        refresh: async (signal) => {
+            const tickets = [...waiting.keys()];
+            const ticketId = nextRefresh !== undefined && waiting.has(nextRefresh)
+                ? nextRefresh
+                : tickets[0];
+            if (ticketId === undefined)
+                return [];
+            nextRefresh = tickets[tickets.indexOf(ticketId) + 1] ?? tickets[0];
+            if (source.command === undefined)
+                throw new Error("ticket-source-not-configured");
+            const read = await ticketText(source.command, { ticketId, signal });
+            if (read.kind === "failed")
+                throw new TicketDependencyReadError("ticket-not-available");
+            await check({ ticketId, ticket: read.text, signal });
+            return [...waiting.keys()];
+        },
+        skipped: () => [...waiting].map(([ticketId, dependencies]) => ({
+            ticketId,
+            reason: "ticket-dependency-open",
+            dependencies,
+        })),
+    };
+}

package/dist/src/daemon/ticket-registration.d.ts

@@ -1,8 +1,9 @@
-import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import { type DeclaredWorkflow } from "../workflow-declaration.ts";
 import type { Resolver, WorkflowRegistration } from "../service/registry.ts";
-export declare const ticketWorkflow = "develop-ticket";
-export declare const promptWorkflow = "develop-prompt";
-export declare const issueWorkflow = "develop-issue";
+import type { LocalCommands } from "../command-settings.ts";
+export declare const ticketWorkflow: "develop-ticket";
+export declare const promptWorkflow: "develop-prompt";
+export declare const issueWorkflow: "develop-issue";
 export declare const ticketCapabilities: string[];
 export type TicketRegistrations = {
     readonly kind: "admitted";
@@ -15,12 +16,11 @@ export interface TicketSource {
     readonly repository: string;
     readonly command: readonly string[] | undefined;
     readonly nextCommand?: readonly string[] | undefined;
+    readonly githubCommand?: readonly string[] | undefined;
+    readonly skipTickets?: readonly string[];
 }
-export interface TicketRegistration {
-    readonly workflowId: string;
+export interface TicketRegistration extends Omit<WorkflowRegistration, "manifest" | "availability" | "admission"> {
     readonly functionName: string;
-    readonly resolver: Resolver;
-    readonly submission: WorkflowRegistration["submission"];
     readonly effectLimit: number;
     readonly capabilities?: readonly string[];
 }
@@ -30,4 +30,6 @@ export declare function promptResolver(repository: string): Resolver;
 export declare function ticketRegistrationTable(source: TicketSource, declarations?: readonly DeclaredWorkflow[]): readonly TicketRegistration[];
 export declare function ticketRegistrations(manifests: Readonly<Record<string, string | undefined>>, options: {
     readonly table: readonly TicketRegistration[];
+    readonly bundleDigest?: string;
+    readonly commands?: LocalCommands;
 }): Promise<TicketRegistrations>;