cloving 1.3.0 → 1.4.0

package/dist/src/daemon/turn-attempts.js

@@ -0,0 +1,166 @@
+import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
+import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
+import { capacityRefusal, retryKind, } from "./claude-turn.js";
+import { nativeHarness } from "../harness-defaults.js";
+import { profileCommand } from "../turn-profile.js";
+export function total(values) {
+    if (values.includes(null))
+        return null;
+    return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
+}
+export function combined(attempts) {
+    const { ends } = attempts;
+    const last = ends.at(-1) ?? { exitCode: null, result: undefined };
+    const result = last.result ?? ends.find((end) => end.result)?.result;
+    if (result === undefined)
+        return last;
+    return {
+        ...last,
+        result: {
+            ...result,
+            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
+            ...(last.result === undefined ? { failed: true, text: "" } : {}),
+            usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
+            costUsd: (attempts.start.worker ?? "claude") === "claude"
+                ? sessionCost(ends)
+                : total(ends.map((end) => end.result?.costUsd ?? null)),
+            turns: total(ends.map((end) => end.result?.turns ?? null)),
+        },
+    };
+}
+function selectedCommand(attempts, attempt) {
+    const { start, options } = attempts;
+    const original = start.command;
+    const previous = attempts.ends[attempt - 2];
+    if (options.resume !== undefined &&
+        attempt === options.resume.ends.length + 1) {
+        const model = previous?.result?.model;
+        return {
+            command: model == null ? original : replaceCommandModel(original, model),
+            fallback: undefined,
+        };
+    }
+    const ordinal = attempt -
+        attempts.ends
+            .slice(0, attempt - 1)
+            .filter((end) => end.resumedAfter !== undefined).length;
+    const fallback = previous !== undefined && kindOf(attempts, previous) === "capacity"
+        ? options.fallbackModels?.[ordinal - 3]
+        : undefined;
+    return {
+        command: fallback === undefined
+            ? original
+            : replaceCommandModel(original, fallback),
+        fallback,
+    };
+}
+export function attemptCommand(attempts, attempt) {
+    const { start } = attempts;
+    const { command: selected, fallback } = selectedCommand(attempts, attempt);
+    const budgetUsd = remainingBudget(attempts);
+    const command = budgetUsd === undefined
+        ? selected
+        : profileCommand(selected, { maxBudgetUsd: budgetUsd });
+    const model = start.output === "text" ? null : commandModel(command);
+    return {
+        command,
+        ...(budgetUsd === undefined ? {} : { budgetUsd }),
+        model,
+        ...(fallback !== undefined && model === null
+            ? { fallbackReason: "fallback-model-unavailable" }
+            : {}),
+    };
+}
+export function attemptUsage(attempts, completion) {
+    const { end, attempt } = completion;
+    const result = end.result ?? {
+        model: null,
+        usage: usageOf(null),
+        costUsd: null,
+        pricingModel: undefined,
+    };
+    const model = result.model ?? attemptCommand(attempts, attempt).model;
+    const usage = result.usage;
+    return {
+        attempt,
+        ...(end.resumedAfter === undefined
+            ? {}
+            : { resumedAfter: end.resumedAfter }),
+        retryKind: kindOf(attempts, end),
+        model,
+        ...(end.result?.models === undefined ? {} : { models: end.result.models }),
+        usage,
+        costUsd: (attempts.start.worker ?? "claude") === "claude"
+            ? attemptCost(attempts.ends, attempt - 1)
+            : result.costUsd,
+        estimatedCostUsd: attemptEstimate(attempts, { end, model, usage }),
+    };
+}
+export function sessionCost(ends) {
+    const sessions = new Map();
+    ends.forEach((end, index) => {
+        sessions.set(end.result?.sessionId ?? index, end.result?.costUsd ?? null);
+    });
+    return total([...sessions.values()]);
+}
+export function attemptCost(ends, index) {
+    const result = ends[index]?.result;
+    if (result?.costUsd == null)
+        return null;
+    const earlier = result.sessionId === null
+        ? undefined
+        : ends
+            .slice(0, index)
+            .findLast((end) => end.result?.sessionId === result.sessionId);
+    if (earlier === undefined)
+        return result.costUsd;
+    const cost = earlier.result?.costUsd;
+    return cost == null
+        ? null
+        : priceOf(Number((result.costUsd - cost).toFixed(12)));
+}
+function remainingBudget(attempts) {
+    if ((attempts.start.worker ?? "claude") !== "claude" ||
+        attempts.start.output === "text")
+        return undefined;
+    const budget = commandBudget(attempts.start.command);
+    if (budget === undefined)
+        return undefined;
+    const spent = sessionCost(attempts.ends);
+    return Math.max(0, Number((budget - (spent ?? 0)).toFixed(12)));
+}
+function commandBudget(original) {
+    const separator = original.indexOf("--");
+    const command = separator < 0 ? original : original.slice(0, separator);
+    const index = command.findLastIndex((part) => part === "--max-budget-usd" || part.startsWith("--max-budget-usd="));
+    if (index < 0)
+        return undefined;
+    const argument = command[index] ?? "";
+    const budget = Number(argument.includes("=") ? argument.split("=")[1] : command[index + 1]);
+    if (!Number.isFinite(budget) || budget < 0)
+        return undefined;
+    return budget;
+}
+export function attemptEstimate(attempts, result) {
+    const worker = attempts.start.worker ?? "claude";
+    if (!nativeHarness(worker))
+        return null;
+    const models = result.end.result?.models;
+    const prices = attempts.options.tokenPrices ?? {};
+    return estimatedCostUsd(result.usage, worker === "gemini" && models !== undefined
+        ? { worker, models, prices }
+        : {
+            worker,
+            price: priceForModel(prices, result.end.result?.pricingModel === undefined
+                ? result.model
+                : result.end.result.pricingModel),
+        });
+}
+export function kindOf(attempts, end) {
+    const { worker, output } = attempts.start;
+    if (output === "text" || !["claude", "codex"].includes(worker ?? "claude"))
+        return null;
+    if (worker !== "codex")
+        return end.retryKind ?? retryKind(end.result);
+    return capacityRefusal(end.result) ? "capacity" : null;
+}

package/dist/src/daemon/turn-report.d.ts

@@ -1,5 +1,6 @@
+import type { Recorded } from "./operation-journal.ts";
 import { type WorkTree } from "./work-tree.ts";
-import type { StreamResult } from "./claude-turn.ts";
+import { type StreamResult, type TurnEnd } from "./claude-turn.ts";
 interface ReportLimits {
     readonly summaryCharacters: number;
     readonly resultBytes: number;
@@ -15,4 +16,10 @@ export declare function turnReport(start: {
     readonly text?: string;
     readonly reason?: string;
 }>;
+export declare function turnOutcome(running: Parameters<typeof turnReport>[0] & {
+    readonly stopping: {
+        readonly requested: boolean;
+        readonly stoppedBy?: "daemon-shutdown";
+    };
+}, end: TurnEnd): Promise<Recorded>;
 export {};

package/dist/src/daemon/turn-report.js

@@ -2,6 +2,7 @@ import { ticketStatus } from "../ticket-status.js";
 import { reviewReport } from "../review-subject.js";
 import { sumPrices } from "../worker-usage.js";
 import { changes } from "./work-tree.js";
+import { succeeded } from "./claude-turn.js";
 function tailReport(fields, limit) {
     let text = JSON.stringify({ ...fields, summary: "" });
     if (Buffer.byteLength(text) > limit)
@@ -77,3 +78,41 @@ export async function turnReport(start, result) {
         turns: result.turns,
     }, { ...harness, limit: limits.resultBytes });
 }
+export async function turnOutcome(running, end) {
+    const endedAt = new Date().toISOString();
+    if (end.stopped === true && running.stopping.stoppedBy === "daemon-shutdown")
+        return stoppedOutcome(running.stopping, endedAt);
+    const { result } = end;
+    const report = result === undefined
+        ? {}
+        : await turnReport({
+            tree: running.tree,
+            harness: running.harness,
+            limits: running.limits,
+        }, result);
+    const { text, reason } = report;
+    if (succeeded(end) && reason === undefined && text !== undefined)
+        return { outcome: "completed", result: text, endedAt };
+    if (running.stopping.requested)
+        return stoppedOutcome(running.stopping, endedAt);
+    return failedOutcome(end, { ...report, endedAt });
+}
+function failedOutcome(end, report) {
+    const { text, reason, endedAt } = report;
+    return {
+        outcome: "failed",
+        reason: end.reason ?? reason ?? "claude-turn-failed",
+        ...(end.detail === undefined ? {} : { detail: end.detail }),
+        ...(text === undefined ? {} : { result: text }),
+        endedAt,
+    };
+}
+function stoppedOutcome(stopping, endedAt) {
+    return {
+        outcome: "stopped",
+        endedAt,
+        ...(stopping.stoppedBy === undefined
+            ? {}
+            : { stoppedBy: stopping.stoppedBy }),
+    };
+}

package/dist/src/daemon/work-tree.d.ts

@@ -25,3 +25,12 @@ export declare function treePlace(request: Pick<TreeRequest, "worktrees" | "targ
 export declare function workTree(request: TreeRequest): Promise<WorkTree>;
 export declare function treeFailure(error: unknown): string;
 export declare function changes(tree: WorkTree): Promise<Changes>;
+export declare function firstTurn(settings: {
+    readonly journal: string;
+}, opening: {
+    readonly tree: WorkTree;
+    readonly intent: {
+        readonly target: string;
+        readonly runId: string;
+    };
+}): Promise<WorkTree>;

package/dist/src/daemon/work-tree.js

@@ -1,3 +1,4 @@
+import { readRecord, writeRecord, operationKey } from "./operation-journal.js";
 import { execFile, spawn } from "node:child_process";
 import { existsSync } from "node:fs";
 import { mkdir, realpath } from "node:fs/promises";
@@ -207,3 +208,22 @@ async function changeLines(call) {
         .slice(0, lineLimit)
         .map((line) => line.slice(0, lineLength));
 }
+export async function firstTurn(settings, opening) {
+    const { tree, intent } = opening;
+    const { target, runId } = intent;
+    const folder = target.startsWith("branch:")
+        ? join(settings.journal, "runs", operationKey(runId), "targets")
+        : join(settings.journal, "targets");
+    const name = `${operationKey(target)}.json`;
+    const record = await readRecord(folder, name);
+    const startCommit = typeof record === "object" && record !== null
+        ? Reflect.get(record, "startCommit")
+        : undefined;
+    if (typeof startCommit === "string")
+        return { ...tree, startCommit };
+    await writeRecord(folder, {
+        name,
+        value: { target, startCommit: tree.startCommit },
+    });
+    return tree;
+}

package/dist/src/daemon/worker-profile.d.ts

@@ -0,0 +1,8 @@
+import { type HarnessSettings } from "../harness-defaults.ts";
+import type { TurnProfile, TurnProfiles } from "../turn-profile.ts";
+import type { ProtectedIntent } from "../broker/actor/delivery-intents.ts";
+export interface WorkerProfiles extends HarnessSettings {
+    readonly profiles?: TurnProfiles;
+    readonly workflowId?: (runId: string) => Promise<string | undefined>;
+}
+export declare function selectedProfile(settings: WorkerProfiles, intent: ProtectedIntent): Promise<TurnProfile | string>;

package/dist/src/daemon/worker-profile.js

@@ -0,0 +1,40 @@
+import { turnHarness } from "../harness-defaults.js";
+async function runWorkflow(settings, runId) {
+    try {
+        return await settings.workflowId?.(runId);
+    }
+    catch {
+        return undefined;
+    }
+}
+function unsupportedProfile(profile, harness) {
+    if (harness.output === "text" && profile.model !== undefined)
+        return "profile-model-not-supported";
+    if (harness.worker === "claude")
+        return undefined;
+    return [
+        profile.allowedTools,
+        profile.permissionMode,
+        profile.maxBudgetUsd,
+    ].some((value) => value !== undefined)
+        ? "profile-needs-claude"
+        : undefined;
+}
+export async function selectedProfile(settings, intent) {
+    if (intent.method === "run-turn")
+        return {};
+    const name = intent.method.slice("run-turn:".length);
+    const profiles = settings.profiles ?? {};
+    const profile = Object.hasOwn(profiles, name) ? profiles[name] : undefined;
+    if (profile === undefined)
+        return "unknown-profile";
+    const unsupported = unsupportedProfile(profile, turnHarness(settings, profile));
+    if (unsupported !== undefined)
+        return unsupported;
+    if (profile.workflows === undefined)
+        return profile;
+    const workflowId = await runWorkflow(settings, intent.runId);
+    if (workflowId === undefined || !profile.workflows.includes(workflowId))
+        return "profile-not-allowed";
+    return profile;
+}

package/dist/src/daemon/workflow-schedules.d.ts

@@ -1,10 +1,12 @@
 import type { WorkerPort } from "../service/worker-port.ts";
 import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { SchedulePort } from "../schedule-progress.ts";
 interface ScheduleSettings {
     readonly directory: string;
     readonly runtimeDirectory: string;
     readonly declarations?: readonly DeclaredWorkflow[];
     readonly scheduleClock?: () => number;
+    readonly scheduleObservationClock?: () => number;
     readonly submit: WorkerPort["submit"];
 }
 export declare function scheduledDeclarations(settings: {
@@ -23,5 +25,6 @@ export declare function scheduledDeclarations(settings: {
 };
 export declare function workflowSchedules(settings: ScheduleSettings): Promise<{
     readonly stop: () => Promise<void>;
+    readonly request: SchedulePort;
 }>;
 export {};

package/dist/src/daemon/workflow-schedules.js

@@ -1,10 +1,12 @@
 import { join } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { mutate } from "../cli/client.js";
-import { record } from "../protocol.js";
+import { record, } from "../protocol.js";
 import { runIdentity } from "../service/submission-handler.js";
 import { configuredSchedules } from "../workflow-schedule.js";
 import { declaredInput } from "./declared-input.js";
+import { finishedScheduleRun } from "../schedule-progress.js";
+import { scheduleControl } from "./schedule-control.js";
 import { scheduleJournal, } from "./schedule-journal.js";
 export function scheduledDeclarations(settings, limits) {
     try {
@@ -60,6 +62,8 @@ async function pending(settings, entry) {
     const { pending: planned, ...rest } = progress;
     if (planned === undefined)
         return progress;
+    if (progress.state !== "active")
+        throw new Error("invalid-schedule-state");
     const identity = commandId(workflowId, progress.lastSlot);
     if (planned.kind === "skip") {
         const receipt = await settings.submit({
@@ -86,10 +90,7 @@ async function pending(settings, entry) {
         throw new Error(reply.error.code);
     return { ...rest, activeCommandId: identity };
 }
-async function active(settings, progress) {
-    if (progress.activeCommandId === undefined)
-        return undefined;
-    const runId = runIdentity(progress.activeCommandId);
+async function scheduledRunStatus(settings, runId) {
     const reply = await request(settings, {
         operation: "status",
         parameters: { runId },
@@ -99,45 +100,76 @@ async function active(settings, progress) {
     const run = record(reply.result["run"]);
     if (typeof run?.["state"] !== "string")
         throw new Error("invalid-schedule-run");
+    return run;
+}
+async function active(settings, progress) {
+    if (progress.activeCommandId === undefined)
+        return progress;
+    const now = (settings.scheduleClock ?? Date.now)();
+    const observedAt = settings.observationClock();
+    const next = settings.nextObservations.get(progress.activeCommandId) ?? 0;
+    const due = progress.state === "active" && now >= progress.nextAt;
+    if (!due && observedAt < next)
+        return progress;
+    const runId = runIdentity(progress.activeCommandId);
+    const run = await scheduledRunStatus(settings, runId);
+    settings.nextObservations.set(progress.activeCommandId, observedAt + 5000);
     if (run["condition"] === "cancelling")
-        return runId;
-    return ["completed", "failed", "cancelled"].includes(run["state"])
+        return progress;
+    if (!["completed", "failed", "cancelled"].includes(String(run["state"])))
+        return progress;
+    settings.nextObservations.delete(progress.activeCommandId);
+    return finishedScheduleRun(progress, {
+        runId,
+        status: run,
+        now: (settings.scheduleClock ?? Date.now)(),
+    });
+}
+async function advance(settings, declaration) {
+    settings.signal.throwIfAborted();
+    const { workflowId, schedule } = declaration;
+    if (schedule === undefined)
+        return;
+    let progress = settings.journal.entries.get(workflowId);
+    if (progress === undefined)
+        throw new Error("missing-schedule-state");
+    if (progress.pending !== undefined) {
+        progress = await pending(settings, { workflowId, progress });
+        settings.journal.entries.set(workflowId, progress);
+        await settings.journal.save();
+        if (settings.now < progress.nextAt)
+            return;
+    }
+    const observed = await active(settings, progress);
+    if (observed !== progress) {
+        progress = observed;
+        settings.journal.entries.set(workflowId, progress);
+        await settings.journal.save();
+    }
+    if (progress.state !== "active" || settings.now < progress.nextAt)
+        return;
+    const runId = progress.activeCommandId === undefined
         ? undefined
-        : runId;
+        : runIdentity(progress.activeCommandId);
+    const planned = runId === undefined
+        ? { kind: "run", input: schedule.input }
+        : { kind: "skip", runId };
+    const elapsed = Math.floor((settings.now - progress.nextAt) / (schedule.everySeconds * 1000)) + 1;
+    progress = {
+        ...progress,
+        lastSlot: progress.lastSlot + elapsed,
+        nextAt: progress.nextAt + elapsed * schedule.everySeconds * 1000,
+        pending: planned,
+    };
+    settings.journal.entries.set(workflowId, progress);
+    await settings.journal.save();
+    settings.journal.entries.set(workflowId, await pending(settings, { workflowId, progress }));
+    await settings.journal.save();
 }
 async function tick(settings, journal) {
     const now = (settings.scheduleClock ?? Date.now)();
-    for (const declaration of settings.declarations ?? []) {
-        settings.signal.throwIfAborted();
-        const { workflowId, schedule } = declaration;
-        if (schedule === undefined)
-            continue;
-        let progress = journal.entries.get(workflowId);
-        if (progress === undefined)
-            throw new Error("missing-schedule-state");
-        if (progress.pending !== undefined) {
-            progress = await pending(settings, { workflowId, progress });
-            journal.entries.set(workflowId, progress);
-            await journal.save();
-        }
-        if (now < progress.nextAt)
-            continue;
-        const runId = await active(settings, progress);
-        const planned = runId === undefined
-            ? { kind: "run", input: schedule.input }
-            : { kind: "skip", runId };
-        const elapsed = Math.floor((now - progress.nextAt) / (schedule.everySeconds * 1000)) + 1;
-        progress = {
-            ...progress,
-            lastSlot: progress.lastSlot + elapsed,
-            nextAt: progress.nextAt + elapsed * schedule.everySeconds * 1000,
-            pending: planned,
-        };
-        journal.entries.set(workflowId, progress);
-        await journal.save();
-        journal.entries.set(workflowId, await pending(settings, { workflowId, progress }));
-        await journal.save();
-    }
+    for (const declaration of settings.declarations ?? [])
+        await advance({ ...settings, journal, now }, declaration);
 }
 async function initialized(settings) {
     const journal = await scheduleJournal(join(settings.directory, "state"));
@@ -150,36 +182,57 @@ async function initialized(settings) {
         if (prior?.everySeconds === schedule.everySeconds)
             continue;
         journal.entries.set(workflowId, {
-            ...prior,
+            ...(prior ?? {
+                lastSlot: 0,
+                state: "active",
+                failuresInRow: 0,
+                failedRunIds: [],
+            }),
             everySeconds: schedule.everySeconds,
-            lastSlot: prior?.lastSlot ?? 0,
             nextAt: now + schedule.everySeconds * 1000,
         });
     }
-    await journal.save();
+    if (settings.declarations?.some((entry) => entry.schedule !== undefined))
+        await journal.save();
     return journal;
 }
 export async function workflowSchedules(settings) {
-    if (!(settings.declarations ?? []).some((entry) => entry.schedule !== undefined))
-        return { stop: () => Promise.resolve() };
+    const declarations = settings.declarations ?? [];
     const journal = await initialized(settings);
     const stopping = new AbortController();
     const stopped = () => stopping.signal.aborted;
+    const context = {
+        ...settings,
+        signal: stopping.signal,
+        nextObservations: new Map(),
+        observationClock: settings.scheduleObservationClock ?? (() => performance.now()),
+    };
+    const control = scheduleControl({
+        journal,
+        declarations,
+        clock: settings.scheduleClock ?? Date.now,
+        signal: stopping.signal,
+        recoverPending: (entry) => pending(context, entry),
+        tick: () => tick(context, journal),
+    });
     const running = (async () => {
         while (!stopped()) {
             await delay(250, undefined, { signal: stopping.signal }).catch(() => undefined);
             if (stopped())
                 return;
-            await tick({ ...settings, signal: stopping.signal }, journal);
+            await control.tick();
         }
     })().catch((error) => {
         if (!stopped())
             process.stderr.write(`Schedules stopped: ${String(error)}\n`);
+        stopping.abort();
     });
     return {
+        request: control.request,
         stop: async () => {
             stopping.abort();
             await running;
+            await control.drain();
         },
     };
 }

package/dist/src/daemon/worktree-processes.d.ts

@@ -0,0 +1,9 @@
+export declare function recordedProcessDirectory(processId: number): Promise<string | undefined>;
+interface StopRequest {
+    readonly directory: string;
+    readonly folder: string;
+    readonly signal: AbortSignal;
+    readonly stopMilliseconds?: number;
+}
+export declare function stopTreeProcesses(request: StopRequest): Promise<boolean>;
+export {};