cloving 1.3.0 → 1.5.0

package/dist/src/daemon/claude-worker.js

@@ -1,94 +1,109 @@
-import { join } from "node:path";
 import { turnHarness } from "../harness-defaults.js";
-import { profileCommand, } from "../turn-profile.js";
-import { succeeded } from "./claude-turn.js";
-import { turnReport } from "./turn-report.js";
+import { attachmentEnvironment } from "./attachments.js";
+import { profileCommand } from "../turn-profile.js";
+import {} from "./claude-turn.js";
+import { turnOutcome } from "./turn-report.js";
 import { startRetriedTurn } from "./retrying-turn.js";
-import { interrupted, located, operationKey, readRecord as read, replied, reportOf, writeRecord as written, } from "./operation-journal.js";
-import { treeFailure, workTree } from "./work-tree.js";
+import { located, readRecord as read, replied, reportOf, writeRecord as written, } from "./operation-journal.js";
+import { firstTurn, treeFailure, workTree, } from "./work-tree.js";
+import { recordedHarness, interruptedReply, processAlive, uncertainProcess, resumePlan, recordedField, resumableHarness, recoverTurn, turnIntent, } from "./claude-recovery.js";
+import { checkedProfiles, selectedProfile } from "./worker-profile.js";
 export const claudeLimits = {
     summaryCharacters: 16_000,
     resultBytes: 48_000,
-    stopMilliseconds: 2_000,
+    stopMilliseconds: 15_000,
 };
 const worker = "claude";
-async function recordedHarness(journal, operationId) {
-    const { folder } = located(journal, operationId);
-    const started = await read(folder, "started.json").catch(() => undefined);
-    const harness = Reflect.get(Object(started), "harness");
-    return typeof harness === "string" ? harness : undefined;
-}
-function alive(processId) {
-    if (typeof processId !== "number")
-        return false;
-    try {
-        process.kill(processId, 0);
-        return true;
-    }
-    catch (error) {
-        return Reflect.get(Object(error), "code") === "EPERM";
-    }
+function reconciled(parts, call) {
+    const { intent } = call;
+    const { key } = located(parts.settings.journal, intent.operationId);
+    const pending = parts.reconciling.get(key);
+    if (pending !== undefined)
+        return pending;
+    const reconciliation = reconcileOperation(parts, call).finally(() => {
+        if (parts.reconciling.get(key) === reconciliation)
+            parts.reconciling.delete(key);
+    });
+    parts.reconciling.set(key, reconciliation);
+    return reconciliation;
 }
-async function reconciled(parts, intent) {
+async function reconcileOperation(parts, call) {
+    const { intent, signal } = call;
     const { key, folder } = located(parts.settings.journal, intent.operationId);
     const result = await read(folder, "result.json");
-    if (result !== undefined)
+    if (result !== undefined &&
+        (recordedField(result, "stoppedBy") !== "daemon-shutdown" ||
+            parts.stopping.signal.aborted))
         return replied({ receipt: key, worker }, result);
     if (parts.active.has(key))
         return { kind: "accepted", receipt: key, worker };
     const started = await read(folder, "started.json");
     if (started === undefined)
         return { kind: "absent" };
-    if (alive(Reflect.get(Object(started), "processId")))
+    if (await uncertainProcess(started))
         return { kind: "unknown", evidence: `claude-turn:${key}` };
-    const ending = interrupted("claude-turn-interrupted");
-    await written(folder, { name: "result.json", value: ending });
-    return replied({ receipt: key, worker }, ending);
+    return resumed(parts, { intent, key, folder, started, signal });
 }
-async function outcome(running, end) {
-    const endedAt = new Date().toISOString();
-    const { result } = end;
-    const report = result === undefined
-        ? {}
-        : await turnReport({
-            tree: running.tree,
-            harness: running.harness,
-            limits: claudeLimits,
-        }, result);
-    const { text, reason } = report;
-    if (succeeded(end) && reason === undefined && text !== undefined)
-        return { outcome: "completed", result: text, endedAt };
-    if (running.stopping.requested)
-        return { outcome: "stopped", endedAt };
-    return {
-        outcome: "failed",
-        reason: end.reason ?? reason ?? "claude-turn-failed",
-        ...(end.detail === undefined ? {} : { detail: end.detail }),
-        ...(text === undefined ? {} : { result: text }),
-        endedAt,
-    };
+async function resumed(parts, request) {
+    const { intent, started, key, folder, signal } = request;
+    const aborted = () => signal.aborted;
+    if (!turnIntent(intent))
+        return interruptedReply(request, "claude-turn-interrupted");
+    if (aborted())
+        return failure("claude-start-aborted");
+    const current = parts.settings.turnSettings();
+    const plan = await resumePlan({ ...parts.settings, ...current }, { folder, started, intent, signal });
+    if (aborted())
+        return failure("claude-start-aborted");
+    if (typeof plan === "string")
+        return interruptedReply(request, plan);
+    const prompt = await parts.settings.document(intent.payload);
+    if (aborted())
+        return failure("claude-start-aborted");
+    if (prompt === undefined)
+        return interruptedReply(request, "claude-turn-interrupted");
+    await begun(parts, {
+        key,
+        folder,
+        intent,
+        ...plan,
+        signal,
+        prompt,
+        turnSettings: current,
+    });
+    return { kind: "accepted", receipt: key, worker };
 }
 async function concluded(parts, running) {
     const { intent, key } = running;
-    const recorded = await outcome(running, await running.turn.ended);
+    const recorded = await turnOutcome({ ...running, limits: claudeLimits }, await running.turn.ended);
     await written(running.folder, { name: "result.json", value: recorded });
+    if (recorded.stoppedBy === "daemon-shutdown")
+        return;
     await parts.settings.report(reportOf({ intent, commandId: `claude-turn:${key}`, worker }, recorded));
 }
 async function begun(parts, start) {
     const { intent, key, folder, tree, prompt } = start;
     const { harness } = start;
     const stopping = { requested: false, controller: new AbortController() };
+    const cancel = () => {
+        stopping.controller.abort();
+    };
+    start.signal.addEventListener("abort", cancel, { once: true });
+    if (start.signal.aborted)
+        cancel();
     const turn = startRetriedTurn({
         ...harness,
         resultBytes: claudeLimits.resultBytes,
         command: profileCommand(harness.command, start.profile),
-        environment: start.profile.env ?? {},
+        environment: await attachmentEnvironment(parts.settings, start),
         turnId: `${intent.runId}-${key}`,
         directory: tree.path,
         folder,
         prompt,
     }, {
         ...parts.settings,
+        ...start.turnSettings,
+        ...(start.resume === undefined ? {} : { resume: start.resume }),
         stopping: stopping.controller,
         metadata: {
             harness: harness.worker,
@@ -100,6 +115,7 @@ async function begun(parts, start) {
         },
     });
     parts.active.set(key, {
+        worker: harness.worker,
         turn,
         stopping,
         concluded: turn
@@ -107,74 +123,13 @@ async function begun(parts, start) {
             .catch(() => undefined)
             .finally(() => parts.active.delete(key)),
     });
-    await turn;
-}
-async function firstTurn(settings, opening) {
-    const { tree, intent } = opening;
-    const { target, runId } = intent;
-    const folder = target.startsWith("branch:")
-        ? join(settings.journal, "runs", operationKey(runId), "targets")
-        : join(settings.journal, "targets");
-    const name = `${operationKey(target)}.json`;
-    const startCommit = Reflect.get(Object(await read(folder, name)), "startCommit");
-    if (typeof startCommit === "string")
-        return { ...tree, startCommit };
-    await written(folder, {
-        name,
-        value: { target, startCommit: tree.startCommit },
+    await turn.finally(() => {
+        start.signal.removeEventListener("abort", cancel);
     });
-    return tree;
-}
-function turnIntent(intent) {
-    return (intent.capability === worker &&
-        (intent.method === "run-turn" ||
-            /^run-turn:[A-Za-z0-9_-]{1,119}$/.test(intent.method ?? "")) &&
-        intent.target !== undefined &&
-        intent.payload !== undefined &&
-        intent.leaseGeneration !== undefined);
 }
 function failure(reason) {
     return { kind: "failed", reason };
 }
-async function runWorkflow(settings, runId) {
-    try {
-        return await settings.workflowId?.(runId);
-    }
-    catch {
-        return undefined;
-    }
-}
-function unsupportedProfile(profile, harness) {
-    if (harness.output === "text" && profile.model !== undefined)
-        return "profile-model-not-supported";
-    if (harness.worker === "claude")
-        return undefined;
-    return [
-        profile.allowedTools,
-        profile.permissionMode,
-        profile.maxBudgetUsd,
-    ].some((value) => value !== undefined)
-        ? "profile-needs-claude"
-        : undefined;
-}
-async function selectedProfile(settings, intent) {
-    if (intent.method === "run-turn")
-        return {};
-    const name = intent.method.slice("run-turn:".length);
-    const profiles = settings.profiles ?? {};
-    const profile = Object.hasOwn(profiles, name) ? profiles[name] : undefined;
-    if (profile === undefined)
-        return "unknown-profile";
-    const unsupported = unsupportedProfile(profile, turnHarness(settings, profile));
-    if (unsupported !== undefined)
-        return unsupported;
-    if (profile.workflows === undefined)
-        return profile;
-    const workflowId = await runWorkflow(settings, intent.runId);
-    if (workflowId === undefined || !profile.workflows.includes(workflowId))
-        return "profile-not-allowed";
-    return profile;
-}
 async function delivered(parts, call) {
     const { intent, signal } = call;
     const { settings } = parts;
@@ -183,10 +138,11 @@ async function delivered(parts, call) {
         return failure("unsupported-effect");
     if (aborted())
         return failure("claude-start-aborted");
-    const known = await reconciled(parts, intent);
+    const known = await reconciled(parts, call);
     if (known.kind !== "absent")
         return known;
-    const profile = await selectedProfile(settings, intent);
+    const current = settings.turnSettings();
+    const profile = await selectedProfile({ ...settings, ...current }, intent);
     if (typeof profile === "string")
         return failure(profile);
     const prompt = await settings.document(intent.payload);
@@ -213,6 +169,8 @@ async function delivered(parts, call) {
         prompt,
         profile,
         harness: turnHarness(settings, profile),
+        signal,
+        turnSettings: current,
     });
     return { receipt: place.key, worker };
 }
@@ -224,37 +182,80 @@ async function halted(parts, active) {
     await active.concluded;
 }
 async function stopped(parts, intent) {
-    const { key } = located(parts.settings.journal, intent.operationId);
+    const { key, folder } = located(parts.settings.journal, intent.operationId);
     const active = parts.active.get(key);
-    if (active !== undefined)
+    if (active !== undefined) {
+        delete active.stopping.stoppedBy;
         await halted(parts, active);
-    return reconciled(parts, intent);
+    }
+    const result = await read(folder, "result.json");
+    if (result !== undefined &&
+        recordedField(result, "stoppedBy") !== "daemon-shutdown")
+        return replied({ receipt: key, worker }, result);
+    const started = await read(folder, "started.json");
+    if (started === undefined)
+        return { kind: "absent" };
+    if (!resumableHarness(recordedField(started, "harness")))
+        return reconcileOperation(parts, { intent, signal: parts.stopping.signal });
+    if (processAlive(recordedField(started, "processId")))
+        return { kind: "unknown", evidence: `claude-turn:${key}` };
+    if (!turnIntent(intent))
+        return failure("unsupported-effect");
+    const recorded = {
+        outcome: "stopped",
+        endedAt: new Date().toISOString(),
+    };
+    await written(folder, { name: "result.json", value: recorded });
+    await parts.settings.report(reportOf({ intent, commandId: `claude-turn:${key}`, worker }, recorded));
+    return replied({ receipt: key, worker }, recorded);
+}
+function stoppingOperation(parts, intent) {
+    const { key } = located(parts.settings.journal, intent.operationId);
+    const pending = parts.reconciling.get(key);
+    const stop = () => stopped(parts, intent);
+    const stopping = pending === undefined ? stop() : pending.then(stop, stop);
+    const settled = stopping.finally(() => {
+        if (parts.reconciling.get(key) === settled)
+            parts.reconciling.delete(key);
+    });
+    parts.reconciling.set(key, settled);
+    return settled;
 }
 export function claudeWorker(settings) {
     if (settings.command.length === 0)
         throw new Error("invalid-claude-command");
     turnHarness(settings);
-    for (const profile of Object.values(settings.profiles ?? {}))
-        turnHarness(settings, profile);
+    checkedProfiles(settings, settings.turnSettings().profiles);
     const parts = {
         settings,
         active: new Map(),
+        reconciling: new Map(),
         stopping: new AbortController(),
     };
     const guarded = (intent, run) => intent.method === "run-turn:analyze-chats"
         ? Promise.resolve(failure("unsupported-effect"))
         : run();
     return {
+        recover: (intent) => turnIntent(intent)
+            ? recoverTurn({ ...settings, intent, signal: parts.stopping.signal }, () => reconciled(parts, { intent, signal: parts.stopping.signal }))
+            : Promise.resolve(),
         harness: (operationId) => recordedHarness(settings.journal, operationId),
         deliver: (intent, signal) => guarded(intent, () => delivered(parts, {
             intent,
             signal: AbortSignal.any([signal, parts.stopping.signal]),
         })),
-        reconcile: (intent) => guarded(intent, () => reconciled(parts, intent)),
-        stop: (intent) => guarded(intent, () => stopped(parts, intent)),
+        reconcile: (intent, signal) => guarded(intent, () => reconciled(parts, {
+            intent,
+            signal: AbortSignal.any([signal, parts.stopping.signal]),
+        })),
+        stop: (intent) => guarded(intent, () => stoppingOperation(parts, intent)),
         close: async () => {
             parts.stopping.abort();
-            await Promise.all([...parts.active.values()].map((active) => halted(parts, active)));
+            await Promise.all([...parts.active.values()].map((active) => {
+                if (resumableHarness(active.worker) && !active.stopping.requested)
+                    active.stopping.stoppedBy = "daemon-shutdown";
+                return halted(parts, active);
+            }));
         },
     };
 }

package/dist/src/daemon/daemon-adapters.d.ts

@@ -12,12 +12,15 @@ import type { LocalCommands } from "../command-settings.ts";
 import type { DeclaredWorkflow } from "../workflow-declaration.ts";
 import type { WorkflowRefusal } from "../workflow-refusals.ts";
 import type { HarnessSettings } from "../harness-defaults.ts";
+import type { RetentionSettings } from "../record-retention.ts";
+import { type SettingDigests, type TurnSettingsSource } from "./settings-reload.ts";
 export interface DaemonSettings extends HarnessSettings {
     readonly commands?: LocalCommands;
     readonly commandSeconds?: number;
     readonly fallbackModels?: readonly string[];
     readonly tokenPrices?: TokenPrices;
     readonly profiles?: TurnProfiles;
+    readonly settingDigests?: SettingDigests;
     readonly directory: string;
     readonly runtimeDirectory: string;
     readonly repository: string;
@@ -28,6 +31,7 @@ export interface DaemonSettings extends HarnessSettings {
     readonly workflows: Readonly<Record<string, WorkflowFunction | undefined>>;
     readonly handler: WorkflowHandler;
     readonly loadBundle?: WorkflowBundleLoader;
+    readonly reloadCommand?: readonly string[];
     readonly command: readonly string[];
     readonly ticketCommand?: readonly string[];
     readonly nextCommand?: readonly string[];
@@ -35,8 +39,11 @@ export interface DaemonSettings extends HarnessSettings {
     readonly notifyMilliseconds?: number;
     readonly queueMilliseconds?: number;
     readonly scheduleClock?: () => number;
+    readonly retention?: RetentionSettings;
+    readonly retentionClock?: () => number;
     readonly schedules?: unknown;
     readonly queueLanes?: number;
+    readonly queueDailySpendLimitUsd?: number;
     readonly resetQueue?: boolean;
     readonly githubCommand?: readonly string[];
     readonly requiredChecks?: readonly string[] | null;
@@ -46,6 +53,7 @@ export interface DaemonSettings extends HarnessSettings {
 }
 export interface Adapters {
     readonly worker: ClaudeWorker;
+    readonly turnSettings: TurnSettingsSource;
     readonly github: GithubWorker;
     readonly runs: StartDelivery;
     readonly handoff: EffectDelivery;
@@ -56,3 +64,4 @@ export declare function adapters(settings: DaemonSettings, options: {
     readonly generations: ReturnType<typeof workflowGenerations>;
 }): Adapters;
 export declare function recoverCommands(commands: CommandWorker, ports: HostPorts): Promise<void>;
+export declare function recoverClaude(worker: Pick<ClaudeWorker, "harness" | "recover">, ports: Pick<HostPorts, "state" | "operations">): Promise<void>;

package/dist/src/daemon/daemon-adapters.js

@@ -6,11 +6,14 @@ import { claudeWorker } from "./claude-worker.js";
 import { githubWorker } from "./github-worker.js";
 import { commandWorker } from "./command-worker.js";
 import { handoffWorker } from "./handoff-worker.js";
+import { resumableHarness } from "./claude-recovery.js";
+import { turnSettingsSource, } from "./settings-reload.js";
 function githubOf(settings, given) {
     const { directory, repository, checkCommand } = settings;
     return githubWorker({
         document: async (digest) => (await given).document(digest),
         report: async (effect) => (await given).reportEffect(effect),
+        observe: async (observation) => (await given).submit({ kind: "report-observation", ...observation }),
         operationStatus: async (operationId) => (await (await given).operation(operationId))?.status,
         runStatus: async (runId) => (await (await given).state()).runs[runId]?.status,
         repository,
@@ -40,11 +43,14 @@ export function adapters(settings, options) {
     const { given, generations } = options;
     const document = async (digest) => (await given).document(digest);
     const report = async (effect) => (await given).reportEffect(effect);
-    const worker = claudeWorker({
+    const turnSettings = turnSettingsSource({
+        profiles: settings.profiles ?? {},
         fallbackModels: settings.fallbackModels ?? [],
         tokenPrices: settings.tokenPrices ?? {},
+    });
+    const worker = claudeWorker({
         ...workerHarness(settings),
-        profiles: settings.profiles ?? {},
+        turnSettings: turnSettings.current,
         workflowId: async (runId) => (await given).workflow.id(runId),
         repository,
         document,
@@ -70,7 +76,7 @@ export function adapters(settings, options) {
         document,
         report,
     });
-    return { worker, github, runs, handoff, commands };
+    return { worker, github, runs, handoff, commands, turnSettings };
 }
 function handoffOf(settings, given) {
     return handoffWorker({
@@ -81,11 +87,33 @@ function handoffOf(settings, given) {
     });
 }
 export async function recoverCommands(commands, ports) {
+    await commands.recover(await recoveryCalls(ports, "command"));
+}
+export async function recoverClaude(worker, ports) {
+    for (const intent of await recoveryCalls(ports, "claude")) {
+        if (!resumableHarness(await worker.harness(intent.operationId)))
+            continue;
+        if (!(await stillRecoverable(ports, intent)))
+            continue;
+        await worker.recover(intent);
+    }
+}
+async function stillRecoverable(ports, intent) {
+    const operation = (await ports.operations(intent.runId)).find((current) => current.operationId === intent.operationId);
+    const run = (await ports.state()).runs[intent.runId];
+    return (run?.status === "running" &&
+        operation !== undefined &&
+        operation.leaseGeneration === intent.leaseGeneration &&
+        ["running", "recovering", "dispatch-pending"].includes(operation.status));
+}
+async function recoveryCalls(ports, capability) {
     const state = await ports.state();
     const calls = [];
     for (const run of Object.values(state.runs)) {
+        if (capability === "claude" && run.status !== "running")
+            continue;
         for (const operation of await ports.operations(run.runId)) {
-            if (operation.capability !== "command" ||
+            if (operation.capability !== capability ||
                 operation.leaseGeneration === null ||
                 !["running", "recovering", "dispatch-pending"].includes(operation.status))
                 continue;
@@ -98,5 +126,5 @@ export async function recoverCommands(commands, ports) {
             });
         }
     }
-    await commands.recover(calls);
+    return calls;
 }

package/dist/src/daemon/daemon-loops.d.ts

@@ -1,11 +1,15 @@
+import type { scheduleConnection } from "./schedule-control.ts";
+import type { queueConnection } from "./queue-control.ts";
 import { workflowSchedules } from "./workflow-schedules.ts";
 interface LoopSettings extends ParametersOfSchedules {
+    readonly scheduleConnection?: ReturnType<typeof scheduleConnection>;
+    readonly queueConnection?: ReturnType<typeof queueConnection>;
     readonly notifyCommand?: readonly string[];
     readonly notifyMilliseconds?: number;
     readonly queueMilliseconds?: number;
     readonly queueLanes?: number;
     readonly resetQueue?: boolean;
-    readonly queueWorkflowId: string;
+    readonly queueWorkflowId: string | undefined;
 }
 type ParametersOfSchedules = Parameters<typeof workflowSchedules>[0];
 export declare function daemonLoops(settings: LoopSettings): Promise<{

package/dist/src/daemon/daemon-loops.js

@@ -1,8 +1,25 @@
 import { workflowSchedules } from "./workflow-schedules.js";
 import { issueQueue } from "./issue-queue.js";
 import { notifyRuns } from "./notify.js";
+async function startedQueue(settings) {
+    if (settings.queueMilliseconds === undefined ||
+        settings.queueWorkflowId === undefined)
+        return undefined;
+    const queue = issueQueue({
+        directory: settings.directory,
+        runtimeDirectory: settings.runtimeDirectory,
+        intervalMilliseconds: settings.queueMilliseconds,
+        lanes: settings.queueLanes ?? 1,
+        resetQueue: settings.resetQueue ?? false,
+        workflowId: settings.queueWorkflowId,
+    });
+    settings.queueConnection?.connect(queue.port);
+    await queue.ready;
+    return queue;
+}
 export async function daemonLoops(settings) {
     const schedules = await workflowSchedules(settings);
+    settings.scheduleConnection?.connect(schedules.request);
     try {
         const notifier = settings.notifyCommand === undefined
             ? undefined
@@ -14,16 +31,7 @@ export async function daemonLoops(settings) {
                     ? {}
                     : { intervalMilliseconds: settings.notifyMilliseconds }),
             });
-        const queue = settings.queueMilliseconds === undefined
-            ? undefined
-            : issueQueue({
-                directory: settings.directory,
-                runtimeDirectory: settings.runtimeDirectory,
-                intervalMilliseconds: settings.queueMilliseconds,
-                lanes: settings.queueLanes ?? 1,
-                resetQueue: settings.resetQueue ?? false,
-                workflowId: settings.queueWorkflowId,
-            });
+        const queue = await startedQueue(settings);
         return {
             stop: async () => {
                 await schedules.stop();