cloving 1.2.0 → 1.3.0

package/dist/src/daemon/command-process.js

@@ -0,0 +1,164 @@
+import { spawn } from "node:child_process";
+import { closeSync, mkdtempSync, openSync, rmSync, writeSync } from "node:fs";
+import { join } from "node:path";
+import { turnEnvironment } from "./claude-turn.js";
+import { commandLimits } from "../command-settings.js";
+import { isInstant } from "../authority/instants.js";
+const commandClock = {
+    now: () => Date.now(),
+    timer: (wait) => {
+        const timer = setTimeout(wait.expired, wait.milliseconds);
+        return () => {
+            clearTimeout(timer);
+        };
+    },
+};
+function boundedOutput(stream, options) {
+    let received = 0;
+    stream.on("data", (chunk) => {
+        const kept = chunk.subarray(0, Math.max(0, options.bytes - received));
+        received += chunk.length;
+        if (kept.length > 0)
+            options.receive(kept);
+        if (received > options.bytes)
+            options.fail("command-output-limit");
+    });
+}
+function stopChild(child) {
+    try {
+        if (child.pid !== undefined)
+            process.kill(-child.pid, "SIGKILL");
+    }
+    catch (error) {
+        const code = Reflect.get(Object(error), "code");
+        if (code !== "ESRCH" && code !== "EPERM")
+            throw error;
+        child.kill("SIGKILL");
+    }
+    child.stdin.destroy();
+    child.stdout.destroy();
+    child.stderr.destroy();
+}
+function spawnCommand(start, configuration) {
+    const [executable = "", ...argumentsList] = start.command;
+    return spawn(executable, argumentsList, {
+        cwd: start.directory,
+        env: {
+            ...turnEnvironment(configuration, { ...start, prompt: start.payload }),
+            CLOVING_RUN_ID: start.runId,
+            CLOVING_OPERATION_ID: start.operationId,
+        },
+        stdio: ["pipe", "pipe", "pipe"],
+        detached: true,
+    });
+}
+function writeStderr(descriptor, chunk) {
+    try {
+        writeSync(descriptor, chunk);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function releaseCommand(resources) {
+    try {
+        if (resources.stderr !== undefined)
+            closeSync(resources.stderr);
+    }
+    finally {
+        rmSync(resources.configuration, { recursive: true, force: true });
+    }
+}
+function prepareCommand(start, clock) {
+    const configuration = mkdtempSync(join(start.folder, "gh-config-"));
+    let stderr;
+    let child;
+    try {
+        stderr = openSync(join(start.folder, "command.stderr.log"), "w", 0o600);
+        const reason = deadlineRefusal(start, clock);
+        if (reason !== undefined)
+            return { reason };
+        child = spawnCommand(start, configuration);
+        return { child, stderr, configuration };
+    }
+    finally {
+        if (child === undefined)
+            releaseCommand({ stderr, configuration });
+    }
+}
+function refusedCommand(reason) {
+    return {
+        ended: Promise.resolve({
+            stdout: "",
+            exitCode: null,
+            durationMilliseconds: 0,
+            reason,
+        }),
+        stop: () => undefined,
+    };
+}
+function deadlineRefusal(start, clock) {
+    if (start.deadline !== undefined && !isInstant(start.deadline))
+        return "invalid-command-deadline";
+    if (start.deadline !== undefined && Date.parse(start.deadline) <= clock.now())
+        return "command-wait-timeout";
+    return undefined;
+}
+export function startCommand(start, clock = commandClock) {
+    let reason = deadlineRefusal(start, clock);
+    if (reason !== undefined)
+        return refusedCommand(reason);
+    const began = performance.now();
+    const prepared = prepareCommand(start, clock);
+    if ("reason" in prepared)
+        return refusedCommand(prepared.reason);
+    const { child, stderr, configuration } = prepared;
+    const stdout = [];
+    const closed = Promise.withResolvers();
+    const fail = (failure = "command-exit") => {
+        reason ??= failure;
+        stopChild(child);
+        closed.resolve(child.exitCode);
+    };
+    boundedOutput(child.stdout, {
+        bytes: commandLimits.stdoutBytes,
+        receive: (chunk) => stdout.push(chunk),
+        fail,
+    });
+    boundedOutput(child.stderr, {
+        bytes: commandLimits.stderrBytes,
+        receive: (chunk) => {
+            if (!writeStderr(stderr, chunk))
+                fail("command-exit");
+        },
+        fail,
+    });
+    const remaining = start.deadline === undefined
+        ? Infinity
+        : Date.parse(start.deadline) - clock.now();
+    const execution = start.seconds * 1000;
+    const cancelTimer = clock.timer({
+        expired: () => {
+            fail(remaining <= execution ? "command-wait-timeout" : "command-timeout");
+        },
+        milliseconds: Math.min(execution, remaining),
+    });
+    child.stdin.on("error", () => undefined).end(start.payload);
+    child.once("error", () => undefined).once("close", closed.resolve);
+    return {
+        ended: closed.promise.then((exitCode) => {
+            cancelTimer();
+            releaseCommand({ stderr, configuration });
+            reason ??= deadlineRefusal(start, clock);
+            reason ??= exitCode === 0 ? undefined : "command-exit";
+            return {
+                stdout: Buffer.concat(stdout).toString("utf8"),
+                exitCode,
+                durationMilliseconds: Math.round(performance.now() - began),
+                ...(reason === undefined ? {} : { reason }),
+            };
+        }),
+        stop: fail,
+    };
+}

package/dist/src/daemon/command-worker.d.ts

@@ -0,0 +1,19 @@
+import type { LocalCommands } from "../command-settings.ts";
+import type { Delivery, DeliveryIntent } from "../broker/actor/delivery-intents.ts";
+import { type WorkerReport } from "./operation-journal.ts";
+import { type CommandClock } from "./command-process.ts";
+export interface CommandWorkerSettings {
+    readonly repository: string;
+    readonly journal: string;
+    readonly commands: LocalCommands;
+    readonly seconds?: number;
+    readonly commandClock?: CommandClock;
+    readonly document: (digest: string) => Promise<string | undefined>;
+    readonly report: (report: WorkerReport) => Promise<unknown>;
+}
+export interface CommandWorker extends Delivery {
+    readonly recover: (intents: readonly DeliveryIntent[]) => Promise<void>;
+    readonly stop: NonNullable<Delivery["stop"]>;
+    readonly close: () => Promise<void>;
+}
+export declare function commandWorker(settings: CommandWorkerSettings): CommandWorker;

package/dist/src/daemon/command-worker.js

@@ -0,0 +1,182 @@
+import { located, readRecord, writeRecord, replied, reportOf, } from "./operation-journal.js";
+import { startCommand, } from "./command-process.js";
+import { claimCommand, commandStarted, recordCommandResult, } from "./command-journal.js";
+function commandIntent(intent) {
+    return (intent.capability === "command" &&
+        /^run:[A-Za-z0-9-]{1,64}$/.test(intent.method ?? "") &&
+        intent.payload !== undefined &&
+        intent.target !== undefined &&
+        intent.leaseGeneration !== undefined);
+}
+async function reconciled(parts, call) {
+    const { intent, origin } = call;
+    const { key, folder } = located(parts.settings.journal, intent.operationId);
+    const result = await readRecord(folder, "result.json");
+    if (result !== undefined)
+        return replied({ receipt: key, worker: "command" }, result);
+    const inFlight = () => parts.active.has(key) || (origin !== "dispatch" && parts.pending.has(key));
+    if (inFlight())
+        return { kind: "accepted", receipt: key, worker: "command" };
+    if (!(await commandStarted(folder)))
+        return { kind: "absent" };
+    const started = await readRecord(folder, "started.json");
+    if (inFlight())
+        return { kind: "accepted", receipt: key, worker: "command" };
+    const duration = Date.now() - Date.parse(String(Reflect.get(Object(started), "startedAt")));
+    await recordCommandResult(folder, {
+        outcome: "failed",
+        reason: "command-interrupted",
+        exitCode: null,
+        durationMilliseconds: Number.isFinite(duration) ? Math.max(0, duration) : 0,
+        endedAt: new Date().toISOString(),
+    });
+    return replied({ receipt: key, worker: "command" }, await readRecord(folder, "result.json"));
+}
+function ending(end, stopping) {
+    const endedAt = new Date().toISOString();
+    if (stopping.requested)
+        return { outcome: "stopped", endedAt };
+    if (end.reason !== undefined)
+        return { outcome: "failed", reason: end.reason, endedAt };
+    return { outcome: "completed", result: end.stdout, endedAt };
+}
+function begun(parts, start) {
+    const { settings } = parts;
+    const { intent, key, folder, payload, command } = start;
+    const running = startCommand({
+        command,
+        directory: settings.repository,
+        folder,
+        payload,
+        runId: intent.runId,
+        operationId: intent.operationId,
+        seconds: settings.seconds ?? 60,
+        ...(intent.commandDeadline === undefined
+            ? {}
+            : { deadline: intent.commandDeadline }),
+    }, settings.commandClock);
+    const stopping = { requested: false };
+    const concluded = running.ended
+        .then(async (end) => {
+        const recorded = {
+            ...ending(end, stopping),
+            exitCode: end.exitCode,
+            durationMilliseconds: end.durationMilliseconds,
+        };
+        if (!(await recordCommandResult(folder, recorded)))
+            return;
+        await settings.report(reportOf({ intent, commandId: `command:${key}`, worker: "command" }, recorded));
+    })
+        .catch(() => undefined)
+        .finally(() => parts.active.delete(key));
+    parts.active.set(key, { command: running, concluded, stopping });
+}
+async function delivered(parts, call) {
+    const { intent, signal } = call;
+    const aborted = () => signal.aborted;
+    if (!commandIntent(intent))
+        return { kind: "failed", reason: "unsupported-effect" };
+    if (aborted())
+        return { kind: "failed", reason: "command-start-aborted" };
+    const known = await reconciled(parts, { intent, origin: "dispatch" });
+    if (known.kind !== "absent")
+        return known;
+    const { settings } = parts;
+    const name = intent.method.slice(4);
+    const command = Object.hasOwn(settings.commands, name)
+        ? settings.commands[name]
+        : undefined;
+    if (command === undefined)
+        return { kind: "failed", reason: "command-unknown" };
+    const payload = await settings.document(intent.payload);
+    if (payload === undefined)
+        return { kind: "failed", reason: "payload-not-available" };
+    if (aborted())
+        return { kind: "failed", reason: "command-start-aborted" };
+    const place = located(settings.journal, intent.operationId);
+    if (!(await claimCommand(place.folder)))
+        return reconciled(parts, { intent, origin: "dispatch" });
+    await writeRecord(place.folder, {
+        name: "started.json",
+        value: {
+            runId: intent.runId,
+            operationId: intent.operationId,
+            name,
+            startedAt: new Date().toISOString(),
+            deadline: intent.commandDeadline,
+        },
+    });
+    if (aborted())
+        return reconciled(parts, { intent, origin: "dispatch" });
+    begun(parts, { ...place, intent, payload, command });
+    return { receipt: place.key, worker: "command" };
+}
+async function recovered(parts, intents) {
+    for (const intent of intents) {
+        if (!commandIntent(intent))
+            continue;
+        await parts.pending.get(located(parts.settings.journal, intent.operationId).key)?.opening;
+        const result = await reconciled(parts, { intent });
+        if (result.kind === "completed" ||
+            result.kind === "failed" ||
+            result.kind === "stopped")
+            await parts.settings.report(reportOf({
+                intent,
+                commandId: `command:${located(parts.settings.journal, intent.operationId).key}`,
+                worker: "command",
+            }, {
+                ...result,
+                outcome: result.kind,
+                endedAt: new Date().toISOString(),
+            }));
+    }
+}
+export function commandWorker(settings) {
+    const parts = { settings, active: new Map(), pending: new Map() };
+    const closing = new AbortController();
+    const { pending } = parts;
+    return {
+        recover: (intents) => recovered(parts, intents),
+        deliver: (intent, signal) => {
+            const { key } = located(settings.journal, intent.operationId);
+            const known = pending.get(key);
+            if (known !== undefined)
+                return known.opening;
+            const stopping = new AbortController();
+            const opening = delivered(parts, {
+                intent,
+                signal: AbortSignal.any([signal, closing.signal, stopping.signal]),
+            }).finally(() => pending.delete(key));
+            pending.set(key, { opening, stopping });
+            return opening;
+        },
+        reconcile: (intent) => {
+            const { key } = located(settings.journal, intent.operationId);
+            return pending.has(key)
+                ? Promise.resolve({ kind: "accepted", receipt: key, worker: "command" })
+                : reconciled(parts, { intent });
+        },
+        stop: async (intent) => {
+            const { key } = located(settings.journal, intent.operationId);
+            const opening = pending.get(key);
+            opening?.stopping.abort();
+            await opening?.opening;
+            const active = parts.active.get(key);
+            if (active !== undefined)
+                active.stopping.requested = true;
+            active?.command.stop();
+            await active?.concluded;
+            return reconciled(parts, { intent });
+        },
+        close: async () => {
+            closing.abort();
+            await Promise.all([...pending.values()].map((entry) => entry.opening));
+            const active = [...parts.active.values()];
+            for (const running of active) {
+                running.stopping.requested = true;
+                running.command.stop();
+            }
+            await Promise.all(active.map((running) => running.concluded));
+        },
+    };
+}

package/dist/src/daemon/cursor-stream.d.ts

@@ -0,0 +1,4 @@
+export declare function cursorStream(): {
+    readonly line: (line: string) => readonly string[];
+    readonly result: (exitCode: number | null) => string;
+};

package/dist/src/daemon/cursor-stream.js

@@ -0,0 +1,154 @@
+import { record } from "../protocol.js";
+import { tokenCount, usageOf } from "../worker-usage.js";
+function text(value) {
+    return typeof value === "string" ? value : null;
+}
+function cursorUsage(value) {
+    const fields = record(value);
+    const counts = {
+        input_tokens: tokenCount(fields?.["inputTokens"]),
+        cache_read_input_tokens: tokenCount(fields?.["cacheReadTokens"]),
+        cache_creation_input_tokens: tokenCount(fields?.["cacheWriteTokens"]),
+        output_tokens: tokenCount(fields?.["outputTokens"]),
+    };
+    return Object.values(counts).includes(null) ? usageOf(null) : counts;
+}
+function received(event, state) {
+    const type = event["type"];
+    if (type === "system" || type === "result") {
+        state.model = text(event["model"]) ?? state.model;
+        state.sessionId = text(event["session_id"]) ?? state.sessionId;
+    }
+    if (type === "assistant")
+        state.report = assistantText(event) ?? state.report;
+    if (type === "error")
+        state.error = text(event["message"]) ?? "cursor-agent stream error";
+    if (type === "result")
+        state.terminal = event;
+}
+function assistantText(event) {
+    const content = record(event["message"])?.["content"];
+    if (!Array.isArray(content))
+        return null;
+    const blocks = content
+        .map((block) => record(block))
+        .filter((block) => block?.["type"] === "text");
+    if (blocks.length === 0)
+        return null;
+    return blocks.map((block) => text(block?.["text"]) ?? "").join("");
+}
+function functionToolCall(fields) {
+    const name = text(fields["name"]);
+    if (!name)
+        return undefined;
+    try {
+        const input = record(JSON.parse(String(fields["arguments"])));
+        return input === undefined ? undefined : { name, input };
+    }
+    catch {
+        return undefined;
+    }
+}
+function nativeToolCall(key, fields) {
+    if (!key.endsWith("ToolCall"))
+        return undefined;
+    const args = record(fields["args"]);
+    if (args === undefined)
+        return undefined;
+    return {
+        name: key === "shellToolCall"
+            ? "Bash"
+            : key.slice(0, 1).toUpperCase() + key.slice(1, -8),
+        input: { ...args, file_path: args["path"] },
+    };
+}
+function toolCall(key, value) {
+    const fields = record(value);
+    if (fields === undefined)
+        return undefined;
+    return key === "function"
+        ? functionToolCall(fields)
+        : nativeToolCall(key, fields);
+}
+function toolActivity(event, state) {
+    if (!["started", "completed"].includes(String(event["subtype"])))
+        return [];
+    const callId = text(event["call_id"]);
+    if (callId !== null && state.toolCalls.has(callId))
+        return [];
+    const tools = record(event["tool_call"]) ?? {};
+    const messages = Object.entries(tools).flatMap(([key, value]) => {
+        const tool = toolCall(key, value);
+        if (tool === undefined)
+            return [];
+        return [
+            JSON.stringify({
+                type: "assistant",
+                message: {
+                    content: [
+                        {
+                            type: "tool_use",
+                            id: callId,
+                            ...tool,
+                        },
+                    ],
+                },
+            }),
+        ];
+    });
+    if (callId !== null && messages.length > 0)
+        state.toolCalls.add(callId);
+    return messages;
+}
+function failure(state, exitCode) {
+    if (state.error !== null)
+        return state.error;
+    if (exitCode !== 0)
+        return `cursor-agent ended with the code ${String(exitCode)}`;
+    const terminal = state.terminal;
+    if (terminal === undefined)
+        return "cursor-agent stream ended before its result";
+    if (terminal["subtype"] === "success" && terminal["is_error"] === false)
+        return null;
+    return text(terminal["result"]) ?? "cursor-agent result did not succeed";
+}
+function result(state, exitCode) {
+    const error = failure(state, exitCode);
+    const success = error === null;
+    return JSON.stringify({
+        type: "result",
+        subtype: success ? "success" : "error_during_execution",
+        is_error: !success,
+        result: error ?? state.report,
+        model: state.model,
+        session_id: state.sessionId,
+        usage: cursorUsage(state.terminal?.["usage"]),
+    });
+}
+export function cursorStream() {
+    const state = {
+        model: null,
+        sessionId: null,
+        report: "",
+        error: null,
+        terminal: undefined,
+        toolCalls: new Set(),
+    };
+    return {
+        line: (line) => {
+            let value;
+            try {
+                value = JSON.parse(line);
+            }
+            catch {
+                return [];
+            }
+            const event = record(value) ?? {};
+            received(event, state);
+            if (event["type"] === "tool_call")
+                return toolActivity(event, state);
+            return event["type"] === "result" ? [] : [line];
+        },
+        result: (exitCode) => result(state, exitCode),
+    };
+}

package/dist/src/daemon/daemon-adapters.d.ts

@@ -0,0 +1,58 @@
+import { type StartDelivery, type WorkflowFunction } from "../host/start-delivery.ts";
+import type { WorkflowHandler } from "../host/world-host.ts";
+import type { HostPorts } from "../service/service.ts";
+import { workflowGenerations, type WorkflowBundleLoader } from "./workflow-generations.ts";
+import { type ClaudeWorker } from "./claude-worker.ts";
+import { type GithubWorker } from "./github-worker.ts";
+import { type CommandWorker } from "./command-worker.ts";
+import type { EffectDelivery } from "./delivery-router.ts";
+import type { TurnProfiles } from "../turn-profile.ts";
+import type { TokenPrices } from "../token-prices.ts";
+import type { LocalCommands } from "../command-settings.ts";
+import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { WorkflowRefusal } from "../workflow-refusals.ts";
+import type { HarnessSettings } from "../harness-defaults.ts";
+export interface DaemonSettings extends HarnessSettings {
+    readonly commands?: LocalCommands;
+    readonly commandSeconds?: number;
+    readonly fallbackModels?: readonly string[];
+    readonly tokenPrices?: TokenPrices;
+    readonly profiles?: TurnProfiles;
+    readonly directory: string;
+    readonly runtimeDirectory: string;
+    readonly repository: string;
+    readonly bundleDigest?: string;
+    readonly manifests: Readonly<Record<string, string | undefined>>;
+    readonly declarations?: readonly DeclaredWorkflow[];
+    readonly workflowRefusals?: readonly WorkflowRefusal[];
+    readonly workflows: Readonly<Record<string, WorkflowFunction | undefined>>;
+    readonly handler: WorkflowHandler;
+    readonly loadBundle?: WorkflowBundleLoader;
+    readonly command: readonly string[];
+    readonly ticketCommand?: readonly string[];
+    readonly nextCommand?: readonly string[];
+    readonly notifyCommand?: readonly string[];
+    readonly notifyMilliseconds?: number;
+    readonly queueMilliseconds?: number;
+    readonly scheduleClock?: () => number;
+    readonly schedules?: unknown;
+    readonly queueLanes?: number;
+    readonly resetQueue?: boolean;
+    readonly githubCommand?: readonly string[];
+    readonly requiredChecks?: readonly string[] | null;
+    readonly checkCommand?: readonly string[];
+    readonly privateTexts?: readonly string[];
+    readonly concurrency: number;
+}
+export interface Adapters {
+    readonly worker: ClaudeWorker;
+    readonly github: GithubWorker;
+    readonly runs: StartDelivery;
+    readonly handoff: EffectDelivery;
+    readonly commands: CommandWorker;
+}
+export declare function adapters(settings: DaemonSettings, options: {
+    readonly given: Promise<HostPorts>;
+    readonly generations: ReturnType<typeof workflowGenerations>;
+}): Adapters;
+export declare function recoverCommands(commands: CommandWorker, ports: HostPorts): Promise<void>;

package/dist/src/daemon/daemon-adapters.js

@@ -0,0 +1,102 @@
+import { join } from "node:path";
+import { startDelivery, } from "../host/start-delivery.js";
+import { cancelled, taggedRuns } from "../host/world-runs.js";
+import { workflowGenerations, } from "./workflow-generations.js";
+import { claudeWorker } from "./claude-worker.js";
+import { githubWorker } from "./github-worker.js";
+import { commandWorker } from "./command-worker.js";
+import { handoffWorker } from "./handoff-worker.js";
+function githubOf(settings, given) {
+    const { directory, repository, checkCommand } = settings;
+    return githubWorker({
+        document: async (digest) => (await given).document(digest),
+        report: async (effect) => (await given).reportEffect(effect),
+        operationStatus: async (operationId) => (await (await given).operation(operationId))?.status,
+        runStatus: async (runId) => (await (await given).state()).runs[runId]?.status,
+        repository,
+        gh: settings.githubCommand ?? ["gh"],
+        ...(settings.requiredChecks === undefined
+            ? {}
+            : { requiredChecks: settings.requiredChecks }),
+        worktrees: join(directory, "worktrees"),
+        journal: join(directory, "github"),
+        ...(settings.privateTexts === undefined
+            ? {}
+            : { privateTexts: settings.privateTexts }),
+        ...(checkCommand === undefined ? {} : { check: checkCommand }),
+    });
+}
+function workerHarness(settings) {
+    return {
+        worker: settings.worker ?? settings.harness?.name ?? "claude",
+        workerCommands: settings.workerCommands ?? {},
+        harness: settings.harness ?? {},
+        shell: settings.shell ?? process.env["SHELL"] ?? "/bin/sh",
+        command: settings.command,
+    };
+}
+export function adapters(settings, options) {
+    const { directory, repository } = settings;
+    const { given, generations } = options;
+    const document = async (digest) => (await given).document(digest);
+    const report = async (effect) => (await given).reportEffect(effect);
+    const worker = claudeWorker({
+        fallbackModels: settings.fallbackModels ?? [],
+        tokenPrices: settings.tokenPrices ?? {},
+        ...workerHarness(settings),
+        profiles: settings.profiles ?? {},
+        workflowId: async (runId) => (await given).workflow.id(runId),
+        repository,
+        document,
+        report,
+        worktrees: join(directory, "worktrees"),
+        journal: join(directory, "journal"),
+    });
+    const github = githubOf(settings, given);
+    const runs = startDelivery({
+        document,
+        runs: taggedRuns,
+        cancel: cancelled,
+        input: async (runId) => (await given).input(runId),
+        workflow: async ({ runId }) => (await generations.resolve(await (await given).workflow.manifest(runId)))
+            ?.workflow,
+    });
+    const handoff = handoffOf(settings, given);
+    const commands = commandWorker({
+        repository,
+        journal: join(directory, "commands"),
+        commands: settings.commands ?? {},
+        seconds: settings.commandSeconds ?? 60,
+        document,
+        report,
+    });
+    return { worker, github, runs, handoff, commands };
+}
+function handoffOf(settings, given) {
+    return handoffWorker({
+        runtimeDirectory: settings.runtimeDirectory,
+        journal: join(settings.directory, "handoffs"),
+        document: async (digest) => (await given).document(digest),
+        runStatus: async (runId) => (await (await given).state()).runs[runId]?.status,
+    });
+}
+export async function recoverCommands(commands, ports) {
+    const state = await ports.state();
+    const calls = [];
+    for (const run of Object.values(state.runs)) {
+        for (const operation of await ports.operations(run.runId)) {
+            if (operation.capability !== "command" ||
+                operation.leaseGeneration === null ||
+                !["running", "recovering", "dispatch-pending"].includes(operation.status))
+                continue;
+            calls.push({
+                ...operation,
+                kind: "protected-effect",
+                purpose: "reconcile",
+                workflow: run.workflow,
+                leaseGeneration: operation.leaseGeneration,
+            });
+        }
+    }
+    await commands.recover(calls);
+}