@botbotgo/agent-harness 0.0.40 → 0.0.42

package/dist/runtime/harness.js

@@ -5,7 +5,7 @@ import { AGENT_INTERRUPT_SENTINEL_PREFIX, AgentRuntimeAdapter, RuntimeOperationT
 import { createResourceBackendResolver, createResourceToolResolver } from "../resource/resource.js";
 import { EventBus } from "./event-bus.js";
 import { PolicyEngine } from "./policy-engine.js";
-import { getRoutingDefaultAgentId, getRoutingRules, getRoutingSystemPrompt, isModelRoutingEnabled, matchRoutingRule, } from "../workspace/support/workspace-ref-utils.js";
+import { getConcurrencyConfig, getRecoveryConfig, getRoutingDefaultAgentId, getRoutingRules, getRoutingSystemPrompt, isModelRoutingEnabled, matchRoutingRule, } from "../workspace/support/workspace-ref-utils.js";
 import { createHarnessEvent, createPendingApproval, heuristicRoute, inferRoutingBindings, renderRuntimeFailure, renderToolFailure, } from "./support/harness-support.js";
 import { createCheckpointerForConfig, createStoreForConfig } from "./support/runtime-factories.js";
 import { resolveCompiledEmbeddingModel, resolveCompiledEmbeddingModelRef } from "./support/embedding-models.js";
@@ -35,6 +35,10 @@ export class AgentHarnessRuntime {
     unregisterThreadMemorySync;
     resolvedRuntimeAdapterOptions;
     checkpointMaintenance;
+    recoveryConfig;
+    concurrencyConfig;
+    activeRunSlots = 0;
+    pendingRunSlots = [];
     listHostBindings() {
         return inferRoutingBindings(this.workspace).hostBindings;
     }
@@ -137,6 +141,10 @@ export class AgentHarnessRuntime {
                         return existing;
                     }
                     const resolvedConfig = binding.harnessRuntime.checkpointer ?? { kind: "FileCheckpointer", path: "checkpoints.json" };
+                    if (typeof resolvedConfig === "boolean") {
+                        this.checkpointers.set(key, resolvedConfig);
+                        return resolvedConfig;
+                    }
                     const saver = createCheckpointerForConfig(resolvedConfig, binding.harnessRuntime.runRoot);
                     this.checkpointers.set(key, saver);
                     return saver;
@@ -157,10 +165,13 @@ export class AgentHarnessRuntime {
         this.checkpointMaintenance = checkpointMaintenanceConfig
             ? new CheckpointMaintenanceLoop(discoverCheckpointMaintenanceTargets(workspace), checkpointMaintenanceConfig)
             : null;
+        this.recoveryConfig = getRecoveryConfig(workspace.refs);
+        this.concurrencyConfig = getConcurrencyConfig(workspace.refs);
     }
     async initialize() {
         await this.persistence.initialize();
         await this.checkpointMaintenance?.start();
+        await this.recoverStartupRuns();
     }
     subscribe(listener) {
         return this.eventBus.subscribe(listener);
@@ -338,6 +349,11 @@ export class AgentHarnessRuntime {
         const history = await this.persistence.listThreadMessages(threadId);
         return history.filter((message) => message.runId !== runId);
     }
+    async loadRunInput(threadId, runId) {
+        const history = await this.persistence.listThreadMessages(threadId, 100);
+        const userTurn = history.find((message) => message.runId === runId && message.role === "user");
+        return userTurn?.content ?? "";
+    }
     async appendAssistantMessage(threadId, runId, content) {
         if (!content) {
             return;
@@ -349,9 +365,31 @@ export class AgentHarnessRuntime {
             createdAt: new Date().toISOString(),
         });
     }
-    async invokeWithHistory(binding, input, threadId, runId, resumePayload) {
+    async invokeWithHistory(binding, input, threadId, runId, resumePayload, options = {}) {
         const priorHistory = await this.loadPriorHistory(threadId, runId);
-        return this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory);
+        return this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory, options);
+    }
+    checkpointRefForState(threadId, runId, state) {
+        return state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null;
+    }
+    async finalizeContinuedRun(threadId, runId, input, actual, options) {
+        let approval;
+        await this.appendAssistantMessage(threadId, runId, actual.output);
+        const checkpointRef = this.checkpointRefForState(threadId, runId, actual.state);
+        await this.setRunStateAndEmit(threadId, runId, options.stateSequence, actual.state, {
+            previousState: options.previousState,
+            checkpointRef,
+        });
+        if (actual.state === "waiting_for_approval" && options.approvalSequence) {
+            approval = (await this.requestApprovalAndEmit(threadId, runId, input, actual.interruptContent, checkpointRef, options.approvalSequence)).approval;
+        }
+        return {
+            ...actual,
+            threadId,
+            runId,
+            approvalId: approval?.approvalId ?? actual.approvalId,
+            pendingActionId: approval?.pendingActionId ?? actual.pendingActionId,
+        };
     }
     async emitOutputDeltaAndCreateItem(threadId, runId, agentId, content) {
         await this.emit(threadId, runId, 3, "output.delta", {
@@ -438,6 +476,28 @@ export class AgentHarnessRuntime {
         }
         await listener(value);
     }
+    async acquireRunSlot() {
+        const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
+        if (!maxConcurrentRuns) {
+            return () => undefined;
+        }
+        if (this.activeRunSlots >= maxConcurrentRuns) {
+            await new Promise((resolve) => {
+                this.pendingRunSlots.push(resolve);
+            });
+        }
+        this.activeRunSlots += 1;
+        let released = false;
+        return () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
+            const next = this.pendingRunSlots.shift();
+            next?.();
+        };
+    }
     async dispatchRunListeners(stream, listeners) {
         let latestEvent;
         let output = "";
@@ -499,275 +559,304 @@ export class AgentHarnessRuntime {
         if (options.listeners) {
             return this.dispatchRunListeners(this.streamEvents(options), options.listeners);
         }
-        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-        const binding = this.workspace.bindings.get(selectedAgentId);
-        if (!binding) {
-            throw new Error(`Unknown agent ${selectedAgentId}`);
-        }
-        const policyDecision = this.policyEngine.evaluate(binding);
-        if (!policyDecision.allowed) {
-            throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
-        }
-        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-        await this.emitRunCreated(threadId, runId, {
-            agentId: binding.agent.id,
-            requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
-            selectedAgentId,
-            executionMode: binding.agent.executionMode,
-        });
+        const releaseRunSlot = await this.acquireRunSlot();
         try {
-            const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
-            let approval;
-            await this.appendAssistantMessage(threadId, runId, actual.output);
-            const checkpointRef = actual.state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null;
-            await this.setRunStateAndEmit(threadId, runId, 3, actual.state, {
-                previousState: null,
-                checkpointRef,
-            });
-            if (actual.state === "waiting_for_approval") {
-                approval = (await this.requestApprovalAndEmit(threadId, runId, options.input, actual.interruptContent, checkpointRef, 4)).approval;
+            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+            const binding = this.workspace.bindings.get(selectedAgentId);
+            if (!binding) {
+                throw new Error(`Unknown agent ${selectedAgentId}`);
             }
-            return {
-                ...actual,
-                threadId,
-                runId,
-                agentId: selectedAgentId,
-                approvalId: approval?.approvalId ?? actual.approvalId,
-                pendingActionId: approval?.pendingActionId ?? actual.pendingActionId,
-            };
-        }
-        catch (error) {
-            await this.emitSyntheticFallback(threadId, runId, selectedAgentId, error);
-            await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                previousState: null,
-                error: error instanceof Error ? error.message : String(error),
-            });
-            return {
-                threadId,
-                runId,
-                agentId: selectedAgentId,
-                state: "failed",
-                output: renderRuntimeFailure(error),
-            };
-        }
-    }
-    async *streamEvents(options) {
-        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-        const binding = this.workspace.bindings.get(selectedAgentId);
-        if (!binding) {
-            const result = await this.run(options);
-            for (const line of result.output.split("\n")) {
-                yield {
-                    type: "content",
-                    threadId: result.threadId,
-                    runId: result.runId,
-                    agentId: result.agentId ?? selectedAgentId,
-                    content: `${line}\n`,
-                };
+            const policyDecision = this.policyEngine.evaluate(binding);
+            if (!policyDecision.allowed) {
+                throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
             }
-            return;
-        }
-        let emitted = false;
-        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-        yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
-                agentId: selectedAgentId,
+            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+            await this.emitRunCreated(threadId, runId, {
+                agentId: binding.agent.id,
                 requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
                 selectedAgentId,
-                input: options.input,
-                state: "running",
-            }) };
-        try {
-            const priorHistory = await this.loadPriorHistory(threadId, runId);
-            let assistantOutput = "";
-            const toolErrors = [];
-            for await (const chunk of this.runtimeAdapter.stream(binding, options.input, threadId, priorHistory)) {
-                if (chunk) {
-                    const normalizedChunk = typeof chunk === "string"
-                        ? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
-                            ? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
-                            : { kind: "content", content: chunk }
-                        : chunk;
-                    if (normalizedChunk.kind === "interrupt") {
-                        const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
-                        const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 4, "waiting_for_approval", {
-                            previousState: null,
-                            checkpointRef,
-                        });
-                        const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 5);
-                        yield {
-                            type: "event",
-                            event: waitingEvent,
-                        };
-                        yield {
-                            type: "event",
-                            event: approvalRequest.event,
-                        };
-                        return;
-                    }
-                    if (normalizedChunk.kind === "reasoning") {
-                        await this.emit(threadId, runId, 3, "reasoning.delta", {
-                            content: normalizedChunk.content,
-                        });
-                        yield {
-                            type: "reasoning",
-                            threadId,
-                            runId,
-                            agentId: selectedAgentId,
-                            content: normalizedChunk.content,
-                        };
-                        continue;
-                    }
-                    if (normalizedChunk.kind === "step") {
-                        yield {
-                            type: "step",
-                            threadId,
-                            runId,
-                            agentId: selectedAgentId,
-                            content: normalizedChunk.content,
-                        };
-                        continue;
-                    }
-                    if (normalizedChunk.kind === "tool-result") {
-                        if (normalizedChunk.isError) {
-                            toolErrors.push(renderToolFailure(normalizedChunk.toolName, normalizedChunk.output));
-                        }
-                        yield {
-                            type: "tool-result",
-                            threadId,
-                            runId,
-                            agentId: selectedAgentId,
-                            toolName: normalizedChunk.toolName,
-                            output: normalizedChunk.output,
-                            isError: normalizedChunk.isError,
-                        };
-                        continue;
-                    }
-                    emitted = true;
-                    assistantOutput += normalizedChunk.content;
-                    yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, normalizedChunk.content);
-                }
-            }
-            if (!assistantOutput && toolErrors.length > 0) {
-                assistantOutput = toolErrors.join("\n\n");
-                emitted = true;
-                yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, assistantOutput);
-            }
-            if (!assistantOutput) {
-                const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
-                if (actual.output) {
-                    assistantOutput = actual.output;
-                    emitted = true;
-                    yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
-                }
-            }
-            await this.appendAssistantMessage(threadId, runId, assistantOutput);
-            yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "completed", {
+                executionMode: binding.agent.executionMode,
+            });
+            try {
+                const actual = await this.invokeWithHistory(binding, options.input, threadId, runId, undefined, {
+                    context: options.context,
+                    state: options.state,
+                    files: options.files,
+                });
+                const finalized = await this.finalizeContinuedRun(threadId, runId, options.input, actual, {
                     previousState: null,
-                }) };
-            return;
-        }
-        catch (error) {
-            if (emitted) {
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                        previousState: null,
-                        error: error instanceof Error ? error.message : String(error),
-                    }) };
-                return;
+                    stateSequence: 3,
+                    approvalSequence: 4,
+                });
+                return {
+                    ...finalized,
+                    agentId: selectedAgentId,
+                };
             }
-            if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                        previousState: null,
-                        error: error.message,
-                    }) };
-                yield {
-                    type: "content",
+            catch (error) {
+                await this.emitSyntheticFallback(threadId, runId, selectedAgentId, error);
+                await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
+                    previousState: null,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                return {
                     threadId,
                     runId,
                     agentId: selectedAgentId,
-                    content: renderRuntimeFailure(error),
+                    state: "failed",
+                    output: renderRuntimeFailure(error),
                 };
+            }
+        }
+        finally {
+            releaseRunSlot();
+        }
+    }
+    async *streamEvents(options) {
+        const releaseRunSlot = await this.acquireRunSlot();
+        try {
+            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+            const binding = this.workspace.bindings.get(selectedAgentId);
+            if (!binding) {
+                const result = await this.run(options);
+                for (const line of result.output.split("\n")) {
+                    yield {
+                        type: "content",
+                        threadId: result.threadId,
+                        runId: result.runId,
+                        agentId: result.agentId ?? selectedAgentId,
+                        content: `${line}\n`,
+                    };
+                }
                 return;
             }
+            let emitted = false;
+            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+            yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
+                    agentId: selectedAgentId,
+                    requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
+                    selectedAgentId,
+                    input: options.input,
+                    state: "running",
+                }) };
             try {
-                const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
-                await this.appendAssistantMessage(threadId, runId, actual.output);
-                if (actual.output) {
-                    yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
+                const priorHistory = await this.loadPriorHistory(threadId, runId);
+                let assistantOutput = "";
+                const toolErrors = [];
+                for await (const chunk of this.runtimeAdapter.stream(binding, options.input, threadId, priorHistory, {
+                    context: options.context,
+                    state: options.state,
+                    files: options.files,
+                })) {
+                    if (chunk) {
+                        const normalizedChunk = typeof chunk === "string"
+                            ? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
+                                ? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
+                                : { kind: "content", content: chunk }
+                            : chunk;
+                        if (normalizedChunk.kind === "interrupt") {
+                            const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
+                            const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 4, "waiting_for_approval", {
+                                previousState: null,
+                                checkpointRef,
+                            });
+                            const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 5);
+                            yield {
+                                type: "event",
+                                event: waitingEvent,
+                            };
+                            yield {
+                                type: "event",
+                                event: approvalRequest.event,
+                            };
+                            return;
+                        }
+                        if (normalizedChunk.kind === "reasoning") {
+                            await this.emit(threadId, runId, 3, "reasoning.delta", {
+                                content: normalizedChunk.content,
+                            });
+                            yield {
+                                type: "reasoning",
+                                threadId,
+                                runId,
+                                agentId: selectedAgentId,
+                                content: normalizedChunk.content,
+                            };
+                            continue;
+                        }
+                        if (normalizedChunk.kind === "step") {
+                            yield {
+                                type: "step",
+                                threadId,
+                                runId,
+                                agentId: selectedAgentId,
+                                content: normalizedChunk.content,
+                            };
+                            continue;
+                        }
+                        if (normalizedChunk.kind === "tool-result") {
+                            if (normalizedChunk.isError) {
+                                toolErrors.push(renderToolFailure(normalizedChunk.toolName, normalizedChunk.output));
+                            }
+                            yield {
+                                type: "tool-result",
+                                threadId,
+                                runId,
+                                agentId: selectedAgentId,
+                                toolName: normalizedChunk.toolName,
+                                output: normalizedChunk.output,
+                                isError: normalizedChunk.isError,
+                            };
+                            continue;
+                        }
+                        emitted = true;
+                        assistantOutput += normalizedChunk.content;
+                        yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, normalizedChunk.content);
+                    }
                 }
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, actual.state, {
+                if (!assistantOutput && toolErrors.length > 0) {
+                    assistantOutput = toolErrors.join("\n\n");
+                    emitted = true;
+                    yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, assistantOutput);
+                }
+                if (!assistantOutput) {
+                    const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
+                    if (actual.output) {
+                        assistantOutput = actual.output;
+                        emitted = true;
+                        yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
+                    }
+                }
+                await this.appendAssistantMessage(threadId, runId, assistantOutput);
+                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "completed", {
                         previousState: null,
                     }) };
                 return;
             }
-            catch (invokeError) {
-                await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                        previousState: null,
-                        error: invokeError instanceof Error ? invokeError.message : String(invokeError),
-                    }) };
-                yield {
-                    type: "content",
-                    threadId,
-                    runId,
-                    agentId: selectedAgentId,
-                    content: renderRuntimeFailure(invokeError),
-                };
-                return;
+            catch (error) {
+                if (emitted) {
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
+                            previousState: null,
+                            error: error instanceof Error ? error.message : String(error),
+                        }) };
+                    return;
+                }
+                if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
+                            previousState: null,
+                            error: error.message,
+                        }) };
+                    yield {
+                        type: "content",
+                        threadId,
+                        runId,
+                        agentId: selectedAgentId,
+                        content: renderRuntimeFailure(error),
+                    };
+                    return;
+                }
+                try {
+                    const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
+                    await this.appendAssistantMessage(threadId, runId, actual.output);
+                    if (actual.output) {
+                        yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
+                    }
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, actual.state, {
+                            previousState: null,
+                        }) };
+                    return;
+                }
+                catch (invokeError) {
+                    await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
+                            previousState: null,
+                            error: invokeError instanceof Error ? invokeError.message : String(invokeError),
+                        }) };
+                    yield {
+                        type: "content",
+                        threadId,
+                        runId,
+                        agentId: selectedAgentId,
+                        content: renderRuntimeFailure(invokeError),
+                    };
+                    return;
+                }
             }
         }
+        finally {
+            releaseRunSlot();
+        }
     }
     async resume(options) {
-        const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
-        const thread = options.threadId
-            ? await this.getSession(options.threadId)
-            : approvalById
-                ? await this.getSession(approvalById.threadId)
-                : null;
-        if (!thread) {
-            throw new Error("resume requires either threadId or approvalId");
+        const releaseRunSlot = await this.acquireRunSlot();
+        try {
+            const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
+            const thread = options.threadId
+                ? await this.getSession(options.threadId)
+                : approvalById
+                    ? await this.getSession(approvalById.threadId)
+                    : null;
+            if (!thread) {
+                throw new Error("resume requires either threadId or approvalId");
+            }
+            const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
+            const threadId = approval.threadId;
+            const runId = approval.runId;
+            const binding = this.workspace.bindings.get(thread.agentId);
+            if (!binding) {
+                throw new Error(`Unknown agent ${thread.agentId}`);
+            }
+            await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
+            await this.persistence.saveRecoveryIntent(threadId, runId, {
+                kind: "approval-decision",
+                savedAt: new Date().toISOString(),
+                checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
+                resumePayload: options.decision === "edit" && options.editedInput
+                    ? { decision: "edit", editedInput: options.editedInput }
+                    : (options.decision ?? "approve"),
+                attempts: 0,
+            });
+            await this.emit(threadId, runId, 5, "run.resumed", {
+                resumeKind: "cross-restart",
+                checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
+                state: "resuming",
+                approvalId: approval.approvalId,
+                pendingActionId: approval.pendingActionId,
+            });
+            await this.persistence.resolveApproval(threadId, runId, approval.approvalId, options.decision === "reject" ? "rejected" : options.decision === "edit" ? "edited" : "approved");
+            await this.emit(threadId, runId, 6, "approval.resolved", {
+                approvalId: approval.approvalId,
+                pendingActionId: approval.pendingActionId,
+                decision: options.decision ?? "approve",
+                toolName: approval.toolName,
+            });
+            const history = await this.persistence.listThreadMessages(threadId);
+            const priorHistory = history.filter((message) => message.runId !== runId);
+            const runInput = await this.loadRunInput(threadId, runId);
+            const resumeDecision = options.decision === "edit" && options.editedInput
+                ? { decision: "edit", editedInput: options.editedInput }
+                : (options.decision ?? "approve");
+            try {
+                const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumeDecision, priorHistory);
+                await this.persistence.clearRecoveryIntent(threadId, runId);
+                const finalized = await this.finalizeContinuedRun(threadId, runId, runInput, actual, {
+                    previousState: "resuming",
+                    stateSequence: 7,
+                    approvalSequence: 8,
+                });
+                return {
+                    ...finalized,
+                    approvalId: finalized.approvalId ?? approval.approvalId,
+                    pendingActionId: finalized.pendingActionId ?? approval.pendingActionId,
+                };
+            }
+            catch (error) {
+                throw error;
+            }
         }
-        const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
-        const threadId = approval.threadId;
-        const runId = approval.runId;
-        const binding = this.workspace.bindings.get(thread.agentId);
-        if (!binding) {
-            throw new Error(`Unknown agent ${thread.agentId}`);
+        finally {
+            releaseRunSlot();
         }
-        await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
-        await this.emit(threadId, runId, 5, "run.resumed", {
-            resumeKind: "cross-restart",
-            checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
-            state: "resuming",
-            approvalId: approval.approvalId,
-            pendingActionId: approval.pendingActionId,
-        });
-        await this.persistence.resolveApproval(threadId, runId, approval.approvalId, options.decision === "reject" ? "rejected" : options.decision === "edit" ? "edited" : "approved");
-        await this.emit(threadId, runId, 6, "approval.resolved", {
-            approvalId: approval.approvalId,
-            pendingActionId: approval.pendingActionId,
-            decision: options.decision ?? "approve",
-            toolName: approval.toolName,
-        });
-        const history = await this.persistence.listThreadMessages(threadId);
-        const priorHistory = history.filter((message) => message.runId !== runId);
-        const resumeDecision = options.decision === "edit" && options.editedInput
-            ? { decision: "edit", editedInput: options.editedInput }
-            : (options.decision ?? "approve");
-        const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumeDecision, priorHistory);
-        await this.appendAssistantMessage(threadId, runId, actual.output);
-        await this.persistence.setRunState(threadId, runId, actual.state, actual.state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null);
-        await this.emit(threadId, runId, 7, "run.state.changed", {
-            previousState: "resuming",
-            state: actual.state,
-            checkpointRef: actual.state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null,
-        });
-        return {
-            ...actual,
-            threadId,
-            runId,
-            approvalId: approval.approvalId,
-            pendingActionId: approval.pendingActionId,
-        };
     }
     async restartConversation(options) {
         const thread = await this.getSession(options.threadId);
@@ -802,5 +891,62 @@ export class AgentHarnessRuntime {
     async stop() {
         await this.close();
     }
+    async recoverStartupRuns() {
+        if (!this.recoveryConfig.enabled || !this.recoveryConfig.resumeResumingRunsOnStartup) {
+            return;
+        }
+        const threads = await this.persistence.listSessions();
+        for (const thread of threads) {
+            if (thread.status !== "resuming") {
+                continue;
+            }
+            const binding = this.workspace.bindings.get(thread.agentId);
+            if (!binding) {
+                continue;
+            }
+            const recoveryIntent = await this.persistence.getRecoveryIntent(thread.threadId, thread.latestRunId);
+            if (!recoveryIntent || recoveryIntent.kind !== "approval-decision") {
+                continue;
+            }
+            if (recoveryIntent.attempts >= this.recoveryConfig.maxRecoveryAttempts) {
+                await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
+                await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
+                continue;
+            }
+            await this.persistence.saveRecoveryIntent(thread.threadId, thread.latestRunId, {
+                ...recoveryIntent,
+                attempts: recoveryIntent.attempts + 1,
+            });
+            await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
+                resumeKind: "startup-recovery",
+                checkpointRef: recoveryIntent.checkpointRef,
+                state: "resuming",
+            });
+            const history = await this.persistence.listThreadMessages(thread.threadId);
+            const priorHistory = history.filter((message) => message.runId !== thread.latestRunId);
+            const runInput = await this.loadRunInput(thread.threadId, thread.latestRunId);
+            try {
+                const actual = await this.runtimeAdapter.invoke(binding, "", thread.threadId, thread.latestRunId, recoveryIntent.resumePayload, priorHistory);
+                await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
+                await this.finalizeContinuedRun(thread.threadId, thread.latestRunId, runInput, actual, {
+                    previousState: "resuming",
+                    stateSequence: 101,
+                    approvalSequence: 102,
+                });
+            }
+            catch (error) {
+                if (recoveryIntent.attempts + 1 >= this.recoveryConfig.maxRecoveryAttempts) {
+                    await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
+                    await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
+                    await this.emit(thread.threadId, thread.latestRunId, 101, "run.state.changed", {
+                        previousState: "resuming",
+                        state: "failed",
+                        checkpointRef: recoveryIntent.checkpointRef,
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+        }
+    }
 }
 export { AgentHarnessRuntime as AgentHarness };