@botbotgo/agent-harness 0.0.45 → 0.0.47

package/dist/persistence/file-store.js

@@ -7,6 +7,18 @@ export class FilePersistence {
     constructor(runRoot) {
         this.runRoot = runRoot;
     }
+    threadIndexPath(threadId) {
+        return path.join(this.runRoot, "indexes", "threads", `${threadId}.json`);
+    }
+    runIndexPath(runId) {
+        return path.join(this.runRoot, "indexes", "runs", `${runId}.json`);
+    }
+    approvalIndexPath(approvalId) {
+        return path.join(this.runRoot, "indexes", "approvals", `${approvalId}.json`);
+    }
+    delegationIndexPath(delegationId) {
+        return path.join(this.runRoot, "indexes", "delegations", `${delegationId}.json`);
+    }
     async initialize() {
         await Promise.all([
             "indexes/threads",
@@ -266,6 +278,48 @@ export class FilePersistence {
     async getRunLifecycle(threadId, runId) {
         return readJson(path.join(this.runDir(threadId, runId), "lifecycle.json"));
     }
+    async deleteThread(threadId) {
+        const threadDir = this.threadDir(threadId);
+        const threadIndexPath = this.threadIndexPath(threadId);
+        if (!(await fileExists(threadDir)) && !(await fileExists(threadIndexPath))) {
+            return false;
+        }
+        const [runIndexes, approvals, delegations] = await Promise.all([
+            this.listRunIndexes(),
+            this.listApprovals(),
+            this.listDelegations(),
+        ]);
+        await Promise.all([
+            ...runIndexes
+                .filter((record) => record.threadId === threadId)
+                .map((record) => rm(this.runIndexPath(record.runId), { force: true })),
+            ...approvals
+                .filter((record) => record.threadId === threadId)
+                .map((record) => rm(this.approvalIndexPath(record.approvalId), { force: true })),
+            ...delegations
+                .filter((record) => record.threadId === threadId)
+                .map((record) => rm(this.delegationIndexPath(record.delegationId), { force: true })),
+            rm(threadIndexPath, { force: true }),
+            rm(threadDir, { recursive: true, force: true }),
+        ]);
+        return true;
+    }
+    async saveRunRequest(threadId, runId, request) {
+        await writeJson(path.join(this.runDir(threadId, runId), "request.json"), request);
+    }
+    async getRunRequest(threadId, runId) {
+        const requestPath = path.join(this.runDir(threadId, runId), "request.json");
+        if (!(await fileExists(requestPath))) {
+            return null;
+        }
+        return readJson(requestPath);
+    }
+    async clearRunRequest(threadId, runId) {
+        const requestPath = path.join(this.runDir(threadId, runId), "request.json");
+        if (await fileExists(requestPath)) {
+            await rm(requestPath, { force: true });
+        }
+    }
     async listDelegations() {
         const delegationsDir = path.join(this.runRoot, "indexes", "delegations");
         if (!(await fileExists(delegationsDir))) {

package/dist/resource/resource-impl.d.ts

@@ -6,6 +6,7 @@ export type ResourceToolInfo = {
     backendOperation: string;
     name: string;
     description: string;
+    retryable?: boolean;
     hitl?: {
         enabled: boolean;
         allow: Array<"approve" | "edit" | "reject">;

package/dist/runtime/harness.d.ts

@@ -26,6 +26,7 @@ export declare class AgentHarnessRuntime {
     private readonly pendingRunSlots;
     private toPublicApprovalRecord;
     private normalizeInvocationEnvelope;
+    private isTerminalRunState;
     private listHostBindings;
     private defaultRunRoot;
     private heuristicRoute;
@@ -40,6 +41,7 @@ export declare class AgentHarnessRuntime {
     private getBinding;
     private listAgentTools;
     private resolveAgentTools;
+    private supportsRunningReplay;
     listThreads(filter?: {
         agentId?: string;
     }): Promise<ThreadSummary[]>;
@@ -51,6 +53,8 @@ export declare class AgentHarnessRuntime {
         runId?: string;
     }): Promise<ApprovalRecord[]>;
     getApproval(approvalId: string): Promise<ApprovalRecord | null>;
+    private deleteThreadCheckpoints;
+    deleteThread(threadId: string): Promise<boolean>;
     createToolMcpServer(options: ToolMcpServerOptions): Promise<import("@modelcontextprotocol/sdk/server/mcp.js").McpServer>;
     serveToolsOverStdio(options: ToolMcpServerOptions): Promise<import("@modelcontextprotocol/sdk/server/mcp.js").McpServer>;
     routeAgent(input: MessageContent, options?: {
@@ -62,6 +66,8 @@ export declare class AgentHarnessRuntime {
     private loadRunInput;
     private appendAssistantMessage;
     private invokeWithHistory;
+    private buildPersistedRunRequest;
+    private executeQueuedRun;
     private checkpointRefForState;
     private finalizeContinuedRun;
     private emitOutputDeltaAndCreateItem;

package/dist/runtime/harness.js

@@ -53,6 +53,9 @@ export class AgentHarnessRuntime {
             invocation,
         };
     }
+    isTerminalRunState(state) {
+        return state === "completed" || state === "failed";
+    }
     listHostBindings() {
         return inferRoutingBindings(this.workspace).hostBindings;
     }
@@ -213,6 +216,10 @@ export class AgentHarnessRuntime {
             resolvedTool: resolvedTools[index],
         }));
     }
+    supportsRunningReplay(binding) {
+        const tools = getBindingPrimaryTools(binding);
+        return tools.every((tool) => tool.retryable === true);
+    }
     async listThreads(filter) {
         const threadSummaries = await this.persistence.listSessions();
         if (!filter?.agentId) {
@@ -279,6 +286,39 @@ export class AgentHarnessRuntime {
         const approval = await this.persistence.getApproval(approvalId);
         return approval ? this.toPublicApprovalRecord(approval) : null;
     }
+    async deleteThreadCheckpoints(threadId) {
+        const resolver = this.resolvedRuntimeAdapterOptions.checkpointerResolver;
+        if (!resolver) {
+            return;
+        }
+        const seen = new Set();
+        for (const binding of this.workspace.bindings.values()) {
+            const saver = resolver(binding);
+            if (!saver || seen.has(saver)) {
+                continue;
+            }
+            seen.add(saver);
+            const maybeDeleteThread = saver.deleteThread;
+            if (typeof maybeDeleteThread === "function") {
+                await maybeDeleteThread.call(saver, threadId);
+            }
+        }
+    }
+    async deleteThread(threadId) {
+        const thread = await this.getThread(threadId);
+        if (!thread) {
+            return false;
+        }
+        const activeRun = thread.runs.find((run) => !this.isTerminalRunState(run.state));
+        if (activeRun) {
+            throw new Error(`Cannot delete thread ${threadId} while run ${activeRun.runId} is ${activeRun.state}`);
+        }
+        const deleted = await this.persistence.deleteThread(threadId);
+        if (deleted) {
+            await this.deleteThreadCheckpoints(threadId);
+        }
+        return deleted;
+    }
     async createToolMcpServer(options) {
         const tools = this.resolveAgentTools(options.agentId).map(({ compiledTool, resolvedTool }) => ({
             compiledTool,
@@ -385,6 +425,72 @@ export class AgentHarnessRuntime {
         const priorHistory = await this.loadPriorHistory(threadId, runId);
         return this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory, options);
     }
+    buildPersistedRunRequest(input, invocation) {
+        const envelope = invocation.invocation ?? {
+            ...(invocation.context ? { context: invocation.context } : {}),
+            ...(invocation.state ? { inputs: invocation.state } : {}),
+            ...(invocation.files ? { attachments: invocation.files } : {}),
+        };
+        return {
+            input: normalizeMessageContent(input),
+            invocation: envelope && Object.keys(envelope).length > 0
+                ? {
+                    ...(envelope.context ? { context: envelope.context } : {}),
+                    ...(envelope.inputs ? { inputs: envelope.inputs } : {}),
+                    ...(envelope.attachments ? { attachments: envelope.attachments } : {}),
+                    ...(envelope.capabilities ? { capabilities: envelope.capabilities } : {}),
+                }
+                : undefined,
+            savedAt: new Date().toISOString(),
+        };
+    }
+    async executeQueuedRun(binding, input, threadId, runId, agentId, options = {}) {
+        const previousState = options.previousState ?? "running";
+        if (previousState === "queued") {
+            await this.emit(threadId, runId, 101, "run.dequeued", {
+                queuePosition: 0,
+                activeRunCount: this.activeRunSlots,
+                maxConcurrentRuns: this.concurrencyConfig.maxConcurrentRuns,
+                recoveredOnStartup: true,
+            });
+            await this.setRunStateAndEmit(threadId, runId, 102, "running", {
+                previousState: "queued",
+            });
+        }
+        try {
+            const actual = await this.invokeWithHistory(binding, input, threadId, runId, undefined, {
+                context: options.context,
+                state: options.state,
+                files: options.files,
+            });
+            const finalized = await this.finalizeContinuedRun(threadId, runId, input, actual, {
+                previousState: previousState === "queued" ? "running" : previousState,
+                stateSequence: options.stateSequence ?? 103,
+                approvalSequence: options.approvalSequence ?? 104,
+            });
+            return {
+                ...finalized,
+                agentId,
+            };
+        }
+        catch (error) {
+            await this.emitSyntheticFallback(threadId, runId, agentId, error, 103);
+            await this.setRunStateAndEmit(threadId, runId, 104, "failed", {
+                previousState: previousState === "queued" ? "running" : previousState,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return {
+                threadId,
+                runId,
+                agentId,
+                state: "failed",
+                output: renderRuntimeFailure(error),
+            };
+        }
+        finally {
+            await this.persistence.clearRunRequest(threadId, runId);
+        }
+    }
     checkpointRefForState(threadId, runId, state) {
         return state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null;
     }
@@ -492,17 +598,56 @@ export class AgentHarnessRuntime {
         }
         await listener(value);
     }
-    async acquireRunSlot() {
+    async acquireRunSlot(threadId, runId, activeState = "running") {
         const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
         if (!maxConcurrentRuns) {
             return () => undefined;
         }
-        if (this.activeRunSlots >= maxConcurrentRuns) {
-            await new Promise((resolve) => {
-                this.pendingRunSlots.push(resolve);
+        if (this.activeRunSlots < maxConcurrentRuns) {
+            this.activeRunSlots += 1;
+            let released = false;
+            return () => {
+                if (released) {
+                    return;
+                }
+                released = true;
+                this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
+                const next = this.pendingRunSlots.shift();
+                void next?.();
+            };
+        }
+        if (threadId && runId) {
+            const queuePosition = this.pendingRunSlots.length + 1;
+            await this.setRunStateAndEmit(threadId, runId, 2, "queued", {
+                previousState: activeState,
+            });
+            await this.emit(threadId, runId, 3, "run.queued", {
+                queuePosition,
+                activeRunCount: this.activeRunSlots,
+                maxConcurrentRuns,
             });
         }
-        this.activeRunSlots += 1;
+        await new Promise((resolve, reject) => {
+            this.pendingRunSlots.push(async () => {
+                try {
+                    this.activeRunSlots += 1;
+                    if (threadId && runId) {
+                        await this.emit(threadId, runId, 4, "run.dequeued", {
+                            queuePosition: 0,
+                            activeRunCount: this.activeRunSlots,
+                            maxConcurrentRuns,
+                        });
+                        await this.setRunStateAndEmit(threadId, runId, 5, activeState, {
+                            previousState: "queued",
+                        });
+                    }
+                    resolve();
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+        });
         let released = false;
         return () => {
             if (released) {
@@ -511,7 +656,7 @@ export class AgentHarnessRuntime {
             released = true;
             this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
             const next = this.pendingRunSlots.shift();
-            next?.();
+            void next?.();
         };
     }
     async dispatchRunListeners(stream, listeners) {
@@ -587,88 +732,68 @@ export class AgentHarnessRuntime {
         if (options.listeners) {
             return this.dispatchRunListeners(this.streamEvents(options), options.listeners);
         }
-        const releaseRunSlot = await this.acquireRunSlot();
+        const invocation = this.normalizeInvocationEnvelope(options);
+        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+        const binding = this.workspace.bindings.get(selectedAgentId);
+        if (!binding) {
+            throw new Error(`Unknown agent ${selectedAgentId}`);
+        }
+        const policyDecision = this.policyEngine.evaluate(binding);
+        if (!policyDecision.allowed) {
+            throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
+        }
+        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+        await this.persistence.saveRunRequest(threadId, runId, this.buildPersistedRunRequest(options.input, invocation));
+        await this.emitRunCreated(threadId, runId, {
+            agentId: binding.agent.id,
+            requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
+            selectedAgentId,
+            executionMode: getBindingAdapterKind(binding),
+        });
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId);
         try {
-            const invocation = this.normalizeInvocationEnvelope(options);
-            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-            const binding = this.workspace.bindings.get(selectedAgentId);
-            if (!binding) {
-                throw new Error(`Unknown agent ${selectedAgentId}`);
-            }
-            const policyDecision = this.policyEngine.evaluate(binding);
-            if (!policyDecision.allowed) {
-                throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
-            }
-            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-            await this.emitRunCreated(threadId, runId, {
-                agentId: binding.agent.id,
-                requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
-                selectedAgentId,
-                executionMode: getBindingAdapterKind(binding),
+            return await this.executeQueuedRun(binding, options.input, threadId, runId, selectedAgentId, {
+                context: invocation.context,
+                state: invocation.state,
+                files: invocation.files,
+                previousState: "running",
+                stateSequence: 6,
+                approvalSequence: 7,
             });
-            try {
-                const actual = await this.invokeWithHistory(binding, options.input, threadId, runId, undefined, {
-                    context: invocation.context,
-                    state: invocation.state,
-                    files: invocation.files,
-                });
-                const finalized = await this.finalizeContinuedRun(threadId, runId, options.input, actual, {
-                    previousState: null,
-                    stateSequence: 3,
-                    approvalSequence: 4,
-                });
-                return {
-                    ...finalized,
-                    agentId: selectedAgentId,
-                };
-            }
-            catch (error) {
-                await this.emitSyntheticFallback(threadId, runId, selectedAgentId, error);
-                await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                    previousState: null,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-                return {
-                    threadId,
-                    runId,
-                    agentId: selectedAgentId,
-                    state: "failed",
-                    output: renderRuntimeFailure(error),
-                };
-            }
         }
         finally {
             releaseRunSlot();
         }
     }
     async *streamEvents(options) {
-        const releaseRunSlot = await this.acquireRunSlot();
-        try {
-            const invocation = this.normalizeInvocationEnvelope(options);
-            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-            const binding = this.workspace.bindings.get(selectedAgentId);
-            if (!binding) {
-                const result = await this.run(options);
-                for (const line of result.output.split("\n")) {
-                    yield {
-                        type: "content",
-                        threadId: result.threadId,
-                        runId: result.runId,
-                        agentId: result.agentId ?? selectedAgentId,
-                        content: `${line}\n`,
-                    };
-                }
-                return;
+        const invocation = this.normalizeInvocationEnvelope(options);
+        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+        const binding = this.workspace.bindings.get(selectedAgentId);
+        if (!binding) {
+            const result = await this.run(options);
+            for (const line of result.output.split("\n")) {
+                yield {
+                    type: "content",
+                    threadId: result.threadId,
+                    runId: result.runId,
+                    agentId: result.agentId ?? selectedAgentId,
+                    content: `${line}\n`,
+                };
             }
-            let emitted = false;
-            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-            yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
-                    agentId: selectedAgentId,
-                    requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
-                    selectedAgentId,
-                    input: options.input,
-                    state: "running",
-                }) };
+            return;
+        }
+        let emitted = false;
+        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+        await this.persistence.saveRunRequest(threadId, runId, this.buildPersistedRunRequest(options.input, invocation));
+        yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
+                agentId: selectedAgentId,
+                requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
+                selectedAgentId,
+                input: options.input,
+                state: "running",
+            }) };
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId);
+        try {
             try {
                 const priorHistory = await this.loadPriorHistory(threadId, runId);
                 let assistantOutput = "";
@@ -686,11 +811,11 @@ export class AgentHarnessRuntime {
                             : chunk;
                         if (normalizedChunk.kind === "interrupt") {
                             const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
-                            const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 4, "waiting_for_approval", {
-                                previousState: null,
+                            const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 6, "waiting_for_approval", {
+                                previousState: "running",
                                 checkpointRef,
                             });
-                            const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 5);
+                            const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 7);
                             yield {
                                 type: "event",
                                 event: waitingEvent,
@@ -783,22 +908,22 @@ export class AgentHarnessRuntime {
                         finalMessageText: assistantOutput,
                     },
                 };
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "completed", {
-                        previousState: null,
+                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
+                        previousState: "running",
                     }) };
                 return;
             }
             catch (error) {
                 if (emitted) {
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: error instanceof Error ? error.message : String(error),
                         }) };
                     return;
                 }
                 if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: error.message,
                         }) };
                     yield {
@@ -836,15 +961,15 @@ export class AgentHarnessRuntime {
                             agentId: selectedAgentId,
                         },
                     };
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, actual.state, {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
+                            previousState: "running",
                         }) };
                     return;
                 }
                 catch (invokeError) {
                     await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: invokeError instanceof Error ? invokeError.message : String(invokeError),
                         }) };
                     yield {
@@ -870,29 +995,30 @@ export class AgentHarnessRuntime {
             }
         }
         finally {
+            await this.persistence.clearRunRequest(threadId, runId);
             releaseRunSlot();
         }
     }
     async resume(options) {
-        const releaseRunSlot = await this.acquireRunSlot();
+        const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
+        const thread = options.threadId
+            ? await this.getSession(options.threadId)
+            : approvalById
+                ? await this.getSession(approvalById.threadId)
+                : null;
+        if (!thread) {
+            throw new Error("resume requires either threadId or approvalId");
+        }
+        const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
+        const threadId = approval.threadId;
+        const runId = approval.runId;
+        const binding = this.workspace.bindings.get(thread.agentId);
+        if (!binding) {
+            throw new Error(`Unknown agent ${thread.agentId}`);
+        }
+        await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId, "resuming");
         try {
-            const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
-            const thread = options.threadId
-                ? await this.getSession(options.threadId)
-                : approvalById
-                    ? await this.getSession(approvalById.threadId)
-                    : null;
-            if (!thread) {
-                throw new Error("resume requires either threadId or approvalId");
-            }
-            const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
-            const threadId = approval.threadId;
-            const runId = approval.runId;
-            const binding = this.workspace.bindings.get(thread.agentId);
-            if (!binding) {
-                throw new Error(`Unknown agent ${thread.agentId}`);
-            }
-            await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
             await this.persistence.saveRecoveryIntent(threadId, runId, {
                 kind: "approval-decision",
                 savedAt: new Date().toISOString(),
@@ -978,12 +1104,72 @@ export class AgentHarnessRuntime {
         await this.close();
     }
     async recoverStartupRuns() {
-        if (!this.recoveryConfig.enabled || !this.recoveryConfig.resumeResumingRunsOnStartup) {
+        if (!this.recoveryConfig.enabled) {
             return;
         }
         const threads = await this.persistence.listSessions();
         for (const thread of threads) {
-            if (thread.status !== "resuming") {
+            if (thread.status === "queued") {
+                const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
+                const binding = this.workspace.bindings.get(runMeta.agentId);
+                if (!binding) {
+                    continue;
+                }
+                const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
+                if (!request) {
+                    await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
+                        previousState: "queued",
+                        error: "missing persisted run request for queued run recovery",
+                    });
+                    continue;
+                }
+                const releaseRunSlot = await this.acquireRunSlot();
+                try {
+                    await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
+                        context: request.invocation?.context,
+                        state: request.invocation?.inputs,
+                        files: request.invocation?.attachments,
+                        previousState: "queued",
+                        stateSequence: 103,
+                        approvalSequence: 104,
+                    });
+                }
+                finally {
+                    releaseRunSlot();
+                }
+                continue;
+            }
+            if (thread.status === "running") {
+                const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
+                const binding = this.workspace.bindings.get(runMeta.agentId);
+                if (!binding || !this.supportsRunningReplay(binding)) {
+                    continue;
+                }
+                const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
+                if (!request) {
+                    continue;
+                }
+                const releaseRunSlot = await this.acquireRunSlot();
+                try {
+                    await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
+                        resumeKind: "startup-running-recovery",
+                        state: "running",
+                    });
+                    await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
+                        context: request.invocation?.context,
+                        state: request.invocation?.inputs,
+                        files: request.invocation?.attachments,
+                        previousState: "running",
+                        stateSequence: 103,
+                        approvalSequence: 104,
+                    });
+                }
+                finally {
+                    releaseRunSlot();
+                }
+                continue;
+            }
+            if (thread.status !== "resuming" || !this.recoveryConfig.resumeResumingRunsOnStartup) {
                 continue;
             }
             const binding = this.workspace.bindings.get(thread.agentId);

package/dist/runtime/thread-memory-sync.js

@@ -44,6 +44,8 @@ function renderOpenApprovalsMarkdown(approvals) {
 }
 const THREAD_MEMORY_EVENT_TYPES = new Set([
     "run.state.changed",
+    "run.queued",
+    "run.dequeued",
     "approval.resolved",
     "approval.requested",
 ]);

package/dist/tool-modules.d.ts

@@ -9,6 +9,7 @@ export type LoadedToolModule = {
     invoke: (input: unknown, context?: Record<string, unknown>) => Promise<unknown> | unknown;
     schema: SchemaLike;
     description: string;
+    retryable?: boolean;
 };
 export declare function isSupportedToolModulePath(filePath: string): boolean;
 export declare function discoverAnnotatedFunctionNames(sourceText: string): string[];