@botbotgo/agent-harness 0.0.79 → 0.0.81

package/dist/runtime/harness.d.ts

@@ -1,4 +1,4 @@
-import type { ApprovalRecord, HarnessEvent, HarnessStreamItem, RuntimeHealthSnapshot, MessageContent, RunRecord, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, RunSummary, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
+import type { ApprovalRecord, CancelOptions, HarnessEvent, HarnessStreamItem, RuntimeHealthSnapshot, MessageContent, RunRecord, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, RunSummary, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
 import { type ToolMcpServerOptions } from "../mcp.js";
 import { type InventoryAgentRecord, type InventorySkillRecord } from "./inventory.js";
 import type { RequirementAssessmentOptions } from "./skill-requirements.js";
@@ -27,6 +27,7 @@ export declare class AgentHarnessRuntime {
     private readonly healthMonitor;
     private readonly recoveryConfig;
     private readonly concurrencyConfig;
+    private readonly workerId;
     private activeRunSlots;
     private readonly pendingRunSlots;
     private runtimeEventSequence;
@@ -84,6 +85,9 @@ export declare class AgentHarnessRuntime {
     private loadPriorHistory;
     private loadRunInput;
     private appendAssistantMessage;
+    private getRunCancellation;
+    private expirePendingApprovals;
+    private finalizeCancelledRun;
     private invokeWithHistory;
     private buildPersistedRunRequest;
     private executeQueuedRun;
@@ -100,6 +104,7 @@ export declare class AgentHarnessRuntime {
     private isDecisionRun;
     private notifyListener;
     private acquireRunSlot;
+    private dropPendingRunSlot;
     private dispatchRunListeners;
     run(options: RunOptions): Promise<RunResult>;
     streamEvents(options: RunStartOptions): AsyncGenerator<HarnessStreamItem>;
@@ -110,6 +115,9 @@ export declare class AgentHarnessRuntime {
     }>;
     close(): Promise<void>;
     stop(): Promise<void>;
+    cancelRun(options: CancelOptions): Promise<RunResult>;
     private recoverStartupRuns;
+    private reclaimExpiredClaimedRuns;
+    private isStaleRunningRun;
 }
 export { AgentHarnessRuntime as AgentHarness };

package/dist/runtime/harness.js

@@ -44,6 +44,7 @@ export class AgentHarnessRuntime {
     healthMonitor;
     recoveryConfig;
     concurrencyConfig;
+    workerId = `worker-${createPersistentId()}`;
     activeRunSlots = 0;
     pendingRunSlots = [];
     runtimeEventSequence = 0;
@@ -98,6 +99,13 @@ export class AgentHarnessRuntime {
     }
     async resolveSelectedAgentId(input, requestedAgentId, threadId) {
         if (!requestedAgentId || requestedAgentId === AUTO_AGENT_ID) {
+            if (threadId) {
+                const thread = await this.getSession(threadId);
+                const threadBinding = thread ? this.workspace.bindings.get(thread.agentId) : undefined;
+                if (thread?.agentId && threadBinding?.harnessRuntime.hostFacing !== false) {
+                    return thread.agentId;
+                }
+            }
             return this.routeAgent(input, { threadId });
         }
         return requestedAgentId;
@@ -480,6 +488,46 @@ export class AgentHarnessRuntime {
             createdAt: new Date().toISOString(),
         });
     }
+    async getRunCancellation(runId) {
+        const control = await this.persistence.getRunControl(runId);
+        return {
+            requested: control?.cancelRequested === true,
+            ...(control?.cancelReason ? { reason: control.cancelReason } : {}),
+        };
+    }
+    async expirePendingApprovals(threadId, runId) {
+        const approvals = await this.persistence.getRunApprovals(threadId, runId);
+        for (const approval of approvals) {
+            if (approval.status !== "pending") {
+                continue;
+            }
+            await this.persistence.resolveApproval(threadId, runId, approval.approvalId, "expired");
+            await this.emit(threadId, runId, 6, "approval.resolved", {
+                approvalId: approval.approvalId,
+                pendingActionId: approval.pendingActionId,
+                decision: "cancel",
+                toolName: approval.toolName,
+            });
+        }
+    }
+    async finalizeCancelledRun(threadId, runId, previousState, reason) {
+        await this.expirePendingApprovals(threadId, runId);
+        await this.persistence.releaseRunClaim(runId);
+        await this.persistence.clearRunCancel(runId);
+        await this.persistence.clearRunRequest(threadId, runId);
+        await this.setRunStateAndEmit(threadId, runId, 104, "cancelled", {
+            previousState,
+            ...(reason ? { error: reason } : {}),
+        });
+        const runMeta = await this.persistence.getRunMeta(threadId, runId);
+        return {
+            threadId,
+            runId,
+            agentId: runMeta.agentId,
+            state: "cancelled",
+            output: reason ? `cancelled: ${reason}` : "cancelled",
+        };
+    }
     async invokeWithHistory(binding, input, threadId, runId, resumePayload, options = {}) {
         const priorHistory = await this.loadPriorHistory(threadId, runId);
         const startedAt = Date.now();
@@ -514,6 +562,20 @@ export class AgentHarnessRuntime {
     }
     async executeQueuedRun(binding, input, threadId, runId, agentId, options = {}) {
         const previousState = options.previousState ?? "running";
+        const currentRun = await this.persistence.getRun(runId);
+        if (currentRun?.state === "cancelled") {
+            return {
+                threadId,
+                runId,
+                agentId,
+                state: "cancelled",
+                output: "cancelled",
+            };
+        }
+        const cancellation = await this.getRunCancellation(runId);
+        if (cancellation.requested) {
+            return this.finalizeCancelledRun(threadId, runId, previousState, cancellation.reason);
+        }
         if (previousState === "queued") {
             await this.emit(threadId, runId, 101, "run.dequeued", {
                 queuePosition: 0,
@@ -531,6 +593,10 @@ export class AgentHarnessRuntime {
                 state: options.state,
                 files: options.files,
             });
+            const cancelledAfterInvoke = await this.getRunCancellation(runId);
+            if (cancelledAfterInvoke.requested) {
+                return this.finalizeCancelledRun(threadId, runId, previousState === "queued" ? "running" : previousState, cancelledAfterInvoke.reason);
+            }
             const finalized = await this.finalizeContinuedRun(threadId, runId, input, actual, {
                 previousState: previousState === "queued" ? "running" : previousState,
                 stateSequence: options.stateSequence ?? 103,
@@ -676,21 +742,64 @@ export class AgentHarnessRuntime {
         await listener(value);
     }
     async acquireRunSlot(threadId, runId, activeState = "running") {
+        if (threadId && runId) {
+            await this.persistence.enqueueRun({ threadId, runId });
+        }
+        let stopHeartbeat = () => undefined;
+        const beginLease = async () => {
+            if (!threadId || !runId) {
+                return;
+            }
+            const claimedAt = new Date().toISOString();
+            await this.persistence.claimQueuedRun({
+                threadId,
+                runId,
+                workerId: this.workerId,
+                claimedAt,
+                leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
+            });
+            if (this.concurrencyConfig.heartbeatIntervalMs <= 0) {
+                return;
+            }
+            const timer = setInterval(() => {
+                void this.persistence.renewRunLease({
+                    runId,
+                    workerId: this.workerId,
+                    heartbeatAt: new Date().toISOString(),
+                    leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
+                });
+            }, this.concurrencyConfig.heartbeatIntervalMs);
+            timer.unref?.();
+            stopHeartbeat = () => {
+                clearInterval(timer);
+            };
+        };
+        const releaseLease = async () => {
+            stopHeartbeat();
+            if (runId) {
+                await this.persistence.releaseRunClaim(runId);
+            }
+        };
         const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
         if (!maxConcurrentRuns) {
-            return () => undefined;
+            await beginLease();
+            return async () => {
+                await releaseLease();
+            };
         }
         if (this.activeRunSlots < maxConcurrentRuns) {
             this.activeRunSlots += 1;
+            await beginLease();
             let released = false;
-            return () => {
+            return async () => {
                 if (released) {
                     return;
                 }
                 released = true;
+                await releaseLease();
                 this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
                 const next = this.pendingRunSlots.shift();
-                void next?.();
+                void next?.activate();
             };
         }
         if (threadId && runId) {
@@ -704,38 +813,57 @@ export class AgentHarnessRuntime {
                 maxConcurrentRuns,
             });
         }
-        await new Promise((resolve, reject) => {
-            this.pendingRunSlots.push(async () => {
-                try {
-                    this.activeRunSlots += 1;
-                    if (threadId && runId) {
-                        await this.emit(threadId, runId, 4, "run.dequeued", {
-                            queuePosition: 0,
-                            activeRunCount: this.activeRunSlots,
-                            maxConcurrentRuns,
-                        });
-                        await this.setRunStateAndEmit(threadId, runId, 5, activeState, {
-                            previousState: "queued",
-                        });
+        const slotAcquisition = await new Promise((resolve, reject) => {
+            this.pendingRunSlots.push({ runId, activate: async () => {
+                    try {
+                        const currentRun = runId ? await this.persistence.getRun(runId) : null;
+                        if (currentRun?.state === "cancelled") {
+                            resolve("abort");
+                            return;
+                        }
+                        this.activeRunSlots += 1;
+                        if (threadId && runId) {
+                            await this.emit(threadId, runId, 4, "run.dequeued", {
+                                queuePosition: 0,
+                                activeRunCount: this.activeRunSlots,
+                                maxConcurrentRuns,
+                            });
+                            await this.setRunStateAndEmit(threadId, runId, 5, activeState, {
+                                previousState: "queued",
+                            });
+                            await beginLease();
+                        }
+                        resolve("activate");
                     }
-                    resolve();
-                }
-                catch (error) {
-                    reject(error);
-                }
-            });
+                    catch (error) {
+                        reject(error);
+                    }
+                }, abort: () => resolve("abort") });
         });
+        if (slotAcquisition === "abort") {
+            return async () => undefined;
+        }
         let released = false;
-        return () => {
+        return async () => {
             if (released) {
                 return;
             }
             released = true;
+            await releaseLease();
             this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
             const next = this.pendingRunSlots.shift();
-            void next?.();
+            void next?.activate();
         };
     }
+    dropPendingRunSlot(runId) {
+        const index = this.pendingRunSlots.findIndex((entry) => entry.runId === runId);
+        if (index < 0) {
+            return false;
+        }
+        const [entry] = this.pendingRunSlots.splice(index, 1);
+        entry?.abort();
+        return true;
+    }
     async dispatchRunListeners(stream, listeners) {
         let latestEvent;
         let latestResult;
@@ -843,7 +971,7 @@ export class AgentHarnessRuntime {
             });
         }
         finally {
-            releaseRunSlot();
+            await releaseRunSlot();
         }
     }
     async *streamEvents(options) {
@@ -1104,7 +1232,7 @@ export class AgentHarnessRuntime {
         }
         finally {
             await this.persistence.clearRunRequest(threadId, runId);
-            releaseRunSlot();
+            await releaseRunSlot();
         }
     }
     async resume(options) {
@@ -1125,6 +1253,10 @@ export class AgentHarnessRuntime {
             throw new Error(`Unknown agent ${thread.agentId}`);
         }
         const resumePayload = this.buildResumePayload(binding, approval, options);
+        const cancellation = await this.getRunCancellation(runId);
+        if (cancellation.requested) {
+            return this.finalizeCancelledRun(threadId, runId, thread.status, cancellation.reason);
+        }
         await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
         const releaseRunSlot = await this.acquireRunSlot(threadId, runId, "resuming");
         try {
@@ -1156,6 +1288,10 @@ export class AgentHarnessRuntime {
             try {
                 const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
                 this.healthMonitor.recordLlmSuccess(Date.now() - startedAt);
+                const cancelledAfterInvoke = await this.getRunCancellation(runId);
+                if (cancelledAfterInvoke.requested) {
+                    return this.finalizeCancelledRun(threadId, runId, "resuming", cancelledAfterInvoke.reason);
+                }
                 await this.persistence.clearRecoveryIntent(threadId, runId);
                 const finalized = await this.finalizeContinuedRun(threadId, runId, runInput, actual, {
                     previousState: "resuming",
@@ -1174,7 +1310,7 @@ export class AgentHarnessRuntime {
             }
         }
         finally {
-            releaseRunSlot();
+            await releaseRunSlot();
         }
     }
     buildResumePayload(binding, approval, options) {
@@ -1249,10 +1385,44 @@ export class AgentHarnessRuntime {
     async stop() {
         await this.close();
     }
+    async cancelRun(options) {
+        const run = await this.persistence.getRun(options.runId);
+        if (!run) {
+            throw new Error(`Unknown run ${options.runId}`);
+        }
+        if (this.isTerminalRunState(run.state)) {
+            return {
+                threadId: run.threadId,
+                runId: run.runId,
+                agentId: run.agentId,
+                state: run.state,
+                output: run.state,
+            };
+        }
+        await this.persistence.requestRunCancel(run.runId, options.reason);
+        if (run.state === "queued" || run.state === "waiting_for_approval" || run.state === "claimed") {
+            if (run.state === "queued") {
+                this.dropPendingRunSlot(run.runId);
+            }
+            return this.finalizeCancelledRun(run.threadId, run.runId, run.state, options.reason);
+        }
+        await this.setRunStateAndEmit(run.threadId, run.runId, 103, "cancelling", {
+            previousState: run.state,
+            ...(options.reason ? { error: options.reason } : {}),
+        });
+        return {
+            threadId: run.threadId,
+            runId: run.runId,
+            agentId: run.agentId,
+            state: "cancelling",
+            output: options.reason ? `cancelling: ${options.reason}` : "cancelling",
+        };
+    }
     async recoverStartupRuns() {
         if (!this.recoveryConfig.enabled) {
             return;
         }
+        await this.reclaimExpiredClaimedRuns();
         const threads = await this.persistence.listSessions();
         for (const thread of threads) {
             if (thread.status === "queued") {
@@ -1269,7 +1439,7 @@ export class AgentHarnessRuntime {
                     });
                     continue;
                 }
-                const releaseRunSlot = await this.acquireRunSlot();
+                const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId);
                 try {
                     await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
                         context: request.invocation?.context,
@@ -1281,21 +1451,38 @@ export class AgentHarnessRuntime {
                     });
                 }
                 finally {
-                    releaseRunSlot();
+                    await releaseRunSlot();
                 }
                 continue;
             }
             if (thread.status === "running") {
+                const isStale = await this.isStaleRunningRun(thread);
+                if (!isStale) {
+                    continue;
+                }
                 const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
                 const binding = this.workspace.bindings.get(runMeta.agentId);
-                if (!binding || !this.supportsRunningReplay(binding)) {
+                if (!binding) {
+                    continue;
+                }
+                if (!this.supportsRunningReplay(binding)) {
+                    await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
+                        previousState: "running",
+                        error: "stale running run cannot be replayed safely",
+                    });
+                    await this.persistence.releaseRunClaim(thread.latestRunId);
                     continue;
                 }
                 const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
                 if (!request) {
+                    await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
+                        previousState: "running",
+                        error: "missing persisted run request for stale running run recovery",
+                    });
+                    await this.persistence.releaseRunClaim(thread.latestRunId);
                     continue;
                 }
-                const releaseRunSlot = await this.acquireRunSlot();
+                const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running");
                 try {
                     await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
                         resumeKind: "startup-running-recovery",
@@ -1311,7 +1498,7 @@ export class AgentHarnessRuntime {
                     });
                 }
                 finally {
-                    releaseRunSlot();
+                    await releaseRunSlot();
                 }
                 continue;
             }
@@ -1369,5 +1556,49 @@ export class AgentHarnessRuntime {
             }
         }
     }
+    async reclaimExpiredClaimedRuns(nowIso = new Date().toISOString()) {
+        const expiredClaims = await this.persistence.listExpiredClaimedRuns(nowIso);
+        for (const claim of expiredClaims) {
+            const thread = await this.persistence.getSession(claim.threadId);
+            if (!thread) {
+                await this.persistence.releaseRunClaim(claim.runId);
+                continue;
+            }
+            const lifecycle = await this.persistence.getRunLifecycle(claim.threadId, claim.runId);
+            if (lifecycle.state === "claimed") {
+                await this.persistence.enqueueRun({
+                    threadId: claim.threadId,
+                    runId: claim.runId,
+                    priority: claim.priority,
+                    queueKey: claim.queueKey,
+                    availableAt: nowIso,
+                });
+                await this.setRunStateAndEmit(claim.threadId, claim.runId, 99, "queued", {
+                    previousState: "claimed",
+                });
+                await this.emit(claim.threadId, claim.runId, 100, "run.queued", {
+                    queuePosition: 0,
+                    activeRunCount: this.activeRunSlots,
+                    maxConcurrentRuns: this.concurrencyConfig.maxConcurrentRuns,
+                    recoveredOnStartup: true,
+                    reclaimReason: "expired-lease",
+                });
+                continue;
+            }
+            await this.persistence.releaseRunClaim(claim.runId);
+        }
+    }
+    async isStaleRunningRun(thread, nowMs = Date.now()) {
+        const control = await this.persistence.getRunControl(thread.latestRunId);
+        const heartbeatAt = control?.heartbeatAt;
+        if (!heartbeatAt) {
+            return true;
+        }
+        const heartbeatAtMs = Date.parse(heartbeatAt);
+        if (!Number.isFinite(heartbeatAtMs)) {
+            return true;
+        }
+        return nowMs - heartbeatAtMs >= this.concurrencyConfig.heartbeatTimeoutMs;
+    }
 }
 export { AgentHarnessRuntime as AgentHarness };

package/dist/runtime/health-monitor.js

@@ -422,7 +422,7 @@ export class HealthMonitor {
     }
     countStuckRuns(runs, nowMs) {
         return runs.filter((run) => {
-            if (!["running", "resuming", "queued"].includes(run.state)) {
+            if (!["claimed", "running", "resuming", "queued", "cancelling"].includes(run.state)) {
                 return false;
             }
             const updatedAtMs = Date.parse(run.updatedAt);

package/dist/runtime/runtime-record-maintenance.js

@@ -91,9 +91,11 @@ export async function maintainSqliteRuntimeRecords(dbPath, config, nowMs = Date.
             "DELETE FROM artifacts WHERE thread_id = ?",
             "DELETE FROM approvals WHERE thread_id = ?",
             "DELETE FROM events WHERE thread_id = ?",
+            "DELETE FROM run_queue WHERE thread_id = ?",
             "DELETE FROM run_requests WHERE thread_id = ?",
             "DELETE FROM recovery_intents WHERE thread_id = ?",
             "DELETE FROM thread_messages WHERE thread_id = ?",
+            "DELETE FROM run_control WHERE run_id IN (SELECT run_id FROM runs WHERE thread_id = ?)",
             "DELETE FROM runs WHERE thread_id = ?",
             "DELETE FROM threads WHERE thread_id = ?",
         ].map((sql) => ({ sql, args: [threadId] })), "write");