@botbotgo/agent-harness 0.0.100 → 0.0.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/package-version.d.ts +1 -1
  2. package/dist/package-version.js +1 -1
  3. package/dist/persistence/sqlite-run-context-store.d.ts +22 -0
  4. package/dist/persistence/sqlite-run-context-store.js +64 -0
  5. package/dist/persistence/sqlite-run-queue-store.d.ts +41 -0
  6. package/dist/persistence/sqlite-run-queue-store.js +120 -0
  7. package/dist/persistence/sqlite-store.d.ts +2 -2
  8. package/dist/persistence/sqlite-store.js +31 -117
  9. package/dist/resource/mcp-tool-support.d.ts +21 -0
  10. package/dist/resource/mcp-tool-support.js +173 -0
  11. package/dist/resource/resource-impl.d.ts +1 -18
  12. package/dist/resource/resource-impl.js +79 -240
  13. package/dist/runtime/adapter/invoke-runtime.d.ts +22 -0
  14. package/dist/runtime/adapter/invoke-runtime.js +18 -0
  15. package/dist/runtime/adapter/stream-runtime.d.ts +46 -0
  16. package/dist/runtime/adapter/stream-runtime.js +93 -0
  17. package/dist/runtime/agent-runtime-adapter.d.ts +1 -12
  18. package/dist/runtime/agent-runtime-adapter.js +122 -312
  19. package/dist/runtime/harness/run/recovery.d.ts +42 -0
  20. package/dist/runtime/harness/run/recovery.js +139 -0
  21. package/dist/runtime/harness/run/run-operations.d.ts +50 -0
  22. package/dist/runtime/harness/run/run-operations.js +113 -0
  23. package/dist/runtime/harness/run/run-slot-acquisition.d.ts +64 -0
  24. package/dist/runtime/harness/run/run-slot-acquisition.js +157 -0
  25. package/dist/runtime/harness/run/stream-run.d.ts +53 -0
  26. package/dist/runtime/harness/run/stream-run.js +304 -0
  27. package/dist/runtime/harness.d.ts +2 -17
  28. package/dist/runtime/harness.js +157 -773
  29. package/dist/runtime/support/runtime-factories.js +2 -2
  30. package/dist/workspace/object-loader.d.ts +1 -8
  31. package/dist/workspace/object-loader.js +43 -275
  32. package/dist/workspace/yaml-object-reader.d.ts +15 -0
  33. package/dist/workspace/yaml-object-reader.js +202 -0
  34. package/package.json +1 -1
  35. package/dist/runtime/checkpoint-maintenance.d.ts +0 -1
  36. package/dist/runtime/checkpoint-maintenance.js +0 -1
  37. package/dist/runtime/file-checkpoint-saver.d.ts +0 -1
  38. package/dist/runtime/file-checkpoint-saver.js +0 -1
  39. package/dist/runtime/sqlite-maintained-checkpoint-saver.d.ts +0 -1
  40. package/dist/runtime/sqlite-maintained-checkpoint-saver.js +0 -1
@@ -1,26 +1,27 @@
1
1
  import { AUTO_AGENT_ID } from "../contracts/types.js";
2
2
  import { SqlitePersistence } from "../persistence/sqlite-store.js";
3
3
  import { createPersistentId } from "../utils/id.js";
4
- import { AGENT_INTERRUPT_SENTINEL_PREFIX, AgentRuntimeAdapter, RuntimeOperationTimeoutError } from "./agent-runtime-adapter.js";
5
- import { normalizeUpstreamRuntimeEvent } from "./parsing/stream-event-parsing.js";
4
+ import { AgentRuntimeAdapter } from "./agent-runtime-adapter.js";
6
5
  import { createResourceBackendResolver, createResourceToolResolver } from "../resource/resource.js";
7
6
  import { EventBus } from "./harness/events/event-bus.js";
8
7
  import { PolicyEngine } from "./harness/system/policy-engine.js";
9
8
  import { getConcurrencyConfig, getRecoveryConfig, getRoutingDefaultAgentId, getRoutingRules, matchRoutingRule, } from "../workspace/support/workspace-ref-utils.js";
10
- import { createHarnessEvent, inferRoutingBindings, renderRuntimeFailure, renderToolFailure, } from "./support/harness-support.js";
9
+ import { createHarnessEvent, inferRoutingBindings, renderRuntimeFailure, } from "./support/harness-support.js";
11
10
  import { ThreadMemorySync } from "./harness/system/thread-memory-sync.js";
12
11
  import { FileBackedStore } from "./harness/system/store.js";
13
- import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
12
+ import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./maintenance/checkpoint-maintenance.js";
14
13
  import { RuntimeRecordMaintenanceLoop, discoverRuntimeRecordMaintenanceTargets, readRuntimeRecordMaintenanceConfig, } from "./maintenance/runtime-record-maintenance.js";
15
14
  import { HealthMonitor } from "./harness/system/health-monitor.js";
16
15
  import { readHealthMonitorConfig } from "./harness/system/health-monitor.js";
17
16
  import { extractMessageText, normalizeMessageContent } from "../utils/message-content.js";
18
17
  import { buildPersistedRunRequest, isTerminalRunState, normalizeInvocationEnvelope, normalizeRunPriority, resolveRunListeners, toPublicApprovalRecord, } from "./harness/run/helpers.js";
19
- import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent, persistApproval, requestApprovalAndEmitEvent, setRunStateAndEmitEvent, } from "./harness/events/events.js";
20
- import { appendAssistantMessage as appendLifecycleAssistantMessage, checkpointRefForState as getCheckpointRefForRunState, expirePendingApprovals as expireLifecyclePendingApprovals, finalizeCancelledRun as finalizeLifecycleCancelledRun, finalizeContinuedRun as finalizeLifecycleContinuedRun, } from "./harness/run/run-lifecycle.js";
21
- import { createContentBlocksItem as createStreamingContentBlocksItem, createToolResultKey as createStreamingToolResultKey, dispatchRunListeners as dispatchStreamingRunListeners, emitOutputDeltaAndCreateItem as emitStreamingOutputDeltaAndCreateItem, } from "./harness/events/streaming.js";
18
+ import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent, requestApprovalAndEmitEvent, setRunStateAndEmitEvent, } from "./harness/events/events.js";
19
+ import { appendAssistantMessage as appendLifecycleAssistantMessage, finalizeCancelledRun as finalizeLifecycleCancelledRun, finalizeContinuedRun as finalizeLifecycleContinuedRun, } from "./harness/run/run-lifecycle.js";
20
+ import { dispatchRunListeners as dispatchStreamingRunListeners, } from "./harness/events/streaming.js";
22
21
  import { buildResumePayload as buildHarnessResumePayload, resolveApprovalRecord as resolveHarnessApprovalRecord, } from "./harness/run/resume.js";
23
- import { dropPendingRunSlot, enqueuePendingRunSlot, shiftNextPendingRunSlot } from "./harness/run/run-queue.js";
22
+ import { cancelRunOperation, resumeRun } from "./harness/run/run-operations.js";
23
+ import { acquireRunSlot as acquireHarnessRunSlot } from "./harness/run/run-slot-acquisition.js";
24
+ import { dropPendingRunSlot, enqueuePendingRunSlot } from "./harness/run/run-queue.js";
24
25
  import { getDefaultHostAgentId, resolveSelectedAgentId } from "./harness/run/routing.js";
25
26
  import { resolveCheckpointer, resolveEmbeddingModel, resolveStore, resolveStoreFromConfig, resolveVectorStore, } from "./harness/run/resources.js";
26
27
  import { createToolMcpServerFromTools, serveToolsOverStdioFromHarness } from "../mcp.js";
@@ -28,6 +29,8 @@ import { getBindingAdapterKind, getBindingPrimaryTools, getBindingStoreConfig }
28
29
  import { isRuntimeEntryBinding } from "./support/runtime-entry.js";
29
30
  import { describeWorkspaceInventory, listAgentSkills as listWorkspaceAgentSkills, } from "./harness/system/inventory.js";
30
31
  import { createDefaultHealthSnapshot, isInventoryEnabled, isThreadMemorySyncEnabled, } from "./harness/runtime-defaults.js";
32
+ import { recoverQueuedStartupRun, recoverResumingStartupRun, recoverRunningStartupRun, } from "./harness/run/recovery.js";
33
+ import { streamHarnessRun } from "./harness/run/stream-run.js";
31
34
  export class AgentHarnessRuntime {
32
35
  workspace;
33
36
  runtimeAdapterOptions;
@@ -85,42 +88,30 @@ export class AgentHarnessRuntime {
85
88
  getThreadSummary: (currentThreadId) => this.getSession(currentThreadId),
86
89
  });
87
90
  }
88
- resolveStore(binding) {
89
- return resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding);
90
- }
91
- resolveStoreFromConfig(storeConfig, runRoot) {
92
- return resolveStoreFromConfig(this.stores, storeConfig, runRoot);
93
- }
94
- async resolveEmbeddingModel(embeddingModelRef) {
95
- return resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions);
96
- }
97
- async resolveVectorStore(vectorStoreRef) {
98
- return resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions);
99
- }
100
91
  constructor(workspace, runtimeAdapterOptions = {}) {
101
92
  this.workspace = workspace;
102
93
  this.runtimeAdapterOptions = runtimeAdapterOptions;
103
94
  const runRoot = this.defaultRunRoot();
104
95
  this.persistence = new SqlitePersistence(runRoot);
105
96
  const defaultStoreConfig = this.listHostBindings()[0]?.harnessRuntime.store;
106
- this.defaultStore = this.resolveStoreFromConfig(defaultStoreConfig, runRoot) ?? new FileBackedStore(`${runRoot}/store.json`);
97
+ this.defaultStore = resolveStoreFromConfig(this.stores, defaultStoreConfig, runRoot) ?? new FileBackedStore(`${runRoot}/store.json`);
107
98
  const runtimeMemoryStoreConfig = typeof this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store === "object" &&
108
99
  this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
109
100
  ? this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
110
101
  : undefined;
111
- this.runtimeMemoryStore = this.resolveStoreFromConfig(runtimeMemoryStoreConfig, runRoot) ?? this.defaultStore;
102
+ this.runtimeMemoryStore = resolveStoreFromConfig(this.stores, runtimeMemoryStoreConfig, runRoot) ?? this.defaultStore;
112
103
  this.resolvedRuntimeAdapterOptions = {
113
104
  ...runtimeAdapterOptions,
114
105
  toolResolver: runtimeAdapterOptions.toolResolver ??
115
106
  createResourceToolResolver(workspace, {
116
- getStore: (binding) => this.resolveStore(binding),
117
- getEmbeddingModel: (embeddingModelRef) => this.resolveEmbeddingModel(embeddingModelRef),
118
- getVectorStore: (vectorStoreRef) => this.resolveVectorStore(vectorStoreRef),
107
+ getStore: (binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding),
108
+ getEmbeddingModel: (embeddingModelRef) => resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions),
109
+ getVectorStore: (vectorStoreRef) => resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions),
119
110
  }),
120
111
  checkpointerResolver: runtimeAdapterOptions.checkpointerResolver ??
121
112
  ((binding) => resolveCheckpointer(this.checkpointers, binding)),
122
113
  storeResolver: runtimeAdapterOptions.storeResolver ??
123
- ((binding) => this.resolveStore(binding)),
114
+ ((binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding)),
124
115
  backendResolver: runtimeAdapterOptions.backendResolver ??
125
116
  ((binding) => createResourceBackendResolver(workspace)(binding)),
126
117
  };
@@ -183,22 +174,12 @@ export class AgentHarnessRuntime {
183
174
  }
184
175
  return createDefaultHealthSnapshot(this.activeRunSlots, this.pendingRunSlots.length);
185
176
  }
186
- getBinding(agentId) {
187
- return this.workspace.bindings.get(agentId);
188
- }
189
- listAgentTools(agentId) {
190
- const binding = this.getBinding(agentId);
191
- if (!binding) {
192
- throw new Error(`Unknown agent ${agentId}`);
193
- }
194
- return getBindingPrimaryTools(binding);
195
- }
196
177
  resolveAgentTools(agentId) {
197
- const binding = this.getBinding(agentId);
178
+ const binding = this.workspace.bindings.get(agentId);
198
179
  if (!binding) {
199
180
  throw new Error(`Unknown agent ${agentId}`);
200
181
  }
201
- const compiledTools = this.listAgentTools(agentId);
182
+ const compiledTools = getBindingPrimaryTools(binding);
202
183
  const resolver = this.resolvedRuntimeAdapterOptions.toolResolver;
203
184
  const resolvedTools = resolver ? resolver(compiledTools.map((tool) => tool.id), binding) : [];
204
185
  return compiledTools.map((compiledTool, index) => ({
@@ -416,9 +397,6 @@ export class AgentHarnessRuntime {
416
397
  const userTurn = history.find((message) => message.runId === runId && message.role === "user");
417
398
  return userTurn?.content ?? "";
418
399
  }
419
- async appendAssistantMessage(threadId, runId, content) {
420
- return appendLifecycleAssistantMessage(this.persistence, threadId, runId, content);
421
- }
422
400
  async getRunCancellation(runId) {
423
401
  const control = await this.persistence.getRunControl(runId);
424
402
  return {
@@ -426,12 +404,6 @@ export class AgentHarnessRuntime {
426
404
  ...(control?.cancelReason ? { reason: control.cancelReason } : {}),
427
405
  };
428
406
  }
429
- async expirePendingApprovals(threadId, runId) {
430
- return expireLifecyclePendingApprovals({
431
- persistence: this.persistence,
432
- emit: (currentThreadId, currentRunId, sequence, eventType, payload, source) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload, source),
433
- }, threadId, runId);
434
- }
435
407
  async finalizeCancelledRun(threadId, runId, previousState, reason) {
436
408
  return finalizeLifecycleCancelledRun({
437
409
  persistence: this.persistence,
@@ -507,7 +479,12 @@ export class AgentHarnessRuntime {
507
479
  };
508
480
  }
509
481
  catch (error) {
510
- await this.emitSyntheticFallback(threadId, runId, agentId, error, 103);
482
+ await emitSyntheticFallbackEvent({
483
+ persistence: this.persistence,
484
+ publishEvent: (event) => this.eventBus.publish(event),
485
+ trackBackgroundTask: (task) => this.trackBackgroundTask(task),
486
+ backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
487
+ }, threadId, runId, agentId, error, 103);
511
488
  await this.setRunStateAndEmit(threadId, runId, 104, "failed", {
512
489
  previousState: previousState === "queued" ? "running" : previousState,
513
490
  error: error instanceof Error ? error.message : String(error),
@@ -524,9 +501,6 @@ export class AgentHarnessRuntime {
524
501
  await this.persistence.clearRunRequest(threadId, runId);
525
502
  }
526
503
  }
527
- checkpointRefForState(threadId, runId, state) {
528
- return getCheckpointRefForRunState(threadId, runId, state);
529
- }
530
504
  async finalizeContinuedRun(binding, threadId, runId, input, actual, options) {
531
505
  return finalizeLifecycleContinuedRun({
532
506
  persistence: this.persistence,
@@ -535,23 +509,6 @@ export class AgentHarnessRuntime {
535
509
  requestApprovalAndEmit: (currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence) => this.requestApprovalAndEmit(currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence),
536
510
  }, binding, threadId, runId, input, actual, options);
537
511
  }
538
- async emitOutputDeltaAndCreateItem(threadId, runId, agentId, content) {
539
- return emitStreamingOutputDeltaAndCreateItem((currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload), threadId, runId, agentId, content);
540
- }
541
- createContentBlocksItem(threadId, runId, agentId, contentBlocks) {
542
- return createStreamingContentBlocksItem(threadId, runId, agentId, contentBlocks);
543
- }
544
- createToolResultKey(toolName, output, isError) {
545
- return createStreamingToolResultKey(toolName, output, isError);
546
- }
547
- async emitRunCreated(threadId, runId, payload) {
548
- return emitRunCreatedEvent({
549
- persistence: this.persistence,
550
- publishEvent: (event) => this.eventBus.publish(event),
551
- trackBackgroundTask: (task) => this.trackBackgroundTask(task),
552
- backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
553
- }, threadId, runId, payload);
554
- }
555
512
  async setRunStateAndEmit(threadId, runId, sequence, state, options) {
556
513
  return setRunStateAndEmitEvent({
557
514
  persistence: this.persistence,
@@ -568,25 +525,6 @@ export class AgentHarnessRuntime {
568
525
  backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
569
526
  }, threadId, runId, input, interruptContent, checkpointRef, sequence);
570
527
  }
571
- async emitSyntheticFallback(threadId, runId, selectedAgentId, error, sequence = 3) {
572
- await emitSyntheticFallbackEvent({
573
- persistence: this.persistence,
574
- publishEvent: (event) => this.eventBus.publish(event),
575
- trackBackgroundTask: (task) => this.trackBackgroundTask(task),
576
- backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
577
- }, threadId, runId, selectedAgentId, error, sequence);
578
- }
579
- async persistApproval(threadId, runId, checkpointRef, input, interruptContent) {
580
- return persistApproval({
581
- persistence: this.persistence,
582
- publishEvent: (event) => this.eventBus.publish(event),
583
- trackBackgroundTask: (task) => this.trackBackgroundTask(task),
584
- backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
585
- }, threadId, runId, checkpointRef, input, interruptContent);
586
- }
587
- async resolveApprovalRecord(options, thread) {
588
- return resolveHarnessApprovalRecord(this.persistence, options, thread);
589
- }
590
528
  isDecisionRun(options) {
591
529
  return "decision" in options;
592
530
  }
@@ -596,161 +534,75 @@ export class AgentHarnessRuntime {
596
534
  }
597
535
  await listener(value);
598
536
  }
599
- async acquireRunSlot(threadId, runId, activeState = "running", priority = 0) {
600
- let stopHeartbeat = () => undefined;
601
- const beginLease = async (mode) => {
602
- if (!threadId || !runId) {
603
- return;
604
- }
605
- const claimedAt = new Date().toISOString();
606
- if (mode === "queue-claim") {
607
- await this.persistence.claimQueuedRun({
608
- threadId,
609
- runId,
610
- workerId: this.workerId,
611
- claimedAt,
612
- leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
613
- });
614
- }
615
- else {
616
- await this.persistence.renewRunLease({
617
- runId,
618
- workerId: this.workerId,
619
- heartbeatAt: claimedAt,
620
- leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
621
- });
622
- }
623
- if (this.concurrencyConfig.heartbeatIntervalMs <= 0) {
624
- return;
625
- }
626
- const timer = setInterval(() => {
627
- void this.persistence.renewRunLease({
628
- runId,
629
- workerId: this.workerId,
630
- heartbeatAt: new Date().toISOString(),
631
- leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
632
- });
633
- }, this.concurrencyConfig.heartbeatIntervalMs);
634
- timer.unref?.();
635
- stopHeartbeat = () => {
636
- clearInterval(timer);
637
- };
638
- };
639
- const releaseLease = async () => {
640
- stopHeartbeat();
641
- if (runId) {
642
- await this.persistence.releaseRunClaim(runId);
643
- }
644
- };
645
- const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
646
- if (!maxConcurrentRuns) {
647
- await beginLease("direct-heartbeat");
648
- return async () => {
649
- await releaseLease();
650
- };
537
+ async prepareRunStart(options, invocation, runCreatedPayload) {
538
+ const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
539
+ const binding = this.workspace.bindings.get(selectedAgentId);
540
+ if (!binding) {
541
+ throw new Error(`Unknown agent ${selectedAgentId}`);
651
542
  }
652
- const canActivateImmediately = this.activeRunSlots < maxConcurrentRuns;
653
- const useDirectHeartbeatFastPath = canActivateImmediately && maxConcurrentRuns > 1;
654
- if (canActivateImmediately) {
655
- this.activeRunSlots += 1;
656
- if (threadId && runId && !useDirectHeartbeatFastPath) {
657
- await this.persistence.enqueueRun({ threadId, runId, priority });
658
- }
659
- await beginLease(useDirectHeartbeatFastPath ? "direct-heartbeat" : "queue-claim");
660
- let released = false;
661
- return async () => {
662
- if (released) {
663
- return;
664
- }
665
- released = true;
666
- await releaseLease();
667
- this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
668
- const next = shiftNextPendingRunSlot(this.pendingRunSlots);
669
- void next?.activate();
670
- };
543
+ const policyDecision = this.policyEngine.evaluate(binding);
544
+ if (!policyDecision.allowed) {
545
+ throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
671
546
  }
672
- const activateQueuedRun = async () => {
673
- const currentRun = runId ? await this.persistence.getRun(runId) : null;
674
- if (currentRun?.state === "cancelled") {
675
- return "abort";
676
- }
677
- this.activeRunSlots += 1;
678
- if (threadId && runId) {
679
- await this.emit(threadId, runId, 4, "run.dequeued", {
680
- queuePosition: 0,
681
- activeRunCount: this.activeRunSlots,
682
- maxConcurrentRuns,
683
- priority,
684
- });
685
- await this.setRunStateAndEmit(threadId, runId, 5, activeState, {
686
- previousState: "queued",
687
- });
688
- await beginLease("queue-claim");
689
- }
690
- return "activate";
547
+ const priority = normalizeRunPriority(options.priority);
548
+ const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
549
+ const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
550
+ return {
551
+ binding,
552
+ selectedAgentId,
553
+ priority,
554
+ threadId,
555
+ runId,
556
+ isNewThread,
557
+ runCreatedEventPromise: emitRunCreatedEvent({
558
+ persistence: this.persistence,
559
+ publishEvent: (event) => this.eventBus.publish(event),
560
+ trackBackgroundTask: (task) => this.trackBackgroundTask(task),
561
+ backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
562
+ }, threadId, runId, runCreatedPayload(binding, selectedAgentId)),
563
+ releaseRunSlotPromise: this.acquireRunSlot(threadId, runId, "running", priority),
691
564
  };
692
- if (threadId && runId) {
693
- await this.persistence.enqueueRun({ threadId, runId, priority });
694
- const slotAcquisition = new Promise((resolve, reject) => {
695
- const displacedEntries = this.enqueuePendingRunSlot({
696
- threadId,
697
- runId,
698
- priority,
699
- activate: async () => {
700
- try {
701
- resolve(await activateQueuedRun());
702
- }
703
- catch (error) {
704
- reject(error);
705
- }
706
- },
707
- abort: () => resolve("abort"),
708
- });
709
- void Promise.all(displacedEntries.map((candidate) => this.emit(candidate.threadId, candidate.runId, 3, "run.queued", {
710
- queuePosition: candidate.queuePosition,
711
- activeRunCount: this.activeRunSlots,
712
- maxConcurrentRuns,
713
- priority: candidate.priority,
714
- })));
715
- });
716
- const queuePosition = this.pendingRunSlots.findIndex((entry) => entry.runId === runId) + 1;
717
- await this.setRunStateAndEmit(threadId, runId, 2, "queued", {
718
- previousState: activeState,
719
- });
720
- await this.emit(threadId, runId, 3, "run.queued", {
721
- queuePosition,
722
- activeRunCount: this.activeRunSlots,
723
- maxConcurrentRuns,
724
- priority,
725
- });
726
- const slotAcquisitionResult = await slotAcquisition;
727
- if (slotAcquisitionResult === "abort") {
728
- return async () => undefined;
729
- }
730
- let released = false;
731
- return async () => {
732
- if (released) {
733
- return;
734
- }
735
- released = true;
736
- await releaseLease();
737
- this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
738
- const next = this.pendingRunSlots.shift();
739
- void next?.activate();
740
- };
741
- }
742
- let released = false;
743
- return async () => {
744
- if (released) {
745
- return;
746
- }
747
- released = true;
748
- await releaseLease();
749
- this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
750
- const next = shiftNextPendingRunSlot(this.pendingRunSlots);
751
- void next?.activate();
565
+ }
566
+ createStartupRecoveryContext() {
567
+ return {
568
+ persistence: this.persistence,
569
+ workspace: this.workspace,
570
+ runtimeAdapter: this.runtimeAdapter,
571
+ recoveryConfig: this.recoveryConfig,
572
+ concurrencyConfig: this.concurrencyConfig,
573
+ getBinding: (agentId) => this.workspace.bindings.get(agentId),
574
+ acquireRunSlot: (threadId, runId, activeState = "running", priority = 0) => this.acquireRunSlot(threadId, runId, activeState, priority),
575
+ executeQueuedRun: (binding, input, threadId, runId, agentId, options = {}) => this.executeQueuedRun(binding, input, threadId, runId, agentId, options),
576
+ setRunStateAndEmit: (threadId, runId, sequence, state, options) => this.setRunStateAndEmit(threadId, runId, sequence, state, options),
577
+ emit: (threadId, runId, sequence, eventType, payload) => this.emit(threadId, runId, sequence, eventType, payload),
578
+ loadRunInput: (threadId, runId) => this.loadRunInput(threadId, runId),
579
+ finalizeContinuedRun: (binding, threadId, runId, input, actual, options) => this.finalizeContinuedRun(binding, threadId, runId, input, actual, options),
580
+ supportsRunningReplay: (binding) => this.supportsRunningReplay(binding),
581
+ isStaleRunningRun: (thread, nowMs = Date.now()) => this.isStaleRunningRun(thread, nowMs),
582
+ recordLlmSuccess: (startedAt) => this.recordLlmSuccess(startedAt),
583
+ recordLlmFailure: (startedAt) => this.recordLlmFailure(startedAt),
752
584
  };
753
585
  }
586
+ async acquireRunSlot(threadId, runId, activeState = "running", priority = 0) {
587
+ return acquireHarnessRunSlot({
588
+ persistence: this.persistence,
589
+ workerId: this.workerId,
590
+ concurrencyConfig: this.concurrencyConfig,
591
+ pendingRunSlots: this.pendingRunSlots,
592
+ getActiveRunSlots: () => this.activeRunSlots,
593
+ setActiveRunSlots: (count) => {
594
+ this.activeRunSlots = count;
595
+ },
596
+ enqueuePendingRunSlot: (entry) => this.enqueuePendingRunSlot(entry),
597
+ emit: (currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload),
598
+ setRunStateAndEmit: (currentThreadId, currentRunId, sequence, state, stateOptions) => this.setRunStateAndEmit(currentThreadId, currentRunId, sequence, state, stateOptions),
599
+ }, {
600
+ threadId,
601
+ runId,
602
+ activeState,
603
+ priority,
604
+ });
605
+ }
754
606
  dropPendingRunSlot(runId) {
755
607
  return dropPendingRunSlot(this.pendingRunSlots, runId);
756
608
  }
@@ -776,32 +628,12 @@ export class AgentHarnessRuntime {
776
628
  return this.dispatchRunListeners(this.streamEvents(options), resolvedListeners);
777
629
  }
778
630
  const invocation = normalizeInvocationEnvelope(options);
779
- const selectedAgentId = await resolveSelectedAgentId({
780
- workspace: this.workspace,
781
- input: options.input,
782
- requestedAgentId: options.agentId,
783
- threadId: options.threadId,
784
- preferredHostAgentId: this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
785
- getThreadSummary: (threadId) => this.getSession(threadId),
786
- });
787
- const binding = this.workspace.bindings.get(selectedAgentId);
788
- if (!binding) {
789
- throw new Error(`Unknown agent ${selectedAgentId}`);
790
- }
791
- const policyDecision = this.policyEngine.evaluate(binding);
792
- if (!policyDecision.allowed) {
793
- throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
794
- }
795
- const priority = normalizeRunPriority(options.priority);
796
- const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
797
- const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
798
- const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
799
- agentId: binding.agent.id,
631
+ const { binding, selectedAgentId, threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (activeBinding, activeSelectedAgentId) => ({
632
+ agentId: activeBinding.agent.id,
800
633
  requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
801
- selectedAgentId,
802
- executionMode: getBindingAdapterKind(binding),
803
- });
804
- const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
634
+ selectedAgentId: activeSelectedAgentId,
635
+ executionMode: getBindingAdapterKind(activeBinding),
636
+ }));
805
637
  await runCreatedEventPromise;
806
638
  const releaseRunSlot = await releaseRunSlotPromise;
807
639
  try {
@@ -821,14 +653,7 @@ export class AgentHarnessRuntime {
821
653
  }
822
654
  async *streamEvents(options) {
823
655
  const invocation = normalizeInvocationEnvelope(options);
824
- const selectedAgentId = await resolveSelectedAgentId({
825
- workspace: this.workspace,
826
- input: options.input,
827
- requestedAgentId: options.agentId,
828
- threadId: options.threadId,
829
- preferredHostAgentId: this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
830
- getThreadSummary: (threadId) => this.getSession(threadId),
831
- });
656
+ const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
832
657
  const binding = this.workspace.bindings.get(selectedAgentId);
833
658
  if (!binding) {
834
659
  const result = await this.run(options);
@@ -843,357 +668,62 @@ export class AgentHarnessRuntime {
843
668
  }
844
669
  return;
845
670
  }
846
- let emitted = false;
847
- let streamActivityObserved = false;
848
- const priority = normalizeRunPriority(options.priority);
849
- const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
850
- const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
851
- const priorHistoryPromise = Promise.resolve(isNewThread ? [] : undefined).then((historyHint) => historyHint ?? this.loadPriorHistory(threadId, runId));
852
- const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
853
- agentId: selectedAgentId,
671
+ const { threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (_binding, activeSelectedAgentId) => ({
672
+ agentId: activeSelectedAgentId,
854
673
  requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
855
- selectedAgentId,
674
+ selectedAgentId: activeSelectedAgentId,
856
675
  input: options.input,
857
676
  state: "running",
677
+ }));
678
+ yield* streamHarnessRun({
679
+ binding,
680
+ input: options.input,
681
+ invocation,
682
+ threadId,
683
+ runId,
684
+ selectedAgentId,
685
+ isNewThread,
686
+ runCreatedEventPromise,
687
+ releaseRunSlotPromise,
688
+ loadPriorHistory: (currentThreadId, currentRunId) => this.loadPriorHistory(currentThreadId, currentRunId),
689
+ stream: (activeBinding, input, currentThreadId, priorHistory, streamOptions) => this.runtimeAdapter.stream(activeBinding, input, currentThreadId, priorHistory, streamOptions),
690
+ invokeWithHistory: (activeBinding, input, currentThreadId, currentRunId) => this.invokeWithHistory(activeBinding, input, currentThreadId, currentRunId),
691
+ emit: (currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload),
692
+ setRunStateAndEmit: (currentThreadId, currentRunId, sequence, state, stateOptions) => this.setRunStateAndEmit(currentThreadId, currentRunId, sequence, state, stateOptions),
693
+ requestApprovalAndEmit: (currentThreadId, currentRunId, input, interruptContent, checkpointRef, sequence) => this.requestApprovalAndEmit(currentThreadId, currentRunId, input, interruptContent, checkpointRef, sequence),
694
+ appendAssistantMessage: (currentThreadId, currentRunId, content) => appendLifecycleAssistantMessage(this.persistence, currentThreadId, currentRunId, content),
695
+ clearRunRequest: (currentThreadId, currentRunId) => this.persistence.clearRunRequest(currentThreadId, currentRunId),
696
+ emitSyntheticFallback: (currentThreadId, currentRunId, currentSelectedAgentId, error) => emitSyntheticFallbackEvent({
697
+ persistence: this.persistence,
698
+ publishEvent: (event) => this.eventBus.publish(event),
699
+ trackBackgroundTask: (task) => this.trackBackgroundTask(task),
700
+ backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
701
+ }, currentThreadId, currentRunId, currentSelectedAgentId, error),
858
702
  });
859
- yield { type: "event", event: await runCreatedEventPromise };
860
- const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
861
- let releaseRunSlot = async () => undefined;
862
- try {
863
- try {
864
- const [priorHistory, acquiredReleaseRunSlot] = await Promise.all([
865
- priorHistoryPromise,
866
- releaseRunSlotPromise,
867
- ]).then(([loadedPriorHistory, resolvedReleaseRunSlot]) => [loadedPriorHistory, resolvedReleaseRunSlot]);
868
- releaseRunSlot = acquiredReleaseRunSlot;
869
- let assistantOutput = "";
870
- const toolErrors = [];
871
- let lastToolResultKey = null;
872
- for await (const chunk of this.runtimeAdapter.stream(binding, options.input, threadId, priorHistory, {
873
- context: invocation.context,
874
- state: invocation.state,
875
- files: invocation.files,
876
- runId,
877
- })) {
878
- if (chunk) {
879
- streamActivityObserved = true;
880
- const normalizedChunk = typeof chunk === "string"
881
- ? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
882
- ? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
883
- : { kind: "content", content: chunk }
884
- : chunk;
885
- if (normalizedChunk.kind === "upstream-event") {
886
- yield {
887
- type: "upstream-event",
888
- threadId,
889
- runId,
890
- agentId: selectedAgentId,
891
- event: normalizedChunk.event.format === "langgraph-v2"
892
- ? normalizedChunk.event
893
- : normalizeUpstreamRuntimeEvent(normalizedChunk.event.raw ?? normalizedChunk.event),
894
- };
895
- continue;
896
- }
897
- if (normalizedChunk.kind === "interrupt") {
898
- const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
899
- const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 6, "waiting_for_approval", {
900
- previousState: "running",
901
- checkpointRef,
902
- });
903
- const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 7);
904
- yield {
905
- type: "event",
906
- event: waitingEvent,
907
- };
908
- yield {
909
- type: "event",
910
- event: approvalRequest.event,
911
- };
912
- yield {
913
- type: "result",
914
- result: {
915
- threadId,
916
- runId,
917
- agentId: selectedAgentId,
918
- state: "waiting_for_approval",
919
- output: assistantOutput,
920
- finalMessageText: assistantOutput,
921
- interruptContent: normalizedChunk.content,
922
- approvalId: approvalRequest.approval.approvalId,
923
- pendingActionId: approvalRequest.approval.pendingActionId,
924
- },
925
- };
926
- return;
927
- }
928
- if (normalizedChunk.kind === "reasoning") {
929
- await this.emit(threadId, runId, 3, "reasoning.delta", {
930
- content: normalizedChunk.content,
931
- });
932
- yield {
933
- type: "reasoning",
934
- threadId,
935
- runId,
936
- agentId: selectedAgentId,
937
- content: normalizedChunk.content,
938
- };
939
- continue;
940
- }
941
- if (normalizedChunk.kind === "step") {
942
- yield {
943
- type: "step",
944
- threadId,
945
- runId,
946
- agentId: selectedAgentId,
947
- content: normalizedChunk.content,
948
- };
949
- continue;
950
- }
951
- if (normalizedChunk.kind === "tool-result") {
952
- const toolResultKey = this.createToolResultKey(normalizedChunk.toolName, normalizedChunk.output, normalizedChunk.isError);
953
- if (toolResultKey === lastToolResultKey) {
954
- continue;
955
- }
956
- lastToolResultKey = toolResultKey;
957
- if (normalizedChunk.isError) {
958
- toolErrors.push(renderToolFailure(normalizedChunk.toolName, normalizedChunk.output));
959
- }
960
- yield {
961
- type: "tool-result",
962
- threadId,
963
- runId,
964
- agentId: selectedAgentId,
965
- toolName: normalizedChunk.toolName,
966
- output: normalizedChunk.output,
967
- isError: normalizedChunk.isError,
968
- };
969
- continue;
970
- }
971
- emitted = true;
972
- assistantOutput += normalizedChunk.content;
973
- yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, normalizedChunk.content);
974
- }
975
- }
976
- if (!assistantOutput && toolErrors.length > 0) {
977
- assistantOutput = toolErrors.join("\n\n");
978
- emitted = true;
979
- yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, assistantOutput);
980
- }
981
- if (!assistantOutput) {
982
- const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
983
- if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
984
- yield this.createContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
985
- }
986
- if (actual.output) {
987
- assistantOutput = actual.output;
988
- emitted = true;
989
- yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
990
- }
991
- }
992
- await this.appendAssistantMessage(threadId, runId, assistantOutput);
993
- yield {
994
- type: "result",
995
- result: {
996
- threadId,
997
- runId,
998
- agentId: selectedAgentId,
999
- state: "completed",
1000
- output: assistantOutput,
1001
- finalMessageText: assistantOutput,
1002
- },
1003
- };
1004
- yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
1005
- previousState: "running",
1006
- }) };
1007
- return;
1008
- }
1009
- catch (error) {
1010
- if (emitted || streamActivityObserved) {
1011
- const runtimeFailure = renderRuntimeFailure(error);
1012
- yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
1013
- previousState: "running",
1014
- error: error instanceof Error ? error.message : String(error),
1015
- }) };
1016
- yield {
1017
- type: "content",
1018
- threadId,
1019
- runId,
1020
- agentId: selectedAgentId,
1021
- content: runtimeFailure,
1022
- };
1023
- yield {
1024
- type: "result",
1025
- result: {
1026
- threadId,
1027
- runId,
1028
- agentId: selectedAgentId,
1029
- state: "failed",
1030
- output: runtimeFailure,
1031
- finalMessageText: runtimeFailure,
1032
- },
1033
- };
1034
- return;
1035
- }
1036
- if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
1037
- yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
1038
- previousState: "running",
1039
- error: error.message,
1040
- }) };
1041
- yield {
1042
- type: "content",
1043
- threadId,
1044
- runId,
1045
- agentId: selectedAgentId,
1046
- content: renderRuntimeFailure(error),
1047
- };
1048
- yield {
1049
- type: "result",
1050
- result: {
1051
- threadId,
1052
- runId,
1053
- agentId: selectedAgentId,
1054
- state: "failed",
1055
- output: renderRuntimeFailure(error),
1056
- finalMessageText: renderRuntimeFailure(error),
1057
- },
1058
- };
1059
- return;
1060
- }
1061
- try {
1062
- const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
1063
- await this.appendAssistantMessage(threadId, runId, actual.output);
1064
- if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
1065
- yield this.createContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
1066
- }
1067
- if (actual.output) {
1068
- yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
1069
- }
1070
- yield {
1071
- type: "result",
1072
- result: {
1073
- ...actual,
1074
- threadId,
1075
- runId,
1076
- agentId: selectedAgentId,
1077
- },
1078
- };
1079
- yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
1080
- previousState: "running",
1081
- }) };
1082
- return;
1083
- }
1084
- catch (invokeError) {
1085
- await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
1086
- yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
1087
- previousState: "running",
1088
- error: invokeError instanceof Error ? invokeError.message : String(invokeError),
1089
- }) };
1090
- yield {
1091
- type: "content",
1092
- threadId,
1093
- runId,
1094
- agentId: selectedAgentId,
1095
- content: renderRuntimeFailure(invokeError),
1096
- };
1097
- yield {
1098
- type: "result",
1099
- result: {
1100
- threadId,
1101
- runId,
1102
- agentId: selectedAgentId,
1103
- state: "failed",
1104
- output: renderRuntimeFailure(invokeError),
1105
- finalMessageText: renderRuntimeFailure(invokeError),
1106
- },
1107
- };
1108
- return;
1109
- }
1110
- }
1111
- }
1112
- finally {
1113
- await this.persistence.clearRunRequest(threadId, runId);
1114
- await releaseRunSlot();
1115
- }
1116
703
  }
1117
704
  async resume(options) {
1118
- const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
1119
- const thread = options.threadId
1120
- ? await this.getSession(options.threadId)
1121
- : approvalById
1122
- ? await this.getSession(approvalById.threadId)
1123
- : null;
1124
- if (!thread) {
1125
- throw new Error("resume requires either threadId or approvalId");
1126
- }
1127
- const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
1128
- const threadId = approval.threadId;
1129
- const runId = approval.runId;
1130
- const binding = this.workspace.bindings.get(thread.agentId);
1131
- if (!binding) {
1132
- throw new Error(`Unknown agent ${thread.agentId}`);
1133
- }
1134
- const resumePayload = this.buildResumePayload(binding, approval, options);
1135
- const cancellation = await this.getRunCancellation(runId);
1136
- if (cancellation.requested) {
1137
- return this.finalizeCancelledRun(threadId, runId, thread.status, cancellation.reason);
1138
- }
1139
- await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
1140
- const releaseRunSlot = await this.acquireRunSlot(threadId, runId, "resuming", await this.resolvePersistedRunPriority(threadId, runId));
1141
- try {
1142
- await this.persistence.saveRecoveryIntent(threadId, runId, {
1143
- kind: "approval-decision",
1144
- savedAt: new Date().toISOString(),
1145
- checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
1146
- resumePayload,
1147
- attempts: 0,
1148
- });
1149
- await this.emit(threadId, runId, 5, "run.resumed", {
1150
- resumeKind: "cross-restart",
1151
- checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
1152
- state: "resuming",
1153
- approvalId: approval.approvalId,
1154
- pendingActionId: approval.pendingActionId,
1155
- });
1156
- await this.persistence.resolveApproval(threadId, runId, approval.approvalId, options.decision === "reject" ? "rejected" : options.decision === "edit" ? "edited" : "approved");
1157
- await this.emit(threadId, runId, 6, "approval.resolved", {
1158
- approvalId: approval.approvalId,
1159
- pendingActionId: approval.pendingActionId,
1160
- decision: options.decision ?? "approve",
1161
- toolName: approval.toolName,
1162
- });
1163
- const history = await this.persistence.listThreadMessages(threadId);
1164
- const priorHistory = history.filter((message) => message.runId !== runId);
1165
- const runInput = await this.loadRunInput(threadId, runId);
1166
- const startedAt = Date.now();
1167
- try {
1168
- const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
1169
- this.recordLlmSuccess(startedAt);
1170
- const cancelledAfterInvoke = await this.getRunCancellation(runId);
1171
- if (cancelledAfterInvoke.requested) {
1172
- return this.finalizeCancelledRun(threadId, runId, "resuming", cancelledAfterInvoke.reason);
1173
- }
1174
- await this.persistence.clearRecoveryIntent(threadId, runId);
1175
- const finalized = await this.finalizeContinuedRun(binding, threadId, runId, runInput, actual, {
1176
- previousState: "resuming",
1177
- stateSequence: 7,
1178
- approvalSequence: 8,
1179
- });
1180
- return {
1181
- ...finalized,
1182
- approvalId: finalized.approvalId ?? approval.approvalId,
1183
- pendingActionId: finalized.pendingActionId ?? approval.pendingActionId,
1184
- };
1185
- }
1186
- catch (error) {
1187
- this.recordLlmFailure(startedAt);
1188
- throw error;
1189
- }
1190
- }
1191
- finally {
1192
- await releaseRunSlot();
1193
- }
1194
- }
1195
- buildResumePayload(binding, approval, options) {
1196
- return buildHarnessResumePayload(binding, approval, options);
705
+ return resumeRun({
706
+ getApprovalById: (approvalId) => this.persistence.getApproval(approvalId),
707
+ getSession: (threadId) => this.getSession(threadId),
708
+ resolveApprovalRecord: (resumeOptions, thread) => resolveHarnessApprovalRecord(this.persistence, resumeOptions, thread),
709
+ getBinding: (agentId) => this.workspace.bindings.get(agentId),
710
+ buildResumePayload: (binding, approval, resumeOptions) => buildHarnessResumePayload(binding, approval, resumeOptions),
711
+ getRunCancellation: (runId) => this.getRunCancellation(runId),
712
+ finalizeCancelledRun: (threadId, runId, previousState, reason) => this.finalizeCancelledRun(threadId, runId, previousState, reason),
713
+ setRunState: (threadId, runId, state, checkpointRef) => this.persistence.setRunState(threadId, runId, state, checkpointRef),
714
+ acquireRunSlot: (threadId, runId, activeState, priority) => this.acquireRunSlot(threadId, runId, activeState, priority),
715
+ resolvePersistedRunPriority: (threadId, runId) => this.resolvePersistedRunPriority(threadId, runId),
716
+ saveRecoveryIntent: (threadId, runId, payload) => this.persistence.saveRecoveryIntent(threadId, runId, payload),
717
+ emit: (threadId, runId, sequence, eventType, payload) => this.emit(threadId, runId, sequence, eventType, payload),
718
+ resolveApproval: (threadId, runId, approvalId, resolution) => this.persistence.resolveApproval(threadId, runId, approvalId, resolution),
719
+ listThreadMessages: (threadId) => this.persistence.listThreadMessages(threadId),
720
+ loadRunInput: (threadId, runId) => this.loadRunInput(threadId, runId),
721
+ invoke: (binding, input, threadId, runId, resumePayload, priorHistory) => this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory),
722
+ recordLlmSuccess: (startedAt) => this.recordLlmSuccess(startedAt),
723
+ recordLlmFailure: (startedAt) => this.recordLlmFailure(startedAt),
724
+ clearRecoveryIntent: (threadId, runId) => this.persistence.clearRecoveryIntent(threadId, runId),
725
+ finalizeContinuedRun: (binding, threadId, runId, input, actual, operationOptions) => this.finalizeContinuedRun(binding, threadId, runId, input, actual, operationOptions),
726
+ }, options);
1197
727
  }
1198
728
  async restartConversation(options) {
1199
729
  const thread = await this.getSession(options.threadId);
@@ -1232,37 +762,13 @@ export class AgentHarnessRuntime {
1232
762
  await this.close();
1233
763
  }
1234
764
  async cancelRun(options) {
1235
- const run = await this.persistence.getRun(options.runId);
1236
- if (!run) {
1237
- throw new Error(`Unknown run ${options.runId}`);
1238
- }
1239
- if (isTerminalRunState(run.state)) {
1240
- return {
1241
- threadId: run.threadId,
1242
- runId: run.runId,
1243
- agentId: run.agentId,
1244
- state: run.state,
1245
- output: run.state,
1246
- };
1247
- }
1248
- await this.persistence.requestRunCancel(run.runId, options.reason);
1249
- if (run.state === "queued" || run.state === "waiting_for_approval" || run.state === "claimed") {
1250
- if (run.state === "queued") {
1251
- this.dropPendingRunSlot(run.runId);
1252
- }
1253
- return this.finalizeCancelledRun(run.threadId, run.runId, run.state, options.reason);
1254
- }
1255
- await this.setRunStateAndEmit(run.threadId, run.runId, 103, "cancelling", {
1256
- previousState: run.state,
1257
- ...(options.reason ? { error: options.reason } : {}),
1258
- });
1259
- return {
1260
- threadId: run.threadId,
1261
- runId: run.runId,
1262
- agentId: run.agentId,
1263
- state: "cancelling",
1264
- output: options.reason ? `cancelling: ${options.reason}` : "cancelling",
1265
- };
765
+ return cancelRunOperation({
766
+ getRun: (runId) => this.persistence.getRun(runId),
767
+ requestRunCancel: (runId, reason) => this.persistence.requestRunCancel(runId, reason),
768
+ dropPendingRunSlot: (runId) => this.dropPendingRunSlot(runId),
769
+ finalizeCancelledRun: (threadId, runId, previousState, reason) => this.finalizeCancelledRun(threadId, runId, previousState, reason),
770
+ setRunStateAndEmit: (threadId, runId, sequence, state, stateOptions) => this.setRunStateAndEmit(threadId, runId, sequence, state, stateOptions),
771
+ }, options);
1266
772
  }
1267
773
  async recoverStartupRuns() {
1268
774
  if (!this.recoveryConfig.enabled) {
@@ -1270,136 +776,14 @@ export class AgentHarnessRuntime {
1270
776
  }
1271
777
  await this.reclaimExpiredClaimedRuns();
1272
778
  const threads = await this.persistence.listSessions();
779
+ const recoveryContext = this.createStartupRecoveryContext();
1273
780
  for (const thread of threads) {
1274
- if (thread.status === "queued") {
1275
- const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
1276
- const binding = this.workspace.bindings.get(runMeta.agentId);
1277
- if (!binding) {
1278
- continue;
1279
- }
1280
- const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
1281
- if (!request) {
1282
- await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
1283
- previousState: "queued",
1284
- error: "missing persisted run request for queued run recovery",
1285
- });
1286
- continue;
1287
- }
1288
- const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
1289
- try {
1290
- await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
1291
- context: request.invocation?.context,
1292
- state: request.invocation?.inputs,
1293
- files: request.invocation?.attachments,
1294
- previousState: "queued",
1295
- stateSequence: 103,
1296
- approvalSequence: 104,
1297
- });
1298
- }
1299
- finally {
1300
- await releaseRunSlot();
1301
- }
1302
- continue;
1303
- }
1304
- if (thread.status === "running") {
1305
- const isStale = await this.isStaleRunningRun(thread);
1306
- if (!isStale) {
1307
- continue;
1308
- }
1309
- const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
1310
- const binding = this.workspace.bindings.get(runMeta.agentId);
1311
- if (!binding) {
1312
- continue;
1313
- }
1314
- if (!this.supportsRunningReplay(binding)) {
1315
- await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
1316
- previousState: "running",
1317
- error: "stale running run cannot be replayed safely",
1318
- });
1319
- await this.persistence.releaseRunClaim(thread.latestRunId);
1320
- continue;
1321
- }
1322
- const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
1323
- if (!request) {
1324
- await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
1325
- previousState: "running",
1326
- error: "missing persisted run request for stale running run recovery",
1327
- });
1328
- await this.persistence.releaseRunClaim(thread.latestRunId);
1329
- continue;
1330
- }
1331
- const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
1332
- try {
1333
- await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
1334
- resumeKind: "startup-running-recovery",
1335
- state: "running",
1336
- });
1337
- await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
1338
- context: request.invocation?.context,
1339
- state: request.invocation?.inputs,
1340
- files: request.invocation?.attachments,
1341
- previousState: "running",
1342
- stateSequence: 103,
1343
- approvalSequence: 104,
1344
- });
1345
- }
1346
- finally {
1347
- await releaseRunSlot();
1348
- }
1349
- continue;
1350
- }
1351
- if (thread.status !== "resuming" || !this.recoveryConfig.resumeResumingRunsOnStartup) {
781
+ const handled = await recoverQueuedStartupRun(recoveryContext, thread) ||
782
+ await recoverRunningStartupRun(recoveryContext, thread) ||
783
+ await recoverResumingStartupRun(recoveryContext, thread);
784
+ if (handled) {
1352
785
  continue;
1353
786
  }
1354
- const binding = this.workspace.bindings.get(thread.agentId);
1355
- if (!binding) {
1356
- continue;
1357
- }
1358
- const recoveryIntent = await this.persistence.getRecoveryIntent(thread.threadId, thread.latestRunId);
1359
- if (!recoveryIntent || recoveryIntent.kind !== "approval-decision") {
1360
- continue;
1361
- }
1362
- if (recoveryIntent.attempts >= this.recoveryConfig.maxRecoveryAttempts) {
1363
- await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
1364
- await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
1365
- continue;
1366
- }
1367
- await this.persistence.saveRecoveryIntent(thread.threadId, thread.latestRunId, {
1368
- ...recoveryIntent,
1369
- attempts: recoveryIntent.attempts + 1,
1370
- });
1371
- await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
1372
- resumeKind: "startup-recovery",
1373
- checkpointRef: recoveryIntent.checkpointRef,
1374
- state: "resuming",
1375
- });
1376
- const history = await this.persistence.listThreadMessages(thread.threadId);
1377
- const priorHistory = history.filter((message) => message.runId !== thread.latestRunId);
1378
- const runInput = await this.loadRunInput(thread.threadId, thread.latestRunId);
1379
- const startedAt = Date.now();
1380
- try {
1381
- const actual = await this.runtimeAdapter.invoke(binding, "", thread.threadId, thread.latestRunId, recoveryIntent.resumePayload, priorHistory);
1382
- this.recordLlmSuccess(startedAt);
1383
- await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
1384
- await this.finalizeContinuedRun(binding, thread.threadId, thread.latestRunId, runInput, actual, {
1385
- previousState: "resuming",
1386
- stateSequence: 101,
1387
- approvalSequence: 102,
1388
- });
1389
- }
1390
- catch (error) {
1391
- this.recordLlmFailure(startedAt);
1392
- if (recoveryIntent.attempts + 1 >= this.recoveryConfig.maxRecoveryAttempts) {
1393
- await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
1394
- await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
1395
- await this.emit(thread.threadId, thread.latestRunId, 101, "run.state.changed", {
1396
- previousState: "resuming",
1397
- state: "failed",
1398
- checkpointRef: recoveryIntent.checkpointRef,
1399
- error: error instanceof Error ? error.message : String(error),
1400
- });
1401
- }
1402
- }
1403
787
  }
1404
788
  }
1405
789
  async reclaimExpiredClaimedRuns(nowIso = new Date().toISOString()) {