@botbotgo/agent-harness 0.0.100 → 0.0.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package-version.d.ts +1 -1
- package/dist/package-version.js +1 -1
- package/dist/persistence/sqlite-run-context-store.d.ts +22 -0
- package/dist/persistence/sqlite-run-context-store.js +64 -0
- package/dist/persistence/sqlite-run-queue-store.d.ts +41 -0
- package/dist/persistence/sqlite-run-queue-store.js +120 -0
- package/dist/persistence/sqlite-store.d.ts +2 -2
- package/dist/persistence/sqlite-store.js +31 -117
- package/dist/resource/mcp-tool-support.d.ts +21 -0
- package/dist/resource/mcp-tool-support.js +173 -0
- package/dist/resource/resource-impl.d.ts +1 -18
- package/dist/resource/resource-impl.js +79 -240
- package/dist/runtime/adapter/invoke-runtime.d.ts +22 -0
- package/dist/runtime/adapter/invoke-runtime.js +18 -0
- package/dist/runtime/adapter/stream-runtime.d.ts +46 -0
- package/dist/runtime/adapter/stream-runtime.js +93 -0
- package/dist/runtime/agent-runtime-adapter.d.ts +1 -12
- package/dist/runtime/agent-runtime-adapter.js +122 -312
- package/dist/runtime/harness/run/recovery.d.ts +42 -0
- package/dist/runtime/harness/run/recovery.js +139 -0
- package/dist/runtime/harness/run/run-operations.d.ts +50 -0
- package/dist/runtime/harness/run/run-operations.js +113 -0
- package/dist/runtime/harness/run/run-slot-acquisition.d.ts +64 -0
- package/dist/runtime/harness/run/run-slot-acquisition.js +157 -0
- package/dist/runtime/harness/run/stream-run.d.ts +53 -0
- package/dist/runtime/harness/run/stream-run.js +304 -0
- package/dist/runtime/harness.d.ts +2 -17
- package/dist/runtime/harness.js +157 -773
- package/dist/runtime/support/runtime-factories.js +2 -2
- package/dist/workspace/object-loader.d.ts +1 -8
- package/dist/workspace/object-loader.js +43 -275
- package/dist/workspace/yaml-object-reader.d.ts +15 -0
- package/dist/workspace/yaml-object-reader.js +202 -0
- package/package.json +1 -1
- package/dist/runtime/checkpoint-maintenance.d.ts +0 -1
- package/dist/runtime/checkpoint-maintenance.js +0 -1
- package/dist/runtime/file-checkpoint-saver.d.ts +0 -1
- package/dist/runtime/file-checkpoint-saver.js +0 -1
- package/dist/runtime/sqlite-maintained-checkpoint-saver.d.ts +0 -1
- package/dist/runtime/sqlite-maintained-checkpoint-saver.js +0 -1
package/dist/runtime/harness.js
CHANGED
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
import { AUTO_AGENT_ID } from "../contracts/types.js";
|
|
2
2
|
import { SqlitePersistence } from "../persistence/sqlite-store.js";
|
|
3
3
|
import { createPersistentId } from "../utils/id.js";
|
|
4
|
-
import {
|
|
5
|
-
import { normalizeUpstreamRuntimeEvent } from "./parsing/stream-event-parsing.js";
|
|
4
|
+
import { AgentRuntimeAdapter } from "./agent-runtime-adapter.js";
|
|
6
5
|
import { createResourceBackendResolver, createResourceToolResolver } from "../resource/resource.js";
|
|
7
6
|
import { EventBus } from "./harness/events/event-bus.js";
|
|
8
7
|
import { PolicyEngine } from "./harness/system/policy-engine.js";
|
|
9
8
|
import { getConcurrencyConfig, getRecoveryConfig, getRoutingDefaultAgentId, getRoutingRules, matchRoutingRule, } from "../workspace/support/workspace-ref-utils.js";
|
|
10
|
-
import { createHarnessEvent, inferRoutingBindings, renderRuntimeFailure,
|
|
9
|
+
import { createHarnessEvent, inferRoutingBindings, renderRuntimeFailure, } from "./support/harness-support.js";
|
|
11
10
|
import { ThreadMemorySync } from "./harness/system/thread-memory-sync.js";
|
|
12
11
|
import { FileBackedStore } from "./harness/system/store.js";
|
|
13
|
-
import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
|
|
12
|
+
import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./maintenance/checkpoint-maintenance.js";
|
|
14
13
|
import { RuntimeRecordMaintenanceLoop, discoverRuntimeRecordMaintenanceTargets, readRuntimeRecordMaintenanceConfig, } from "./maintenance/runtime-record-maintenance.js";
|
|
15
14
|
import { HealthMonitor } from "./harness/system/health-monitor.js";
|
|
16
15
|
import { readHealthMonitorConfig } from "./harness/system/health-monitor.js";
|
|
17
16
|
import { extractMessageText, normalizeMessageContent } from "../utils/message-content.js";
|
|
18
17
|
import { buildPersistedRunRequest, isTerminalRunState, normalizeInvocationEnvelope, normalizeRunPriority, resolveRunListeners, toPublicApprovalRecord, } from "./harness/run/helpers.js";
|
|
19
|
-
import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent,
|
|
20
|
-
import { appendAssistantMessage as appendLifecycleAssistantMessage,
|
|
21
|
-
import {
|
|
18
|
+
import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent, requestApprovalAndEmitEvent, setRunStateAndEmitEvent, } from "./harness/events/events.js";
|
|
19
|
+
import { appendAssistantMessage as appendLifecycleAssistantMessage, finalizeCancelledRun as finalizeLifecycleCancelledRun, finalizeContinuedRun as finalizeLifecycleContinuedRun, } from "./harness/run/run-lifecycle.js";
|
|
20
|
+
import { dispatchRunListeners as dispatchStreamingRunListeners, } from "./harness/events/streaming.js";
|
|
22
21
|
import { buildResumePayload as buildHarnessResumePayload, resolveApprovalRecord as resolveHarnessApprovalRecord, } from "./harness/run/resume.js";
|
|
23
|
-
import {
|
|
22
|
+
import { cancelRunOperation, resumeRun } from "./harness/run/run-operations.js";
|
|
23
|
+
import { acquireRunSlot as acquireHarnessRunSlot } from "./harness/run/run-slot-acquisition.js";
|
|
24
|
+
import { dropPendingRunSlot, enqueuePendingRunSlot } from "./harness/run/run-queue.js";
|
|
24
25
|
import { getDefaultHostAgentId, resolveSelectedAgentId } from "./harness/run/routing.js";
|
|
25
26
|
import { resolveCheckpointer, resolveEmbeddingModel, resolveStore, resolveStoreFromConfig, resolveVectorStore, } from "./harness/run/resources.js";
|
|
26
27
|
import { createToolMcpServerFromTools, serveToolsOverStdioFromHarness } from "../mcp.js";
|
|
@@ -28,6 +29,8 @@ import { getBindingAdapterKind, getBindingPrimaryTools, getBindingStoreConfig }
|
|
|
28
29
|
import { isRuntimeEntryBinding } from "./support/runtime-entry.js";
|
|
29
30
|
import { describeWorkspaceInventory, listAgentSkills as listWorkspaceAgentSkills, } from "./harness/system/inventory.js";
|
|
30
31
|
import { createDefaultHealthSnapshot, isInventoryEnabled, isThreadMemorySyncEnabled, } from "./harness/runtime-defaults.js";
|
|
32
|
+
import { recoverQueuedStartupRun, recoverResumingStartupRun, recoverRunningStartupRun, } from "./harness/run/recovery.js";
|
|
33
|
+
import { streamHarnessRun } from "./harness/run/stream-run.js";
|
|
31
34
|
export class AgentHarnessRuntime {
|
|
32
35
|
workspace;
|
|
33
36
|
runtimeAdapterOptions;
|
|
@@ -85,42 +88,30 @@ export class AgentHarnessRuntime {
|
|
|
85
88
|
getThreadSummary: (currentThreadId) => this.getSession(currentThreadId),
|
|
86
89
|
});
|
|
87
90
|
}
|
|
88
|
-
resolveStore(binding) {
|
|
89
|
-
return resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding);
|
|
90
|
-
}
|
|
91
|
-
resolveStoreFromConfig(storeConfig, runRoot) {
|
|
92
|
-
return resolveStoreFromConfig(this.stores, storeConfig, runRoot);
|
|
93
|
-
}
|
|
94
|
-
async resolveEmbeddingModel(embeddingModelRef) {
|
|
95
|
-
return resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions);
|
|
96
|
-
}
|
|
97
|
-
async resolveVectorStore(vectorStoreRef) {
|
|
98
|
-
return resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions);
|
|
99
|
-
}
|
|
100
91
|
constructor(workspace, runtimeAdapterOptions = {}) {
|
|
101
92
|
this.workspace = workspace;
|
|
102
93
|
this.runtimeAdapterOptions = runtimeAdapterOptions;
|
|
103
94
|
const runRoot = this.defaultRunRoot();
|
|
104
95
|
this.persistence = new SqlitePersistence(runRoot);
|
|
105
96
|
const defaultStoreConfig = this.listHostBindings()[0]?.harnessRuntime.store;
|
|
106
|
-
this.defaultStore = this.
|
|
97
|
+
this.defaultStore = resolveStoreFromConfig(this.stores, defaultStoreConfig, runRoot) ?? new FileBackedStore(`${runRoot}/store.json`);
|
|
107
98
|
const runtimeMemoryStoreConfig = typeof this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store === "object" &&
|
|
108
99
|
this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
|
|
109
100
|
? this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
|
|
110
101
|
: undefined;
|
|
111
|
-
this.runtimeMemoryStore = this.
|
|
102
|
+
this.runtimeMemoryStore = resolveStoreFromConfig(this.stores, runtimeMemoryStoreConfig, runRoot) ?? this.defaultStore;
|
|
112
103
|
this.resolvedRuntimeAdapterOptions = {
|
|
113
104
|
...runtimeAdapterOptions,
|
|
114
105
|
toolResolver: runtimeAdapterOptions.toolResolver ??
|
|
115
106
|
createResourceToolResolver(workspace, {
|
|
116
|
-
getStore: (binding) => this.
|
|
117
|
-
getEmbeddingModel: (embeddingModelRef) => this.
|
|
118
|
-
getVectorStore: (vectorStoreRef) => this.
|
|
107
|
+
getStore: (binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding),
|
|
108
|
+
getEmbeddingModel: (embeddingModelRef) => resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions),
|
|
109
|
+
getVectorStore: (vectorStoreRef) => resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions),
|
|
119
110
|
}),
|
|
120
111
|
checkpointerResolver: runtimeAdapterOptions.checkpointerResolver ??
|
|
121
112
|
((binding) => resolveCheckpointer(this.checkpointers, binding)),
|
|
122
113
|
storeResolver: runtimeAdapterOptions.storeResolver ??
|
|
123
|
-
((binding) => this.
|
|
114
|
+
((binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding)),
|
|
124
115
|
backendResolver: runtimeAdapterOptions.backendResolver ??
|
|
125
116
|
((binding) => createResourceBackendResolver(workspace)(binding)),
|
|
126
117
|
};
|
|
@@ -183,22 +174,12 @@ export class AgentHarnessRuntime {
|
|
|
183
174
|
}
|
|
184
175
|
return createDefaultHealthSnapshot(this.activeRunSlots, this.pendingRunSlots.length);
|
|
185
176
|
}
|
|
186
|
-
getBinding(agentId) {
|
|
187
|
-
return this.workspace.bindings.get(agentId);
|
|
188
|
-
}
|
|
189
|
-
listAgentTools(agentId) {
|
|
190
|
-
const binding = this.getBinding(agentId);
|
|
191
|
-
if (!binding) {
|
|
192
|
-
throw new Error(`Unknown agent ${agentId}`);
|
|
193
|
-
}
|
|
194
|
-
return getBindingPrimaryTools(binding);
|
|
195
|
-
}
|
|
196
177
|
resolveAgentTools(agentId) {
|
|
197
|
-
const binding = this.
|
|
178
|
+
const binding = this.workspace.bindings.get(agentId);
|
|
198
179
|
if (!binding) {
|
|
199
180
|
throw new Error(`Unknown agent ${agentId}`);
|
|
200
181
|
}
|
|
201
|
-
const compiledTools =
|
|
182
|
+
const compiledTools = getBindingPrimaryTools(binding);
|
|
202
183
|
const resolver = this.resolvedRuntimeAdapterOptions.toolResolver;
|
|
203
184
|
const resolvedTools = resolver ? resolver(compiledTools.map((tool) => tool.id), binding) : [];
|
|
204
185
|
return compiledTools.map((compiledTool, index) => ({
|
|
@@ -416,9 +397,6 @@ export class AgentHarnessRuntime {
|
|
|
416
397
|
const userTurn = history.find((message) => message.runId === runId && message.role === "user");
|
|
417
398
|
return userTurn?.content ?? "";
|
|
418
399
|
}
|
|
419
|
-
async appendAssistantMessage(threadId, runId, content) {
|
|
420
|
-
return appendLifecycleAssistantMessage(this.persistence, threadId, runId, content);
|
|
421
|
-
}
|
|
422
400
|
async getRunCancellation(runId) {
|
|
423
401
|
const control = await this.persistence.getRunControl(runId);
|
|
424
402
|
return {
|
|
@@ -426,12 +404,6 @@ export class AgentHarnessRuntime {
|
|
|
426
404
|
...(control?.cancelReason ? { reason: control.cancelReason } : {}),
|
|
427
405
|
};
|
|
428
406
|
}
|
|
429
|
-
async expirePendingApprovals(threadId, runId) {
|
|
430
|
-
return expireLifecyclePendingApprovals({
|
|
431
|
-
persistence: this.persistence,
|
|
432
|
-
emit: (currentThreadId, currentRunId, sequence, eventType, payload, source) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload, source),
|
|
433
|
-
}, threadId, runId);
|
|
434
|
-
}
|
|
435
407
|
async finalizeCancelledRun(threadId, runId, previousState, reason) {
|
|
436
408
|
return finalizeLifecycleCancelledRun({
|
|
437
409
|
persistence: this.persistence,
|
|
@@ -507,7 +479,12 @@ export class AgentHarnessRuntime {
|
|
|
507
479
|
};
|
|
508
480
|
}
|
|
509
481
|
catch (error) {
|
|
510
|
-
await
|
|
482
|
+
await emitSyntheticFallbackEvent({
|
|
483
|
+
persistence: this.persistence,
|
|
484
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
485
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
486
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
487
|
+
}, threadId, runId, agentId, error, 103);
|
|
511
488
|
await this.setRunStateAndEmit(threadId, runId, 104, "failed", {
|
|
512
489
|
previousState: previousState === "queued" ? "running" : previousState,
|
|
513
490
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -524,9 +501,6 @@ export class AgentHarnessRuntime {
|
|
|
524
501
|
await this.persistence.clearRunRequest(threadId, runId);
|
|
525
502
|
}
|
|
526
503
|
}
|
|
527
|
-
checkpointRefForState(threadId, runId, state) {
|
|
528
|
-
return getCheckpointRefForRunState(threadId, runId, state);
|
|
529
|
-
}
|
|
530
504
|
async finalizeContinuedRun(binding, threadId, runId, input, actual, options) {
|
|
531
505
|
return finalizeLifecycleContinuedRun({
|
|
532
506
|
persistence: this.persistence,
|
|
@@ -535,23 +509,6 @@ export class AgentHarnessRuntime {
|
|
|
535
509
|
requestApprovalAndEmit: (currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence) => this.requestApprovalAndEmit(currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence),
|
|
536
510
|
}, binding, threadId, runId, input, actual, options);
|
|
537
511
|
}
|
|
538
|
-
async emitOutputDeltaAndCreateItem(threadId, runId, agentId, content) {
|
|
539
|
-
return emitStreamingOutputDeltaAndCreateItem((currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload), threadId, runId, agentId, content);
|
|
540
|
-
}
|
|
541
|
-
createContentBlocksItem(threadId, runId, agentId, contentBlocks) {
|
|
542
|
-
return createStreamingContentBlocksItem(threadId, runId, agentId, contentBlocks);
|
|
543
|
-
}
|
|
544
|
-
createToolResultKey(toolName, output, isError) {
|
|
545
|
-
return createStreamingToolResultKey(toolName, output, isError);
|
|
546
|
-
}
|
|
547
|
-
async emitRunCreated(threadId, runId, payload) {
|
|
548
|
-
return emitRunCreatedEvent({
|
|
549
|
-
persistence: this.persistence,
|
|
550
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
551
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
552
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
553
|
-
}, threadId, runId, payload);
|
|
554
|
-
}
|
|
555
512
|
async setRunStateAndEmit(threadId, runId, sequence, state, options) {
|
|
556
513
|
return setRunStateAndEmitEvent({
|
|
557
514
|
persistence: this.persistence,
|
|
@@ -568,25 +525,6 @@ export class AgentHarnessRuntime {
|
|
|
568
525
|
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
569
526
|
}, threadId, runId, input, interruptContent, checkpointRef, sequence);
|
|
570
527
|
}
|
|
571
|
-
async emitSyntheticFallback(threadId, runId, selectedAgentId, error, sequence = 3) {
|
|
572
|
-
await emitSyntheticFallbackEvent({
|
|
573
|
-
persistence: this.persistence,
|
|
574
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
575
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
576
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
577
|
-
}, threadId, runId, selectedAgentId, error, sequence);
|
|
578
|
-
}
|
|
579
|
-
async persistApproval(threadId, runId, checkpointRef, input, interruptContent) {
|
|
580
|
-
return persistApproval({
|
|
581
|
-
persistence: this.persistence,
|
|
582
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
583
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
584
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
585
|
-
}, threadId, runId, checkpointRef, input, interruptContent);
|
|
586
|
-
}
|
|
587
|
-
async resolveApprovalRecord(options, thread) {
|
|
588
|
-
return resolveHarnessApprovalRecord(this.persistence, options, thread);
|
|
589
|
-
}
|
|
590
528
|
isDecisionRun(options) {
|
|
591
529
|
return "decision" in options;
|
|
592
530
|
}
|
|
@@ -596,161 +534,75 @@ export class AgentHarnessRuntime {
|
|
|
596
534
|
}
|
|
597
535
|
await listener(value);
|
|
598
536
|
}
|
|
599
|
-
async
|
|
600
|
-
|
|
601
|
-
const
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
}
|
|
605
|
-
const claimedAt = new Date().toISOString();
|
|
606
|
-
if (mode === "queue-claim") {
|
|
607
|
-
await this.persistence.claimQueuedRun({
|
|
608
|
-
threadId,
|
|
609
|
-
runId,
|
|
610
|
-
workerId: this.workerId,
|
|
611
|
-
claimedAt,
|
|
612
|
-
leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
else {
|
|
616
|
-
await this.persistence.renewRunLease({
|
|
617
|
-
runId,
|
|
618
|
-
workerId: this.workerId,
|
|
619
|
-
heartbeatAt: claimedAt,
|
|
620
|
-
leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
|
|
621
|
-
});
|
|
622
|
-
}
|
|
623
|
-
if (this.concurrencyConfig.heartbeatIntervalMs <= 0) {
|
|
624
|
-
return;
|
|
625
|
-
}
|
|
626
|
-
const timer = setInterval(() => {
|
|
627
|
-
void this.persistence.renewRunLease({
|
|
628
|
-
runId,
|
|
629
|
-
workerId: this.workerId,
|
|
630
|
-
heartbeatAt: new Date().toISOString(),
|
|
631
|
-
leaseExpiresAt: new Date(Date.now() + this.concurrencyConfig.leaseMs).toISOString(),
|
|
632
|
-
});
|
|
633
|
-
}, this.concurrencyConfig.heartbeatIntervalMs);
|
|
634
|
-
timer.unref?.();
|
|
635
|
-
stopHeartbeat = () => {
|
|
636
|
-
clearInterval(timer);
|
|
637
|
-
};
|
|
638
|
-
};
|
|
639
|
-
const releaseLease = async () => {
|
|
640
|
-
stopHeartbeat();
|
|
641
|
-
if (runId) {
|
|
642
|
-
await this.persistence.releaseRunClaim(runId);
|
|
643
|
-
}
|
|
644
|
-
};
|
|
645
|
-
const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
|
|
646
|
-
if (!maxConcurrentRuns) {
|
|
647
|
-
await beginLease("direct-heartbeat");
|
|
648
|
-
return async () => {
|
|
649
|
-
await releaseLease();
|
|
650
|
-
};
|
|
537
|
+
async prepareRunStart(options, invocation, runCreatedPayload) {
|
|
538
|
+
const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
|
|
539
|
+
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
540
|
+
if (!binding) {
|
|
541
|
+
throw new Error(`Unknown agent ${selectedAgentId}`);
|
|
651
542
|
}
|
|
652
|
-
const
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
this.activeRunSlots += 1;
|
|
656
|
-
if (threadId && runId && !useDirectHeartbeatFastPath) {
|
|
657
|
-
await this.persistence.enqueueRun({ threadId, runId, priority });
|
|
658
|
-
}
|
|
659
|
-
await beginLease(useDirectHeartbeatFastPath ? "direct-heartbeat" : "queue-claim");
|
|
660
|
-
let released = false;
|
|
661
|
-
return async () => {
|
|
662
|
-
if (released) {
|
|
663
|
-
return;
|
|
664
|
-
}
|
|
665
|
-
released = true;
|
|
666
|
-
await releaseLease();
|
|
667
|
-
this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
|
|
668
|
-
const next = shiftNextPendingRunSlot(this.pendingRunSlots);
|
|
669
|
-
void next?.activate();
|
|
670
|
-
};
|
|
543
|
+
const policyDecision = this.policyEngine.evaluate(binding);
|
|
544
|
+
if (!policyDecision.allowed) {
|
|
545
|
+
throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
|
|
671
546
|
}
|
|
672
|
-
const
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
}
|
|
690
|
-
return "activate";
|
|
547
|
+
const priority = normalizeRunPriority(options.priority);
|
|
548
|
+
const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
|
|
549
|
+
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
550
|
+
return {
|
|
551
|
+
binding,
|
|
552
|
+
selectedAgentId,
|
|
553
|
+
priority,
|
|
554
|
+
threadId,
|
|
555
|
+
runId,
|
|
556
|
+
isNewThread,
|
|
557
|
+
runCreatedEventPromise: emitRunCreatedEvent({
|
|
558
|
+
persistence: this.persistence,
|
|
559
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
560
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
561
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
562
|
+
}, threadId, runId, runCreatedPayload(binding, selectedAgentId)),
|
|
563
|
+
releaseRunSlotPromise: this.acquireRunSlot(threadId, runId, "running", priority),
|
|
691
564
|
};
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
activeRunCount: this.activeRunSlots,
|
|
712
|
-
maxConcurrentRuns,
|
|
713
|
-
priority: candidate.priority,
|
|
714
|
-
})));
|
|
715
|
-
});
|
|
716
|
-
const queuePosition = this.pendingRunSlots.findIndex((entry) => entry.runId === runId) + 1;
|
|
717
|
-
await this.setRunStateAndEmit(threadId, runId, 2, "queued", {
|
|
718
|
-
previousState: activeState,
|
|
719
|
-
});
|
|
720
|
-
await this.emit(threadId, runId, 3, "run.queued", {
|
|
721
|
-
queuePosition,
|
|
722
|
-
activeRunCount: this.activeRunSlots,
|
|
723
|
-
maxConcurrentRuns,
|
|
724
|
-
priority,
|
|
725
|
-
});
|
|
726
|
-
const slotAcquisitionResult = await slotAcquisition;
|
|
727
|
-
if (slotAcquisitionResult === "abort") {
|
|
728
|
-
return async () => undefined;
|
|
729
|
-
}
|
|
730
|
-
let released = false;
|
|
731
|
-
return async () => {
|
|
732
|
-
if (released) {
|
|
733
|
-
return;
|
|
734
|
-
}
|
|
735
|
-
released = true;
|
|
736
|
-
await releaseLease();
|
|
737
|
-
this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
|
|
738
|
-
const next = this.pendingRunSlots.shift();
|
|
739
|
-
void next?.activate();
|
|
740
|
-
};
|
|
741
|
-
}
|
|
742
|
-
let released = false;
|
|
743
|
-
return async () => {
|
|
744
|
-
if (released) {
|
|
745
|
-
return;
|
|
746
|
-
}
|
|
747
|
-
released = true;
|
|
748
|
-
await releaseLease();
|
|
749
|
-
this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
|
|
750
|
-
const next = shiftNextPendingRunSlot(this.pendingRunSlots);
|
|
751
|
-
void next?.activate();
|
|
565
|
+
}
|
|
566
|
+
createStartupRecoveryContext() {
|
|
567
|
+
return {
|
|
568
|
+
persistence: this.persistence,
|
|
569
|
+
workspace: this.workspace,
|
|
570
|
+
runtimeAdapter: this.runtimeAdapter,
|
|
571
|
+
recoveryConfig: this.recoveryConfig,
|
|
572
|
+
concurrencyConfig: this.concurrencyConfig,
|
|
573
|
+
getBinding: (agentId) => this.workspace.bindings.get(agentId),
|
|
574
|
+
acquireRunSlot: (threadId, runId, activeState = "running", priority = 0) => this.acquireRunSlot(threadId, runId, activeState, priority),
|
|
575
|
+
executeQueuedRun: (binding, input, threadId, runId, agentId, options = {}) => this.executeQueuedRun(binding, input, threadId, runId, agentId, options),
|
|
576
|
+
setRunStateAndEmit: (threadId, runId, sequence, state, options) => this.setRunStateAndEmit(threadId, runId, sequence, state, options),
|
|
577
|
+
emit: (threadId, runId, sequence, eventType, payload) => this.emit(threadId, runId, sequence, eventType, payload),
|
|
578
|
+
loadRunInput: (threadId, runId) => this.loadRunInput(threadId, runId),
|
|
579
|
+
finalizeContinuedRun: (binding, threadId, runId, input, actual, options) => this.finalizeContinuedRun(binding, threadId, runId, input, actual, options),
|
|
580
|
+
supportsRunningReplay: (binding) => this.supportsRunningReplay(binding),
|
|
581
|
+
isStaleRunningRun: (thread, nowMs = Date.now()) => this.isStaleRunningRun(thread, nowMs),
|
|
582
|
+
recordLlmSuccess: (startedAt) => this.recordLlmSuccess(startedAt),
|
|
583
|
+
recordLlmFailure: (startedAt) => this.recordLlmFailure(startedAt),
|
|
752
584
|
};
|
|
753
585
|
}
|
|
586
|
+
async acquireRunSlot(threadId, runId, activeState = "running", priority = 0) {
|
|
587
|
+
return acquireHarnessRunSlot({
|
|
588
|
+
persistence: this.persistence,
|
|
589
|
+
workerId: this.workerId,
|
|
590
|
+
concurrencyConfig: this.concurrencyConfig,
|
|
591
|
+
pendingRunSlots: this.pendingRunSlots,
|
|
592
|
+
getActiveRunSlots: () => this.activeRunSlots,
|
|
593
|
+
setActiveRunSlots: (count) => {
|
|
594
|
+
this.activeRunSlots = count;
|
|
595
|
+
},
|
|
596
|
+
enqueuePendingRunSlot: (entry) => this.enqueuePendingRunSlot(entry),
|
|
597
|
+
emit: (currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload),
|
|
598
|
+
setRunStateAndEmit: (currentThreadId, currentRunId, sequence, state, stateOptions) => this.setRunStateAndEmit(currentThreadId, currentRunId, sequence, state, stateOptions),
|
|
599
|
+
}, {
|
|
600
|
+
threadId,
|
|
601
|
+
runId,
|
|
602
|
+
activeState,
|
|
603
|
+
priority,
|
|
604
|
+
});
|
|
605
|
+
}
|
|
754
606
|
dropPendingRunSlot(runId) {
|
|
755
607
|
return dropPendingRunSlot(this.pendingRunSlots, runId);
|
|
756
608
|
}
|
|
@@ -776,32 +628,12 @@ export class AgentHarnessRuntime {
|
|
|
776
628
|
return this.dispatchRunListeners(this.streamEvents(options), resolvedListeners);
|
|
777
629
|
}
|
|
778
630
|
const invocation = normalizeInvocationEnvelope(options);
|
|
779
|
-
const selectedAgentId = await
|
|
780
|
-
|
|
781
|
-
input: options.input,
|
|
782
|
-
requestedAgentId: options.agentId,
|
|
783
|
-
threadId: options.threadId,
|
|
784
|
-
preferredHostAgentId: this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
785
|
-
getThreadSummary: (threadId) => this.getSession(threadId),
|
|
786
|
-
});
|
|
787
|
-
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
788
|
-
if (!binding) {
|
|
789
|
-
throw new Error(`Unknown agent ${selectedAgentId}`);
|
|
790
|
-
}
|
|
791
|
-
const policyDecision = this.policyEngine.evaluate(binding);
|
|
792
|
-
if (!policyDecision.allowed) {
|
|
793
|
-
throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
|
|
794
|
-
}
|
|
795
|
-
const priority = normalizeRunPriority(options.priority);
|
|
796
|
-
const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
|
|
797
|
-
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
798
|
-
const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
|
|
799
|
-
agentId: binding.agent.id,
|
|
631
|
+
const { binding, selectedAgentId, threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (activeBinding, activeSelectedAgentId) => ({
|
|
632
|
+
agentId: activeBinding.agent.id,
|
|
800
633
|
requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
|
|
801
|
-
selectedAgentId,
|
|
802
|
-
executionMode: getBindingAdapterKind(
|
|
803
|
-
});
|
|
804
|
-
const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
|
|
634
|
+
selectedAgentId: activeSelectedAgentId,
|
|
635
|
+
executionMode: getBindingAdapterKind(activeBinding),
|
|
636
|
+
}));
|
|
805
637
|
await runCreatedEventPromise;
|
|
806
638
|
const releaseRunSlot = await releaseRunSlotPromise;
|
|
807
639
|
try {
|
|
@@ -821,14 +653,7 @@ export class AgentHarnessRuntime {
|
|
|
821
653
|
}
|
|
822
654
|
async *streamEvents(options) {
|
|
823
655
|
const invocation = normalizeInvocationEnvelope(options);
|
|
824
|
-
const selectedAgentId = await resolveSelectedAgentId(
|
|
825
|
-
workspace: this.workspace,
|
|
826
|
-
input: options.input,
|
|
827
|
-
requestedAgentId: options.agentId,
|
|
828
|
-
threadId: options.threadId,
|
|
829
|
-
preferredHostAgentId: this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
830
|
-
getThreadSummary: (threadId) => this.getSession(threadId),
|
|
831
|
-
});
|
|
656
|
+
const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
|
|
832
657
|
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
833
658
|
if (!binding) {
|
|
834
659
|
const result = await this.run(options);
|
|
@@ -843,357 +668,62 @@ export class AgentHarnessRuntime {
|
|
|
843
668
|
}
|
|
844
669
|
return;
|
|
845
670
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
const priority = normalizeRunPriority(options.priority);
|
|
849
|
-
const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
|
|
850
|
-
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
851
|
-
const priorHistoryPromise = Promise.resolve(isNewThread ? [] : undefined).then((historyHint) => historyHint ?? this.loadPriorHistory(threadId, runId));
|
|
852
|
-
const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
|
|
853
|
-
agentId: selectedAgentId,
|
|
671
|
+
const { threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (_binding, activeSelectedAgentId) => ({
|
|
672
|
+
agentId: activeSelectedAgentId,
|
|
854
673
|
requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
|
|
855
|
-
selectedAgentId,
|
|
674
|
+
selectedAgentId: activeSelectedAgentId,
|
|
856
675
|
input: options.input,
|
|
857
676
|
state: "running",
|
|
677
|
+
}));
|
|
678
|
+
yield* streamHarnessRun({
|
|
679
|
+
binding,
|
|
680
|
+
input: options.input,
|
|
681
|
+
invocation,
|
|
682
|
+
threadId,
|
|
683
|
+
runId,
|
|
684
|
+
selectedAgentId,
|
|
685
|
+
isNewThread,
|
|
686
|
+
runCreatedEventPromise,
|
|
687
|
+
releaseRunSlotPromise,
|
|
688
|
+
loadPriorHistory: (currentThreadId, currentRunId) => this.loadPriorHistory(currentThreadId, currentRunId),
|
|
689
|
+
stream: (activeBinding, input, currentThreadId, priorHistory, streamOptions) => this.runtimeAdapter.stream(activeBinding, input, currentThreadId, priorHistory, streamOptions),
|
|
690
|
+
invokeWithHistory: (activeBinding, input, currentThreadId, currentRunId) => this.invokeWithHistory(activeBinding, input, currentThreadId, currentRunId),
|
|
691
|
+
emit: (currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload),
|
|
692
|
+
setRunStateAndEmit: (currentThreadId, currentRunId, sequence, state, stateOptions) => this.setRunStateAndEmit(currentThreadId, currentRunId, sequence, state, stateOptions),
|
|
693
|
+
requestApprovalAndEmit: (currentThreadId, currentRunId, input, interruptContent, checkpointRef, sequence) => this.requestApprovalAndEmit(currentThreadId, currentRunId, input, interruptContent, checkpointRef, sequence),
|
|
694
|
+
appendAssistantMessage: (currentThreadId, currentRunId, content) => appendLifecycleAssistantMessage(this.persistence, currentThreadId, currentRunId, content),
|
|
695
|
+
clearRunRequest: (currentThreadId, currentRunId) => this.persistence.clearRunRequest(currentThreadId, currentRunId),
|
|
696
|
+
emitSyntheticFallback: (currentThreadId, currentRunId, currentSelectedAgentId, error) => emitSyntheticFallbackEvent({
|
|
697
|
+
persistence: this.persistence,
|
|
698
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
699
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
700
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
701
|
+
}, currentThreadId, currentRunId, currentSelectedAgentId, error),
|
|
858
702
|
});
|
|
859
|
-
yield { type: "event", event: await runCreatedEventPromise };
|
|
860
|
-
const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
|
|
861
|
-
let releaseRunSlot = async () => undefined;
|
|
862
|
-
try {
|
|
863
|
-
try {
|
|
864
|
-
const [priorHistory, acquiredReleaseRunSlot] = await Promise.all([
|
|
865
|
-
priorHistoryPromise,
|
|
866
|
-
releaseRunSlotPromise,
|
|
867
|
-
]).then(([loadedPriorHistory, resolvedReleaseRunSlot]) => [loadedPriorHistory, resolvedReleaseRunSlot]);
|
|
868
|
-
releaseRunSlot = acquiredReleaseRunSlot;
|
|
869
|
-
let assistantOutput = "";
|
|
870
|
-
const toolErrors = [];
|
|
871
|
-
let lastToolResultKey = null;
|
|
872
|
-
for await (const chunk of this.runtimeAdapter.stream(binding, options.input, threadId, priorHistory, {
|
|
873
|
-
context: invocation.context,
|
|
874
|
-
state: invocation.state,
|
|
875
|
-
files: invocation.files,
|
|
876
|
-
runId,
|
|
877
|
-
})) {
|
|
878
|
-
if (chunk) {
|
|
879
|
-
streamActivityObserved = true;
|
|
880
|
-
const normalizedChunk = typeof chunk === "string"
|
|
881
|
-
? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
|
|
882
|
-
? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
|
|
883
|
-
: { kind: "content", content: chunk }
|
|
884
|
-
: chunk;
|
|
885
|
-
if (normalizedChunk.kind === "upstream-event") {
|
|
886
|
-
yield {
|
|
887
|
-
type: "upstream-event",
|
|
888
|
-
threadId,
|
|
889
|
-
runId,
|
|
890
|
-
agentId: selectedAgentId,
|
|
891
|
-
event: normalizedChunk.event.format === "langgraph-v2"
|
|
892
|
-
? normalizedChunk.event
|
|
893
|
-
: normalizeUpstreamRuntimeEvent(normalizedChunk.event.raw ?? normalizedChunk.event),
|
|
894
|
-
};
|
|
895
|
-
continue;
|
|
896
|
-
}
|
|
897
|
-
if (normalizedChunk.kind === "interrupt") {
|
|
898
|
-
const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
|
|
899
|
-
const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 6, "waiting_for_approval", {
|
|
900
|
-
previousState: "running",
|
|
901
|
-
checkpointRef,
|
|
902
|
-
});
|
|
903
|
-
const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 7);
|
|
904
|
-
yield {
|
|
905
|
-
type: "event",
|
|
906
|
-
event: waitingEvent,
|
|
907
|
-
};
|
|
908
|
-
yield {
|
|
909
|
-
type: "event",
|
|
910
|
-
event: approvalRequest.event,
|
|
911
|
-
};
|
|
912
|
-
yield {
|
|
913
|
-
type: "result",
|
|
914
|
-
result: {
|
|
915
|
-
threadId,
|
|
916
|
-
runId,
|
|
917
|
-
agentId: selectedAgentId,
|
|
918
|
-
state: "waiting_for_approval",
|
|
919
|
-
output: assistantOutput,
|
|
920
|
-
finalMessageText: assistantOutput,
|
|
921
|
-
interruptContent: normalizedChunk.content,
|
|
922
|
-
approvalId: approvalRequest.approval.approvalId,
|
|
923
|
-
pendingActionId: approvalRequest.approval.pendingActionId,
|
|
924
|
-
},
|
|
925
|
-
};
|
|
926
|
-
return;
|
|
927
|
-
}
|
|
928
|
-
if (normalizedChunk.kind === "reasoning") {
|
|
929
|
-
await this.emit(threadId, runId, 3, "reasoning.delta", {
|
|
930
|
-
content: normalizedChunk.content,
|
|
931
|
-
});
|
|
932
|
-
yield {
|
|
933
|
-
type: "reasoning",
|
|
934
|
-
threadId,
|
|
935
|
-
runId,
|
|
936
|
-
agentId: selectedAgentId,
|
|
937
|
-
content: normalizedChunk.content,
|
|
938
|
-
};
|
|
939
|
-
continue;
|
|
940
|
-
}
|
|
941
|
-
if (normalizedChunk.kind === "step") {
|
|
942
|
-
yield {
|
|
943
|
-
type: "step",
|
|
944
|
-
threadId,
|
|
945
|
-
runId,
|
|
946
|
-
agentId: selectedAgentId,
|
|
947
|
-
content: normalizedChunk.content,
|
|
948
|
-
};
|
|
949
|
-
continue;
|
|
950
|
-
}
|
|
951
|
-
if (normalizedChunk.kind === "tool-result") {
|
|
952
|
-
const toolResultKey = this.createToolResultKey(normalizedChunk.toolName, normalizedChunk.output, normalizedChunk.isError);
|
|
953
|
-
if (toolResultKey === lastToolResultKey) {
|
|
954
|
-
continue;
|
|
955
|
-
}
|
|
956
|
-
lastToolResultKey = toolResultKey;
|
|
957
|
-
if (normalizedChunk.isError) {
|
|
958
|
-
toolErrors.push(renderToolFailure(normalizedChunk.toolName, normalizedChunk.output));
|
|
959
|
-
}
|
|
960
|
-
yield {
|
|
961
|
-
type: "tool-result",
|
|
962
|
-
threadId,
|
|
963
|
-
runId,
|
|
964
|
-
agentId: selectedAgentId,
|
|
965
|
-
toolName: normalizedChunk.toolName,
|
|
966
|
-
output: normalizedChunk.output,
|
|
967
|
-
isError: normalizedChunk.isError,
|
|
968
|
-
};
|
|
969
|
-
continue;
|
|
970
|
-
}
|
|
971
|
-
emitted = true;
|
|
972
|
-
assistantOutput += normalizedChunk.content;
|
|
973
|
-
yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, normalizedChunk.content);
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
if (!assistantOutput && toolErrors.length > 0) {
|
|
977
|
-
assistantOutput = toolErrors.join("\n\n");
|
|
978
|
-
emitted = true;
|
|
979
|
-
yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, assistantOutput);
|
|
980
|
-
}
|
|
981
|
-
if (!assistantOutput) {
|
|
982
|
-
const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
|
|
983
|
-
if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
|
|
984
|
-
yield this.createContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
|
|
985
|
-
}
|
|
986
|
-
if (actual.output) {
|
|
987
|
-
assistantOutput = actual.output;
|
|
988
|
-
emitted = true;
|
|
989
|
-
yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
await this.appendAssistantMessage(threadId, runId, assistantOutput);
|
|
993
|
-
yield {
|
|
994
|
-
type: "result",
|
|
995
|
-
result: {
|
|
996
|
-
threadId,
|
|
997
|
-
runId,
|
|
998
|
-
agentId: selectedAgentId,
|
|
999
|
-
state: "completed",
|
|
1000
|
-
output: assistantOutput,
|
|
1001
|
-
finalMessageText: assistantOutput,
|
|
1002
|
-
},
|
|
1003
|
-
};
|
|
1004
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
|
|
1005
|
-
previousState: "running",
|
|
1006
|
-
}) };
|
|
1007
|
-
return;
|
|
1008
|
-
}
|
|
1009
|
-
catch (error) {
|
|
1010
|
-
if (emitted || streamActivityObserved) {
|
|
1011
|
-
const runtimeFailure = renderRuntimeFailure(error);
|
|
1012
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
1013
|
-
previousState: "running",
|
|
1014
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1015
|
-
}) };
|
|
1016
|
-
yield {
|
|
1017
|
-
type: "content",
|
|
1018
|
-
threadId,
|
|
1019
|
-
runId,
|
|
1020
|
-
agentId: selectedAgentId,
|
|
1021
|
-
content: runtimeFailure,
|
|
1022
|
-
};
|
|
1023
|
-
yield {
|
|
1024
|
-
type: "result",
|
|
1025
|
-
result: {
|
|
1026
|
-
threadId,
|
|
1027
|
-
runId,
|
|
1028
|
-
agentId: selectedAgentId,
|
|
1029
|
-
state: "failed",
|
|
1030
|
-
output: runtimeFailure,
|
|
1031
|
-
finalMessageText: runtimeFailure,
|
|
1032
|
-
},
|
|
1033
|
-
};
|
|
1034
|
-
return;
|
|
1035
|
-
}
|
|
1036
|
-
if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
|
|
1037
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
1038
|
-
previousState: "running",
|
|
1039
|
-
error: error.message,
|
|
1040
|
-
}) };
|
|
1041
|
-
yield {
|
|
1042
|
-
type: "content",
|
|
1043
|
-
threadId,
|
|
1044
|
-
runId,
|
|
1045
|
-
agentId: selectedAgentId,
|
|
1046
|
-
content: renderRuntimeFailure(error),
|
|
1047
|
-
};
|
|
1048
|
-
yield {
|
|
1049
|
-
type: "result",
|
|
1050
|
-
result: {
|
|
1051
|
-
threadId,
|
|
1052
|
-
runId,
|
|
1053
|
-
agentId: selectedAgentId,
|
|
1054
|
-
state: "failed",
|
|
1055
|
-
output: renderRuntimeFailure(error),
|
|
1056
|
-
finalMessageText: renderRuntimeFailure(error),
|
|
1057
|
-
},
|
|
1058
|
-
};
|
|
1059
|
-
return;
|
|
1060
|
-
}
|
|
1061
|
-
try {
|
|
1062
|
-
const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
|
|
1063
|
-
await this.appendAssistantMessage(threadId, runId, actual.output);
|
|
1064
|
-
if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
|
|
1065
|
-
yield this.createContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
|
|
1066
|
-
}
|
|
1067
|
-
if (actual.output) {
|
|
1068
|
-
yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, actual.output);
|
|
1069
|
-
}
|
|
1070
|
-
yield {
|
|
1071
|
-
type: "result",
|
|
1072
|
-
result: {
|
|
1073
|
-
...actual,
|
|
1074
|
-
threadId,
|
|
1075
|
-
runId,
|
|
1076
|
-
agentId: selectedAgentId,
|
|
1077
|
-
},
|
|
1078
|
-
};
|
|
1079
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
|
|
1080
|
-
previousState: "running",
|
|
1081
|
-
}) };
|
|
1082
|
-
return;
|
|
1083
|
-
}
|
|
1084
|
-
catch (invokeError) {
|
|
1085
|
-
await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
|
|
1086
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
1087
|
-
previousState: "running",
|
|
1088
|
-
error: invokeError instanceof Error ? invokeError.message : String(invokeError),
|
|
1089
|
-
}) };
|
|
1090
|
-
yield {
|
|
1091
|
-
type: "content",
|
|
1092
|
-
threadId,
|
|
1093
|
-
runId,
|
|
1094
|
-
agentId: selectedAgentId,
|
|
1095
|
-
content: renderRuntimeFailure(invokeError),
|
|
1096
|
-
};
|
|
1097
|
-
yield {
|
|
1098
|
-
type: "result",
|
|
1099
|
-
result: {
|
|
1100
|
-
threadId,
|
|
1101
|
-
runId,
|
|
1102
|
-
agentId: selectedAgentId,
|
|
1103
|
-
state: "failed",
|
|
1104
|
-
output: renderRuntimeFailure(invokeError),
|
|
1105
|
-
finalMessageText: renderRuntimeFailure(invokeError),
|
|
1106
|
-
},
|
|
1107
|
-
};
|
|
1108
|
-
return;
|
|
1109
|
-
}
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
finally {
|
|
1113
|
-
await this.persistence.clearRunRequest(threadId, runId);
|
|
1114
|
-
await releaseRunSlot();
|
|
1115
|
-
}
|
|
1116
703
|
}
|
|
1117
704
|
async resume(options) {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
:
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
const releaseRunSlot = await this.acquireRunSlot(threadId, runId, "resuming", await this.resolvePersistedRunPriority(threadId, runId));
|
|
1141
|
-
try {
|
|
1142
|
-
await this.persistence.saveRecoveryIntent(threadId, runId, {
|
|
1143
|
-
kind: "approval-decision",
|
|
1144
|
-
savedAt: new Date().toISOString(),
|
|
1145
|
-
checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
|
|
1146
|
-
resumePayload,
|
|
1147
|
-
attempts: 0,
|
|
1148
|
-
});
|
|
1149
|
-
await this.emit(threadId, runId, 5, "run.resumed", {
|
|
1150
|
-
resumeKind: "cross-restart",
|
|
1151
|
-
checkpointRef: `checkpoints/${threadId}/${runId}/cp-1`,
|
|
1152
|
-
state: "resuming",
|
|
1153
|
-
approvalId: approval.approvalId,
|
|
1154
|
-
pendingActionId: approval.pendingActionId,
|
|
1155
|
-
});
|
|
1156
|
-
await this.persistence.resolveApproval(threadId, runId, approval.approvalId, options.decision === "reject" ? "rejected" : options.decision === "edit" ? "edited" : "approved");
|
|
1157
|
-
await this.emit(threadId, runId, 6, "approval.resolved", {
|
|
1158
|
-
approvalId: approval.approvalId,
|
|
1159
|
-
pendingActionId: approval.pendingActionId,
|
|
1160
|
-
decision: options.decision ?? "approve",
|
|
1161
|
-
toolName: approval.toolName,
|
|
1162
|
-
});
|
|
1163
|
-
const history = await this.persistence.listThreadMessages(threadId);
|
|
1164
|
-
const priorHistory = history.filter((message) => message.runId !== runId);
|
|
1165
|
-
const runInput = await this.loadRunInput(threadId, runId);
|
|
1166
|
-
const startedAt = Date.now();
|
|
1167
|
-
try {
|
|
1168
|
-
const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
|
|
1169
|
-
this.recordLlmSuccess(startedAt);
|
|
1170
|
-
const cancelledAfterInvoke = await this.getRunCancellation(runId);
|
|
1171
|
-
if (cancelledAfterInvoke.requested) {
|
|
1172
|
-
return this.finalizeCancelledRun(threadId, runId, "resuming", cancelledAfterInvoke.reason);
|
|
1173
|
-
}
|
|
1174
|
-
await this.persistence.clearRecoveryIntent(threadId, runId);
|
|
1175
|
-
const finalized = await this.finalizeContinuedRun(binding, threadId, runId, runInput, actual, {
|
|
1176
|
-
previousState: "resuming",
|
|
1177
|
-
stateSequence: 7,
|
|
1178
|
-
approvalSequence: 8,
|
|
1179
|
-
});
|
|
1180
|
-
return {
|
|
1181
|
-
...finalized,
|
|
1182
|
-
approvalId: finalized.approvalId ?? approval.approvalId,
|
|
1183
|
-
pendingActionId: finalized.pendingActionId ?? approval.pendingActionId,
|
|
1184
|
-
};
|
|
1185
|
-
}
|
|
1186
|
-
catch (error) {
|
|
1187
|
-
this.recordLlmFailure(startedAt);
|
|
1188
|
-
throw error;
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
finally {
|
|
1192
|
-
await releaseRunSlot();
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
buildResumePayload(binding, approval, options) {
|
|
1196
|
-
return buildHarnessResumePayload(binding, approval, options);
|
|
705
|
+
return resumeRun({
|
|
706
|
+
getApprovalById: (approvalId) => this.persistence.getApproval(approvalId),
|
|
707
|
+
getSession: (threadId) => this.getSession(threadId),
|
|
708
|
+
resolveApprovalRecord: (resumeOptions, thread) => resolveHarnessApprovalRecord(this.persistence, resumeOptions, thread),
|
|
709
|
+
getBinding: (agentId) => this.workspace.bindings.get(agentId),
|
|
710
|
+
buildResumePayload: (binding, approval, resumeOptions) => buildHarnessResumePayload(binding, approval, resumeOptions),
|
|
711
|
+
getRunCancellation: (runId) => this.getRunCancellation(runId),
|
|
712
|
+
finalizeCancelledRun: (threadId, runId, previousState, reason) => this.finalizeCancelledRun(threadId, runId, previousState, reason),
|
|
713
|
+
setRunState: (threadId, runId, state, checkpointRef) => this.persistence.setRunState(threadId, runId, state, checkpointRef),
|
|
714
|
+
acquireRunSlot: (threadId, runId, activeState, priority) => this.acquireRunSlot(threadId, runId, activeState, priority),
|
|
715
|
+
resolvePersistedRunPriority: (threadId, runId) => this.resolvePersistedRunPriority(threadId, runId),
|
|
716
|
+
saveRecoveryIntent: (threadId, runId, payload) => this.persistence.saveRecoveryIntent(threadId, runId, payload),
|
|
717
|
+
emit: (threadId, runId, sequence, eventType, payload) => this.emit(threadId, runId, sequence, eventType, payload),
|
|
718
|
+
resolveApproval: (threadId, runId, approvalId, resolution) => this.persistence.resolveApproval(threadId, runId, approvalId, resolution),
|
|
719
|
+
listThreadMessages: (threadId) => this.persistence.listThreadMessages(threadId),
|
|
720
|
+
loadRunInput: (threadId, runId) => this.loadRunInput(threadId, runId),
|
|
721
|
+
invoke: (binding, input, threadId, runId, resumePayload, priorHistory) => this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory),
|
|
722
|
+
recordLlmSuccess: (startedAt) => this.recordLlmSuccess(startedAt),
|
|
723
|
+
recordLlmFailure: (startedAt) => this.recordLlmFailure(startedAt),
|
|
724
|
+
clearRecoveryIntent: (threadId, runId) => this.persistence.clearRecoveryIntent(threadId, runId),
|
|
725
|
+
finalizeContinuedRun: (binding, threadId, runId, input, actual, operationOptions) => this.finalizeContinuedRun(binding, threadId, runId, input, actual, operationOptions),
|
|
726
|
+
}, options);
|
|
1197
727
|
}
|
|
1198
728
|
async restartConversation(options) {
|
|
1199
729
|
const thread = await this.getSession(options.threadId);
|
|
@@ -1232,37 +762,13 @@ export class AgentHarnessRuntime {
|
|
|
1232
762
|
await this.close();
|
|
1233
763
|
}
|
|
1234
764
|
async cancelRun(options) {
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
runId: run.runId,
|
|
1243
|
-
agentId: run.agentId,
|
|
1244
|
-
state: run.state,
|
|
1245
|
-
output: run.state,
|
|
1246
|
-
};
|
|
1247
|
-
}
|
|
1248
|
-
await this.persistence.requestRunCancel(run.runId, options.reason);
|
|
1249
|
-
if (run.state === "queued" || run.state === "waiting_for_approval" || run.state === "claimed") {
|
|
1250
|
-
if (run.state === "queued") {
|
|
1251
|
-
this.dropPendingRunSlot(run.runId);
|
|
1252
|
-
}
|
|
1253
|
-
return this.finalizeCancelledRun(run.threadId, run.runId, run.state, options.reason);
|
|
1254
|
-
}
|
|
1255
|
-
await this.setRunStateAndEmit(run.threadId, run.runId, 103, "cancelling", {
|
|
1256
|
-
previousState: run.state,
|
|
1257
|
-
...(options.reason ? { error: options.reason } : {}),
|
|
1258
|
-
});
|
|
1259
|
-
return {
|
|
1260
|
-
threadId: run.threadId,
|
|
1261
|
-
runId: run.runId,
|
|
1262
|
-
agentId: run.agentId,
|
|
1263
|
-
state: "cancelling",
|
|
1264
|
-
output: options.reason ? `cancelling: ${options.reason}` : "cancelling",
|
|
1265
|
-
};
|
|
765
|
+
return cancelRunOperation({
|
|
766
|
+
getRun: (runId) => this.persistence.getRun(runId),
|
|
767
|
+
requestRunCancel: (runId, reason) => this.persistence.requestRunCancel(runId, reason),
|
|
768
|
+
dropPendingRunSlot: (runId) => this.dropPendingRunSlot(runId),
|
|
769
|
+
finalizeCancelledRun: (threadId, runId, previousState, reason) => this.finalizeCancelledRun(threadId, runId, previousState, reason),
|
|
770
|
+
setRunStateAndEmit: (threadId, runId, sequence, state, stateOptions) => this.setRunStateAndEmit(threadId, runId, sequence, state, stateOptions),
|
|
771
|
+
}, options);
|
|
1266
772
|
}
|
|
1267
773
|
async recoverStartupRuns() {
|
|
1268
774
|
if (!this.recoveryConfig.enabled) {
|
|
@@ -1270,136 +776,14 @@ export class AgentHarnessRuntime {
|
|
|
1270
776
|
}
|
|
1271
777
|
await this.reclaimExpiredClaimedRuns();
|
|
1272
778
|
const threads = await this.persistence.listSessions();
|
|
779
|
+
const recoveryContext = this.createStartupRecoveryContext();
|
|
1273
780
|
for (const thread of threads) {
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
continue;
|
|
1279
|
-
}
|
|
1280
|
-
const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
|
|
1281
|
-
if (!request) {
|
|
1282
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1283
|
-
previousState: "queued",
|
|
1284
|
-
error: "missing persisted run request for queued run recovery",
|
|
1285
|
-
});
|
|
1286
|
-
continue;
|
|
1287
|
-
}
|
|
1288
|
-
const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
|
|
1289
|
-
try {
|
|
1290
|
-
await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
|
|
1291
|
-
context: request.invocation?.context,
|
|
1292
|
-
state: request.invocation?.inputs,
|
|
1293
|
-
files: request.invocation?.attachments,
|
|
1294
|
-
previousState: "queued",
|
|
1295
|
-
stateSequence: 103,
|
|
1296
|
-
approvalSequence: 104,
|
|
1297
|
-
});
|
|
1298
|
-
}
|
|
1299
|
-
finally {
|
|
1300
|
-
await releaseRunSlot();
|
|
1301
|
-
}
|
|
1302
|
-
continue;
|
|
1303
|
-
}
|
|
1304
|
-
if (thread.status === "running") {
|
|
1305
|
-
const isStale = await this.isStaleRunningRun(thread);
|
|
1306
|
-
if (!isStale) {
|
|
1307
|
-
continue;
|
|
1308
|
-
}
|
|
1309
|
-
const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
|
|
1310
|
-
const binding = this.workspace.bindings.get(runMeta.agentId);
|
|
1311
|
-
if (!binding) {
|
|
1312
|
-
continue;
|
|
1313
|
-
}
|
|
1314
|
-
if (!this.supportsRunningReplay(binding)) {
|
|
1315
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1316
|
-
previousState: "running",
|
|
1317
|
-
error: "stale running run cannot be replayed safely",
|
|
1318
|
-
});
|
|
1319
|
-
await this.persistence.releaseRunClaim(thread.latestRunId);
|
|
1320
|
-
continue;
|
|
1321
|
-
}
|
|
1322
|
-
const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
|
|
1323
|
-
if (!request) {
|
|
1324
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1325
|
-
previousState: "running",
|
|
1326
|
-
error: "missing persisted run request for stale running run recovery",
|
|
1327
|
-
});
|
|
1328
|
-
await this.persistence.releaseRunClaim(thread.latestRunId);
|
|
1329
|
-
continue;
|
|
1330
|
-
}
|
|
1331
|
-
const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
|
|
1332
|
-
try {
|
|
1333
|
-
await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
|
|
1334
|
-
resumeKind: "startup-running-recovery",
|
|
1335
|
-
state: "running",
|
|
1336
|
-
});
|
|
1337
|
-
await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
|
|
1338
|
-
context: request.invocation?.context,
|
|
1339
|
-
state: request.invocation?.inputs,
|
|
1340
|
-
files: request.invocation?.attachments,
|
|
1341
|
-
previousState: "running",
|
|
1342
|
-
stateSequence: 103,
|
|
1343
|
-
approvalSequence: 104,
|
|
1344
|
-
});
|
|
1345
|
-
}
|
|
1346
|
-
finally {
|
|
1347
|
-
await releaseRunSlot();
|
|
1348
|
-
}
|
|
1349
|
-
continue;
|
|
1350
|
-
}
|
|
1351
|
-
if (thread.status !== "resuming" || !this.recoveryConfig.resumeResumingRunsOnStartup) {
|
|
781
|
+
const handled = await recoverQueuedStartupRun(recoveryContext, thread) ||
|
|
782
|
+
await recoverRunningStartupRun(recoveryContext, thread) ||
|
|
783
|
+
await recoverResumingStartupRun(recoveryContext, thread);
|
|
784
|
+
if (handled) {
|
|
1352
785
|
continue;
|
|
1353
786
|
}
|
|
1354
|
-
const binding = this.workspace.bindings.get(thread.agentId);
|
|
1355
|
-
if (!binding) {
|
|
1356
|
-
continue;
|
|
1357
|
-
}
|
|
1358
|
-
const recoveryIntent = await this.persistence.getRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1359
|
-
if (!recoveryIntent || recoveryIntent.kind !== "approval-decision") {
|
|
1360
|
-
continue;
|
|
1361
|
-
}
|
|
1362
|
-
if (recoveryIntent.attempts >= this.recoveryConfig.maxRecoveryAttempts) {
|
|
1363
|
-
await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
|
|
1364
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1365
|
-
continue;
|
|
1366
|
-
}
|
|
1367
|
-
await this.persistence.saveRecoveryIntent(thread.threadId, thread.latestRunId, {
|
|
1368
|
-
...recoveryIntent,
|
|
1369
|
-
attempts: recoveryIntent.attempts + 1,
|
|
1370
|
-
});
|
|
1371
|
-
await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
|
|
1372
|
-
resumeKind: "startup-recovery",
|
|
1373
|
-
checkpointRef: recoveryIntent.checkpointRef,
|
|
1374
|
-
state: "resuming",
|
|
1375
|
-
});
|
|
1376
|
-
const history = await this.persistence.listThreadMessages(thread.threadId);
|
|
1377
|
-
const priorHistory = history.filter((message) => message.runId !== thread.latestRunId);
|
|
1378
|
-
const runInput = await this.loadRunInput(thread.threadId, thread.latestRunId);
|
|
1379
|
-
const startedAt = Date.now();
|
|
1380
|
-
try {
|
|
1381
|
-
const actual = await this.runtimeAdapter.invoke(binding, "", thread.threadId, thread.latestRunId, recoveryIntent.resumePayload, priorHistory);
|
|
1382
|
-
this.recordLlmSuccess(startedAt);
|
|
1383
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1384
|
-
await this.finalizeContinuedRun(binding, thread.threadId, thread.latestRunId, runInput, actual, {
|
|
1385
|
-
previousState: "resuming",
|
|
1386
|
-
stateSequence: 101,
|
|
1387
|
-
approvalSequence: 102,
|
|
1388
|
-
});
|
|
1389
|
-
}
|
|
1390
|
-
catch (error) {
|
|
1391
|
-
this.recordLlmFailure(startedAt);
|
|
1392
|
-
if (recoveryIntent.attempts + 1 >= this.recoveryConfig.maxRecoveryAttempts) {
|
|
1393
|
-
await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
|
|
1394
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1395
|
-
await this.emit(thread.threadId, thread.latestRunId, 101, "run.state.changed", {
|
|
1396
|
-
previousState: "resuming",
|
|
1397
|
-
state: "failed",
|
|
1398
|
-
checkpointRef: recoveryIntent.checkpointRef,
|
|
1399
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1400
|
-
});
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
787
|
}
|
|
1404
788
|
}
|
|
1405
789
|
async reclaimExpiredClaimedRuns(nowIso = new Date().toISOString()) {
|