@botbotgo/agent-harness 0.0.99 → 0.0.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -6
- package/README.zh.md +2 -2
- package/dist/benchmark/upstream-runtime-ab-benchmark.d.ts +1 -1
- package/dist/benchmark/upstream-runtime-ab-benchmark.js +1 -2
- package/dist/contracts/core.d.ts +2 -2
- package/dist/contracts/runtime.d.ts +1 -5
- package/dist/package-version.d.ts +1 -1
- package/dist/package-version.js +1 -1
- package/dist/resource/resource-impl.js +78 -76
- package/dist/runtime/adapter/deepagent-runnable-config.d.ts +30 -0
- package/dist/runtime/adapter/deepagent-runnable-config.js +22 -0
- package/dist/runtime/adapter/index.d.ts +0 -2
- package/dist/runtime/adapter/index.js +0 -2
- package/dist/runtime/adapter/invocation-result.d.ts +13 -0
- package/dist/runtime/adapter/invocation-result.js +40 -0
- package/dist/runtime/adapter/langchain-runnable-config.d.ts +25 -0
- package/dist/runtime/adapter/langchain-runnable-config.js +19 -0
- package/dist/runtime/adapter/local-tool-invocation.d.ts +23 -0
- package/dist/runtime/adapter/local-tool-invocation.js +64 -0
- package/dist/runtime/adapter/runtime-adapter-support.d.ts +18 -0
- package/dist/runtime/adapter/runtime-adapter-support.js +54 -0
- package/dist/runtime/adapter/stream-event-projection.d.ts +19 -0
- package/dist/runtime/adapter/stream-event-projection.js +79 -0
- package/dist/runtime/adapter/stream-text-consumption.d.ts +4 -0
- package/dist/runtime/adapter/stream-text-consumption.js +18 -0
- package/dist/runtime/adapter/tool/builtin-middleware-tools.d.ts +64 -0
- package/dist/runtime/adapter/tool/builtin-middleware-tools.js +144 -0
- package/dist/runtime/adapter/tool/tool-replay.d.ts +18 -0
- package/dist/runtime/adapter/tool/tool-replay.js +26 -0
- package/dist/runtime/agent-runtime-adapter.d.ts +2 -54
- package/dist/runtime/agent-runtime-adapter.js +122 -1568
- package/dist/runtime/harness/run/helpers.js +2 -8
- package/dist/runtime/harness/run/recovery.d.ts +42 -0
- package/dist/runtime/harness/run/recovery.js +139 -0
- package/dist/runtime/harness/run/routing.d.ts +1 -3
- package/dist/runtime/harness/run/routing.js +2 -25
- package/dist/runtime/harness/run/run-lifecycle.d.ts +0 -11
- package/dist/runtime/harness/run/run-lifecycle.js +7 -50
- package/dist/runtime/harness/runtime-defaults.d.ts +4 -0
- package/dist/runtime/harness/runtime-defaults.js +39 -0
- package/dist/runtime/harness/system/inventory.js +2 -1
- package/dist/runtime/harness/system/skill-requirements.d.ts +1 -0
- package/dist/runtime/harness.d.ts +5 -24
- package/dist/runtime/harness.js +356 -536
- package/dist/runtime/index.d.ts +1 -12
- package/dist/runtime/index.js +1 -12
- package/dist/runtime/support/compiled-binding.d.ts +0 -2
- package/dist/runtime/support/compiled-binding.js +3 -22
- package/dist/runtime/support/harness-support.d.ts +0 -11
- package/dist/runtime/support/harness-support.js +1 -44
- package/dist/runtime/support/index.d.ts +1 -1
- package/dist/runtime/support/index.js +1 -1
- package/dist/runtime/support/runtime-factories.js +2 -2
- package/dist/workspace/agent-binding-compiler.js +9 -93
- package/dist/workspace/index.d.ts +0 -5
- package/dist/workspace/index.js +0 -5
- package/dist/workspace/object-loader.js +44 -99
- package/dist/workspace/support/agent-capabilities.js +2 -2
- package/dist/workspace/support/workspace-ref-utils.d.ts +0 -2
- package/dist/workspace/support/workspace-ref-utils.js +0 -17
- package/dist/workspace/validate.js +1 -1
- package/package.json +1 -1
- package/dist/config/workflows/langgraph-workflows.yaml +0 -570
- package/dist/config/workflows/runtime-profiles.yaml +0 -94
- package/dist/runtime/adapter/langgraph/presets.d.ts +0 -25
- package/dist/runtime/adapter/langgraph/presets.js +0 -165
- package/dist/runtime/adapter/langgraph/profiles.d.ts +0 -6
- package/dist/runtime/adapter/langgraph/profiles.js +0 -206
- package/dist/runtime/checkpoint-maintenance.d.ts +0 -1
- package/dist/runtime/checkpoint-maintenance.js +0 -1
- package/dist/runtime/file-checkpoint-saver.d.ts +0 -1
- package/dist/runtime/file-checkpoint-saver.js +0 -1
- package/dist/runtime/sqlite-maintained-checkpoint-saver.d.ts +0 -1
- package/dist/runtime/sqlite-maintained-checkpoint-saver.js +0 -1
package/dist/runtime/harness.js
CHANGED
|
@@ -6,26 +6,29 @@ import { normalizeUpstreamRuntimeEvent } from "./parsing/stream-event-parsing.js
|
|
|
6
6
|
import { createResourceBackendResolver, createResourceToolResolver } from "../resource/resource.js";
|
|
7
7
|
import { EventBus } from "./harness/events/event-bus.js";
|
|
8
8
|
import { PolicyEngine } from "./harness/system/policy-engine.js";
|
|
9
|
-
import { getConcurrencyConfig, getRecoveryConfig, getRoutingDefaultAgentId, getRoutingRules,
|
|
9
|
+
import { getConcurrencyConfig, getRecoveryConfig, getRoutingDefaultAgentId, getRoutingRules, matchRoutingRule, } from "../workspace/support/workspace-ref-utils.js";
|
|
10
10
|
import { createHarnessEvent, inferRoutingBindings, renderRuntimeFailure, renderToolFailure, } from "./support/harness-support.js";
|
|
11
11
|
import { ThreadMemorySync } from "./harness/system/thread-memory-sync.js";
|
|
12
12
|
import { FileBackedStore } from "./harness/system/store.js";
|
|
13
|
-
import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
|
|
13
|
+
import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./maintenance/checkpoint-maintenance.js";
|
|
14
14
|
import { RuntimeRecordMaintenanceLoop, discoverRuntimeRecordMaintenanceTargets, readRuntimeRecordMaintenanceConfig, } from "./maintenance/runtime-record-maintenance.js";
|
|
15
15
|
import { HealthMonitor } from "./harness/system/health-monitor.js";
|
|
16
|
+
import { readHealthMonitorConfig } from "./harness/system/health-monitor.js";
|
|
16
17
|
import { extractMessageText, normalizeMessageContent } from "../utils/message-content.js";
|
|
17
18
|
import { buildPersistedRunRequest, isTerminalRunState, normalizeInvocationEnvelope, normalizeRunPriority, resolveRunListeners, toPublicApprovalRecord, } from "./harness/run/helpers.js";
|
|
18
|
-
import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent,
|
|
19
|
-
import { appendAssistantMessage as appendLifecycleAssistantMessage,
|
|
19
|
+
import { emitHarnessEvent, emitRunCreatedEvent, emitSyntheticFallbackEvent, requestApprovalAndEmitEvent, setRunStateAndEmitEvent, } from "./harness/events/events.js";
|
|
20
|
+
import { appendAssistantMessage as appendLifecycleAssistantMessage, finalizeCancelledRun as finalizeLifecycleCancelledRun, finalizeContinuedRun as finalizeLifecycleContinuedRun, } from "./harness/run/run-lifecycle.js";
|
|
20
21
|
import { createContentBlocksItem as createStreamingContentBlocksItem, createToolResultKey as createStreamingToolResultKey, dispatchRunListeners as dispatchStreamingRunListeners, emitOutputDeltaAndCreateItem as emitStreamingOutputDeltaAndCreateItem, } from "./harness/events/streaming.js";
|
|
21
22
|
import { buildResumePayload as buildHarnessResumePayload, resolveApprovalRecord as resolveHarnessApprovalRecord, } from "./harness/run/resume.js";
|
|
22
23
|
import { dropPendingRunSlot, enqueuePendingRunSlot, shiftNextPendingRunSlot } from "./harness/run/run-queue.js";
|
|
23
|
-
import {
|
|
24
|
+
import { getDefaultHostAgentId, resolveSelectedAgentId } from "./harness/run/routing.js";
|
|
24
25
|
import { resolveCheckpointer, resolveEmbeddingModel, resolveStore, resolveStoreFromConfig, resolveVectorStore, } from "./harness/run/resources.js";
|
|
25
26
|
import { createToolMcpServerFromTools, serveToolsOverStdioFromHarness } from "../mcp.js";
|
|
26
27
|
import { getBindingAdapterKind, getBindingPrimaryTools, getBindingStoreConfig } from "./support/compiled-binding.js";
|
|
27
28
|
import { isRuntimeEntryBinding } from "./support/runtime-entry.js";
|
|
28
29
|
import { describeWorkspaceInventory, listAgentSkills as listWorkspaceAgentSkills, } from "./harness/system/inventory.js";
|
|
30
|
+
import { createDefaultHealthSnapshot, isInventoryEnabled, isThreadMemorySyncEnabled, } from "./harness/runtime-defaults.js";
|
|
31
|
+
import { recoverQueuedStartupRun, recoverResumingStartupRun, recoverRunningStartupRun, } from "./harness/run/recovery.js";
|
|
29
32
|
export class AgentHarnessRuntime {
|
|
30
33
|
workspace;
|
|
31
34
|
runtimeAdapterOptions;
|
|
@@ -47,10 +50,8 @@ export class AgentHarnessRuntime {
|
|
|
47
50
|
vectorStores = new Map();
|
|
48
51
|
defaultStore;
|
|
49
52
|
runtimeMemoryStore;
|
|
50
|
-
routingSystemPrompt;
|
|
51
53
|
routingRules;
|
|
52
54
|
routingDefaultAgentId;
|
|
53
|
-
modelRoutingEnabled;
|
|
54
55
|
threadMemorySync;
|
|
55
56
|
unregisterThreadMemorySync;
|
|
56
57
|
resolvedRuntimeAdapterOptions;
|
|
@@ -72,15 +73,8 @@ export class AgentHarnessRuntime {
|
|
|
72
73
|
return (this.listHostBindings()[0]?.harnessRuntime.runRoot ??
|
|
73
74
|
`${this.workspace.workspaceRoot}/run-data`);
|
|
74
75
|
}
|
|
75
|
-
heuristicRoute(input) {
|
|
76
|
-
return heuristicHostRoute(this.workspace, input);
|
|
77
|
-
}
|
|
78
76
|
getDefaultHostAgentId() {
|
|
79
|
-
return getDefaultHostAgentId(this.workspace, AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID);
|
|
80
|
-
}
|
|
81
|
-
async buildRoutingInput(input, threadId) {
|
|
82
|
-
const history = threadId ? await this.persistence.listThreadMessages(threadId) : [];
|
|
83
|
-
return buildRoutingInput(input, history);
|
|
77
|
+
return getDefaultHostAgentId(this.workspace, this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID);
|
|
84
78
|
}
|
|
85
79
|
async resolveSelectedAgentId(input, requestedAgentId, threadId) {
|
|
86
80
|
return resolveSelectedAgentId({
|
|
@@ -88,56 +82,48 @@ export class AgentHarnessRuntime {
|
|
|
88
82
|
input,
|
|
89
83
|
requestedAgentId,
|
|
90
84
|
threadId,
|
|
91
|
-
preferredHostAgentId: AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
85
|
+
preferredHostAgentId: this.routingDefaultAgentId ?? AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
92
86
|
getThreadSummary: (currentThreadId) => this.getSession(currentThreadId),
|
|
93
87
|
});
|
|
94
88
|
}
|
|
95
|
-
resolveStore(binding) {
|
|
96
|
-
return resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding);
|
|
97
|
-
}
|
|
98
|
-
resolveStoreFromConfig(storeConfig, runRoot) {
|
|
99
|
-
return resolveStoreFromConfig(this.stores, storeConfig, runRoot);
|
|
100
|
-
}
|
|
101
|
-
async resolveEmbeddingModel(embeddingModelRef) {
|
|
102
|
-
return resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions);
|
|
103
|
-
}
|
|
104
|
-
async resolveVectorStore(vectorStoreRef) {
|
|
105
|
-
return resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions);
|
|
106
|
-
}
|
|
107
89
|
constructor(workspace, runtimeAdapterOptions = {}) {
|
|
108
90
|
this.workspace = workspace;
|
|
109
91
|
this.runtimeAdapterOptions = runtimeAdapterOptions;
|
|
110
92
|
const runRoot = this.defaultRunRoot();
|
|
111
93
|
this.persistence = new SqlitePersistence(runRoot);
|
|
112
94
|
const defaultStoreConfig = this.listHostBindings()[0]?.harnessRuntime.store;
|
|
113
|
-
this.defaultStore = this.
|
|
95
|
+
this.defaultStore = resolveStoreFromConfig(this.stores, defaultStoreConfig, runRoot) ?? new FileBackedStore(`${runRoot}/store.json`);
|
|
114
96
|
const runtimeMemoryStoreConfig = typeof this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store === "object" &&
|
|
115
97
|
this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
|
|
116
98
|
? this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store
|
|
117
99
|
: undefined;
|
|
118
|
-
this.runtimeMemoryStore = this.
|
|
100
|
+
this.runtimeMemoryStore = resolveStoreFromConfig(this.stores, runtimeMemoryStoreConfig, runRoot) ?? this.defaultStore;
|
|
119
101
|
this.resolvedRuntimeAdapterOptions = {
|
|
120
102
|
...runtimeAdapterOptions,
|
|
121
103
|
toolResolver: runtimeAdapterOptions.toolResolver ??
|
|
122
104
|
createResourceToolResolver(workspace, {
|
|
123
|
-
getStore: (binding) => this.
|
|
124
|
-
getEmbeddingModel: (embeddingModelRef) => this.
|
|
125
|
-
getVectorStore: (vectorStoreRef) => this.
|
|
105
|
+
getStore: (binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding),
|
|
106
|
+
getEmbeddingModel: (embeddingModelRef) => resolveEmbeddingModel(this.workspace, this.embeddingModels, embeddingModelRef, this.runtimeAdapterOptions),
|
|
107
|
+
getVectorStore: (vectorStoreRef) => resolveVectorStore(this.workspace, this.vectorStores, vectorStoreRef, this.runtimeAdapterOptions),
|
|
126
108
|
}),
|
|
127
109
|
checkpointerResolver: runtimeAdapterOptions.checkpointerResolver ??
|
|
128
110
|
((binding) => resolveCheckpointer(this.checkpointers, binding)),
|
|
129
111
|
storeResolver: runtimeAdapterOptions.storeResolver ??
|
|
130
|
-
((binding) => this.
|
|
112
|
+
((binding) => resolveStore(this.stores, this.defaultStore, this.defaultRunRoot(), (currentBinding) => currentBinding ? getBindingStoreConfig(currentBinding) : undefined, binding)),
|
|
131
113
|
backendResolver: runtimeAdapterOptions.backendResolver ??
|
|
132
114
|
((binding) => createResourceBackendResolver(workspace)(binding)),
|
|
133
115
|
};
|
|
134
116
|
this.runtimeAdapter = new AgentRuntimeAdapter(this.resolvedRuntimeAdapterOptions);
|
|
135
|
-
this.routingSystemPrompt = getRoutingSystemPrompt(workspace.refs);
|
|
136
117
|
this.routingRules = getRoutingRules(workspace.refs);
|
|
137
118
|
this.routingDefaultAgentId = getRoutingDefaultAgentId(workspace.refs);
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
119
|
+
if (isThreadMemorySyncEnabled(workspace)) {
|
|
120
|
+
this.threadMemorySync = new ThreadMemorySync(this.persistence, this.runtimeMemoryStore);
|
|
121
|
+
this.unregisterThreadMemorySync = this.eventBus.registerProjection(this.threadMemorySync);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
this.threadMemorySync = null;
|
|
125
|
+
this.unregisterThreadMemorySync = () => { };
|
|
126
|
+
}
|
|
141
127
|
const checkpointMaintenanceConfig = readCheckpointMaintenanceConfig(workspace);
|
|
142
128
|
this.checkpointMaintenance = checkpointMaintenanceConfig
|
|
143
129
|
? new CheckpointMaintenanceLoop(discoverCheckpointMaintenanceTargets(workspace), checkpointMaintenanceConfig)
|
|
@@ -148,8 +134,12 @@ export class AgentHarnessRuntime {
|
|
|
148
134
|
: null;
|
|
149
135
|
this.recoveryConfig = getRecoveryConfig(workspace.refs);
|
|
150
136
|
this.concurrencyConfig = getConcurrencyConfig(workspace.refs);
|
|
151
|
-
|
|
152
|
-
|
|
137
|
+
const healthConfig = readHealthMonitorConfig(workspace);
|
|
138
|
+
this.healthMonitor = healthConfig.enabled ? this.createHealthMonitor() : null;
|
|
139
|
+
}
|
|
140
|
+
createHealthMonitor() {
|
|
141
|
+
return new HealthMonitor({
|
|
142
|
+
workspace: this.workspace,
|
|
153
143
|
persistence: this.persistence,
|
|
154
144
|
getActiveRunSlots: () => this.activeRunSlots,
|
|
155
145
|
getPendingRunSlots: () => this.pendingRunSlots.length,
|
|
@@ -160,35 +150,34 @@ export class AgentHarnessRuntime {
|
|
|
160
150
|
},
|
|
161
151
|
});
|
|
162
152
|
}
|
|
153
|
+
recordLlmSuccess(startedAt) {
|
|
154
|
+
this.healthMonitor?.recordLlmSuccess(Date.now() - startedAt);
|
|
155
|
+
}
|
|
156
|
+
recordLlmFailure(startedAt) {
|
|
157
|
+
this.healthMonitor?.recordLlmFailure(Date.now() - startedAt);
|
|
158
|
+
}
|
|
163
159
|
async initialize() {
|
|
164
160
|
await this.persistence.initialize();
|
|
165
161
|
await this.checkpointMaintenance?.start();
|
|
166
162
|
await this.runtimeRecordMaintenance?.start();
|
|
167
|
-
await this.healthMonitor
|
|
163
|
+
await this.healthMonitor?.start();
|
|
168
164
|
await this.recoverStartupRuns();
|
|
169
165
|
}
|
|
170
166
|
subscribe(listener) {
|
|
171
167
|
return this.eventBus.subscribe(listener);
|
|
172
168
|
}
|
|
173
169
|
async getHealth() {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
getBinding(agentId) {
|
|
177
|
-
return this.workspace.bindings.get(agentId);
|
|
178
|
-
}
|
|
179
|
-
listAgentTools(agentId) {
|
|
180
|
-
const binding = this.getBinding(agentId);
|
|
181
|
-
if (!binding) {
|
|
182
|
-
throw new Error(`Unknown agent ${agentId}`);
|
|
170
|
+
if (this.healthMonitor) {
|
|
171
|
+
return this.healthMonitor.getSnapshot();
|
|
183
172
|
}
|
|
184
|
-
return
|
|
173
|
+
return createDefaultHealthSnapshot(this.activeRunSlots, this.pendingRunSlots.length);
|
|
185
174
|
}
|
|
186
175
|
resolveAgentTools(agentId) {
|
|
187
|
-
const binding = this.
|
|
176
|
+
const binding = this.workspace.bindings.get(agentId);
|
|
188
177
|
if (!binding) {
|
|
189
178
|
throw new Error(`Unknown agent ${agentId}`);
|
|
190
179
|
}
|
|
191
|
-
const compiledTools =
|
|
180
|
+
const compiledTools = getBindingPrimaryTools(binding);
|
|
192
181
|
const resolver = this.resolvedRuntimeAdapterOptions.toolResolver;
|
|
193
182
|
const resolvedTools = resolver ? resolver(compiledTools.map((tool) => tool.id), binding) : [];
|
|
194
183
|
return compiledTools.map((compiledTool, index) => ({
|
|
@@ -256,10 +245,16 @@ export class AgentHarnessRuntime {
|
|
|
256
245
|
return approval ? toPublicApprovalRecord(approval) : null;
|
|
257
246
|
}
|
|
258
247
|
listAgentSkills(agentId, options = {}) {
|
|
259
|
-
return listWorkspaceAgentSkills(this.workspace, agentId,
|
|
248
|
+
return listWorkspaceAgentSkills(this.workspace, agentId, {
|
|
249
|
+
assessRequirements: isInventoryEnabled(this.workspace),
|
|
250
|
+
...options,
|
|
251
|
+
});
|
|
260
252
|
}
|
|
261
253
|
describeWorkspaceInventory(options = {}) {
|
|
262
|
-
return describeWorkspaceInventory(this.workspace,
|
|
254
|
+
return describeWorkspaceInventory(this.workspace, {
|
|
255
|
+
assessRequirements: isInventoryEnabled(this.workspace),
|
|
256
|
+
...options,
|
|
257
|
+
});
|
|
263
258
|
}
|
|
264
259
|
async deleteThreadCheckpoints(threadId) {
|
|
265
260
|
const resolver = this.resolvedRuntimeAdapterOptions.checkpointerResolver;
|
|
@@ -311,10 +306,7 @@ export class AgentHarnessRuntime {
|
|
|
311
306
|
return serveToolsOverStdioFromHarness(tools, options);
|
|
312
307
|
}
|
|
313
308
|
async routeAgent(input, options = {}) {
|
|
314
|
-
const routingHistory = options.threadId ? await this.persistence.listThreadMessages(options.threadId) : [];
|
|
315
|
-
const routingInput = buildRoutingInput(input, routingHistory);
|
|
316
309
|
const rawInput = extractMessageText(input);
|
|
317
|
-
const { primaryBinding, secondaryBinding } = inferRoutingBindings(this.workspace);
|
|
318
310
|
const configuredRule = this.routingRules.find((rule) => matchRoutingRule(rawInput, rule, options));
|
|
319
311
|
if (configuredRule) {
|
|
320
312
|
const configuredBinding = this.workspace.bindings.get(configuredRule.agentId);
|
|
@@ -322,25 +314,13 @@ export class AgentHarnessRuntime {
|
|
|
322
314
|
return configuredBinding.agent.id;
|
|
323
315
|
}
|
|
324
316
|
}
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
return defaultBinding.agent.id;
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
if (!primaryBinding || !secondaryBinding) {
|
|
334
|
-
return heuristicHostRoute(this.workspace, rawInput);
|
|
335
|
-
}
|
|
336
|
-
try {
|
|
337
|
-
return await this.runtimeAdapter.route(routingInput, primaryBinding, secondaryBinding, {
|
|
338
|
-
systemPrompt: this.routingSystemPrompt,
|
|
339
|
-
});
|
|
340
|
-
}
|
|
341
|
-
catch {
|
|
342
|
-
return heuristicHostRoute(this.workspace, rawInput);
|
|
317
|
+
const defaultBinding = this.routingDefaultAgentId
|
|
318
|
+
? this.workspace.bindings.get(this.routingDefaultAgentId)
|
|
319
|
+
: undefined;
|
|
320
|
+
if (defaultBinding && isRuntimeEntryBinding(defaultBinding)) {
|
|
321
|
+
return defaultBinding.agent.id;
|
|
343
322
|
}
|
|
323
|
+
return this.getDefaultHostAgentId();
|
|
344
324
|
}
|
|
345
325
|
async emit(threadId, runId, sequence, eventType, payload, source = "runtime") {
|
|
346
326
|
return emitHarnessEvent({
|
|
@@ -415,9 +395,6 @@ export class AgentHarnessRuntime {
|
|
|
415
395
|
const userTurn = history.find((message) => message.runId === runId && message.role === "user");
|
|
416
396
|
return userTurn?.content ?? "";
|
|
417
397
|
}
|
|
418
|
-
async appendAssistantMessage(threadId, runId, content) {
|
|
419
|
-
return appendLifecycleAssistantMessage(this.persistence, threadId, runId, content);
|
|
420
|
-
}
|
|
421
398
|
async getRunCancellation(runId) {
|
|
422
399
|
const control = await this.persistence.getRunControl(runId);
|
|
423
400
|
return {
|
|
@@ -425,12 +402,6 @@ export class AgentHarnessRuntime {
|
|
|
425
402
|
...(control?.cancelReason ? { reason: control.cancelReason } : {}),
|
|
426
403
|
};
|
|
427
404
|
}
|
|
428
|
-
async expirePendingApprovals(threadId, runId) {
|
|
429
|
-
return expireLifecyclePendingApprovals({
|
|
430
|
-
persistence: this.persistence,
|
|
431
|
-
emit: (currentThreadId, currentRunId, sequence, eventType, payload, source) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload, source),
|
|
432
|
-
}, threadId, runId);
|
|
433
|
-
}
|
|
434
405
|
async finalizeCancelledRun(threadId, runId, previousState, reason) {
|
|
435
406
|
return finalizeLifecycleCancelledRun({
|
|
436
407
|
persistence: this.persistence,
|
|
@@ -443,11 +414,11 @@ export class AgentHarnessRuntime {
|
|
|
443
414
|
const startedAt = Date.now();
|
|
444
415
|
try {
|
|
445
416
|
const result = await this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, history, options);
|
|
446
|
-
this.
|
|
417
|
+
this.recordLlmSuccess(startedAt);
|
|
447
418
|
return result;
|
|
448
419
|
}
|
|
449
420
|
catch (error) {
|
|
450
|
-
this.
|
|
421
|
+
this.recordLlmFailure(startedAt);
|
|
451
422
|
throw error;
|
|
452
423
|
}
|
|
453
424
|
}
|
|
@@ -506,7 +477,12 @@ export class AgentHarnessRuntime {
|
|
|
506
477
|
};
|
|
507
478
|
}
|
|
508
479
|
catch (error) {
|
|
509
|
-
await
|
|
480
|
+
await emitSyntheticFallbackEvent({
|
|
481
|
+
persistence: this.persistence,
|
|
482
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
483
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
484
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
485
|
+
}, threadId, runId, agentId, error, 103);
|
|
510
486
|
await this.setRunStateAndEmit(threadId, runId, 104, "failed", {
|
|
511
487
|
previousState: previousState === "queued" ? "running" : previousState,
|
|
512
488
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -523,47 +499,14 @@ export class AgentHarnessRuntime {
|
|
|
523
499
|
await this.persistence.clearRunRequest(threadId, runId);
|
|
524
500
|
}
|
|
525
501
|
}
|
|
526
|
-
checkpointRefForState(threadId, runId, state) {
|
|
527
|
-
return getCheckpointRefForRunState(threadId, runId, state);
|
|
528
|
-
}
|
|
529
502
|
async finalizeContinuedRun(binding, threadId, runId, input, actual, options) {
|
|
530
503
|
return finalizeLifecycleContinuedRun({
|
|
531
504
|
persistence: this.persistence,
|
|
532
505
|
emit: (currentThreadId, currentRunId, sequence, eventType, payload, source) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload, source),
|
|
533
506
|
setRunStateAndEmit: (currentThreadId, currentRunId, sequence, state, lifecycleOptions) => this.setRunStateAndEmit(currentThreadId, currentRunId, sequence, state, lifecycleOptions),
|
|
534
507
|
requestApprovalAndEmit: (currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence) => this.requestApprovalAndEmit(currentThreadId, currentRunId, lifecycleInput, interruptContent, checkpointRef, sequence),
|
|
535
|
-
synthesizeFinalResult: (currentBinding, lifecycleInput, lifecycleActual) => this.runtimeAdapter.synthesizeFinalResult(currentBinding, lifecycleInput, lifecycleActual),
|
|
536
|
-
reviewRunResult: (currentBinding, lifecycleInput, lifecycleActual) => this.runtimeAdapter.reviewRunResult(currentBinding, lifecycleInput, lifecycleActual),
|
|
537
508
|
}, binding, threadId, runId, input, actual, options);
|
|
538
509
|
}
|
|
539
|
-
async synthesizeCompletedRun(binding, input, actual) {
|
|
540
|
-
return synthesizeLifecycleCompletedRun({
|
|
541
|
-
synthesizeFinalResult: (currentBinding, lifecycleInput, lifecycleActual) => this.runtimeAdapter.synthesizeFinalResult(currentBinding, lifecycleInput, lifecycleActual),
|
|
542
|
-
}, binding, input, actual);
|
|
543
|
-
}
|
|
544
|
-
async reviewCompletedRun(binding, threadId, runId, input, actual) {
|
|
545
|
-
return reviewLifecycleCompletedRun({
|
|
546
|
-
reviewRunResult: (currentBinding, lifecycleInput, lifecycleActual) => this.runtimeAdapter.reviewRunResult(currentBinding, lifecycleInput, lifecycleActual),
|
|
547
|
-
emit: (currentThreadId, currentRunId, sequence, eventType, payload, source) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload, source),
|
|
548
|
-
}, binding, threadId, runId, input, actual);
|
|
549
|
-
}
|
|
550
|
-
async emitOutputDeltaAndCreateItem(threadId, runId, agentId, content) {
|
|
551
|
-
return emitStreamingOutputDeltaAndCreateItem((currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload), threadId, runId, agentId, content);
|
|
552
|
-
}
|
|
553
|
-
createContentBlocksItem(threadId, runId, agentId, contentBlocks) {
|
|
554
|
-
return createStreamingContentBlocksItem(threadId, runId, agentId, contentBlocks);
|
|
555
|
-
}
|
|
556
|
-
createToolResultKey(toolName, output, isError) {
|
|
557
|
-
return createStreamingToolResultKey(toolName, output, isError);
|
|
558
|
-
}
|
|
559
|
-
async emitRunCreated(threadId, runId, payload) {
|
|
560
|
-
return emitRunCreatedEvent({
|
|
561
|
-
persistence: this.persistence,
|
|
562
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
563
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
564
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
565
|
-
}, threadId, runId, payload);
|
|
566
|
-
}
|
|
567
510
|
async setRunStateAndEmit(threadId, runId, sequence, state, options) {
|
|
568
511
|
return setRunStateAndEmitEvent({
|
|
569
512
|
persistence: this.persistence,
|
|
@@ -580,25 +523,6 @@ export class AgentHarnessRuntime {
|
|
|
580
523
|
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
581
524
|
}, threadId, runId, input, interruptContent, checkpointRef, sequence);
|
|
582
525
|
}
|
|
583
|
-
async emitSyntheticFallback(threadId, runId, selectedAgentId, error, sequence = 3) {
|
|
584
|
-
await emitSyntheticFallbackEvent({
|
|
585
|
-
persistence: this.persistence,
|
|
586
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
587
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
588
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
589
|
-
}, threadId, runId, selectedAgentId, error, sequence);
|
|
590
|
-
}
|
|
591
|
-
async persistApproval(threadId, runId, checkpointRef, input, interruptContent) {
|
|
592
|
-
return persistApproval({
|
|
593
|
-
persistence: this.persistence,
|
|
594
|
-
publishEvent: (event) => this.eventBus.publish(event),
|
|
595
|
-
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
596
|
-
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
597
|
-
}, threadId, runId, checkpointRef, input, interruptContent);
|
|
598
|
-
}
|
|
599
|
-
async resolveApprovalRecord(options, thread) {
|
|
600
|
-
return resolveHarnessApprovalRecord(this.persistence, options, thread);
|
|
601
|
-
}
|
|
602
526
|
isDecisionRun(options) {
|
|
603
527
|
return "decision" in options;
|
|
604
528
|
}
|
|
@@ -608,6 +532,55 @@ export class AgentHarnessRuntime {
|
|
|
608
532
|
}
|
|
609
533
|
await listener(value);
|
|
610
534
|
}
|
|
535
|
+
async prepareRunStart(options, invocation, runCreatedPayload) {
|
|
536
|
+
const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
|
|
537
|
+
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
538
|
+
if (!binding) {
|
|
539
|
+
throw new Error(`Unknown agent ${selectedAgentId}`);
|
|
540
|
+
}
|
|
541
|
+
const policyDecision = this.policyEngine.evaluate(binding);
|
|
542
|
+
if (!policyDecision.allowed) {
|
|
543
|
+
throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
|
|
544
|
+
}
|
|
545
|
+
const priority = normalizeRunPriority(options.priority);
|
|
546
|
+
const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
|
|
547
|
+
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
548
|
+
return {
|
|
549
|
+
binding,
|
|
550
|
+
selectedAgentId,
|
|
551
|
+
priority,
|
|
552
|
+
threadId,
|
|
553
|
+
runId,
|
|
554
|
+
isNewThread,
|
|
555
|
+
runCreatedEventPromise: emitRunCreatedEvent({
|
|
556
|
+
persistence: this.persistence,
|
|
557
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
558
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
559
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
560
|
+
}, threadId, runId, runCreatedPayload(binding, selectedAgentId)),
|
|
561
|
+
releaseRunSlotPromise: this.acquireRunSlot(threadId, runId, "running", priority),
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
createStartupRecoveryContext() {
|
|
565
|
+
return {
|
|
566
|
+
persistence: this.persistence,
|
|
567
|
+
workspace: this.workspace,
|
|
568
|
+
runtimeAdapter: this.runtimeAdapter,
|
|
569
|
+
recoveryConfig: this.recoveryConfig,
|
|
570
|
+
concurrencyConfig: this.concurrencyConfig,
|
|
571
|
+
getBinding: (agentId) => this.workspace.bindings.get(agentId),
|
|
572
|
+
acquireRunSlot: (threadId, runId, activeState = "running", priority = 0) => this.acquireRunSlot(threadId, runId, activeState, priority),
|
|
573
|
+
executeQueuedRun: (binding, input, threadId, runId, agentId, options = {}) => this.executeQueuedRun(binding, input, threadId, runId, agentId, options),
|
|
574
|
+
setRunStateAndEmit: (threadId, runId, sequence, state, options) => this.setRunStateAndEmit(threadId, runId, sequence, state, options),
|
|
575
|
+
emit: (threadId, runId, sequence, eventType, payload) => this.emit(threadId, runId, sequence, eventType, payload),
|
|
576
|
+
loadRunInput: (threadId, runId) => this.loadRunInput(threadId, runId),
|
|
577
|
+
finalizeContinuedRun: (binding, threadId, runId, input, actual, options) => this.finalizeContinuedRun(binding, threadId, runId, input, actual, options),
|
|
578
|
+
supportsRunningReplay: (binding) => this.supportsRunningReplay(binding),
|
|
579
|
+
isStaleRunningRun: (thread, nowMs = Date.now()) => this.isStaleRunningRun(thread, nowMs),
|
|
580
|
+
recordLlmSuccess: (startedAt) => this.recordLlmSuccess(startedAt),
|
|
581
|
+
recordLlmFailure: (startedAt) => this.recordLlmFailure(startedAt),
|
|
582
|
+
};
|
|
583
|
+
}
|
|
611
584
|
async acquireRunSlot(threadId, runId, activeState = "running", priority = 0) {
|
|
612
585
|
let stopHeartbeat = () => undefined;
|
|
613
586
|
const beginLease = async (mode) => {
|
|
@@ -788,32 +761,12 @@ export class AgentHarnessRuntime {
|
|
|
788
761
|
return this.dispatchRunListeners(this.streamEvents(options), resolvedListeners);
|
|
789
762
|
}
|
|
790
763
|
const invocation = normalizeInvocationEnvelope(options);
|
|
791
|
-
const selectedAgentId = await
|
|
792
|
-
|
|
793
|
-
input: options.input,
|
|
794
|
-
requestedAgentId: options.agentId,
|
|
795
|
-
threadId: options.threadId,
|
|
796
|
-
preferredHostAgentId: AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
797
|
-
getThreadSummary: (threadId) => this.getSession(threadId),
|
|
798
|
-
});
|
|
799
|
-
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
800
|
-
if (!binding) {
|
|
801
|
-
throw new Error(`Unknown agent ${selectedAgentId}`);
|
|
802
|
-
}
|
|
803
|
-
const policyDecision = this.policyEngine.evaluate(binding);
|
|
804
|
-
if (!policyDecision.allowed) {
|
|
805
|
-
throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
|
|
806
|
-
}
|
|
807
|
-
const priority = normalizeRunPriority(options.priority);
|
|
808
|
-
const runRequest = buildPersistedRunRequest(options.input, invocation, priority);
|
|
809
|
-
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
810
|
-
const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
|
|
811
|
-
agentId: binding.agent.id,
|
|
764
|
+
const { binding, selectedAgentId, threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (activeBinding, activeSelectedAgentId) => ({
|
|
765
|
+
agentId: activeBinding.agent.id,
|
|
812
766
|
requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
|
|
813
|
-
selectedAgentId,
|
|
814
|
-
executionMode: getBindingAdapterKind(
|
|
815
|
-
});
|
|
816
|
-
const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
|
|
767
|
+
selectedAgentId: activeSelectedAgentId,
|
|
768
|
+
executionMode: getBindingAdapterKind(activeBinding),
|
|
769
|
+
}));
|
|
817
770
|
await runCreatedEventPromise;
|
|
818
771
|
const releaseRunSlot = await releaseRunSlotPromise;
|
|
819
772
|
try {
|
|
@@ -833,14 +786,7 @@ export class AgentHarnessRuntime {
|
|
|
833
786
|
}
|
|
834
787
|
async *streamEvents(options) {
|
|
835
788
|
const invocation = normalizeInvocationEnvelope(options);
|
|
836
|
-
const selectedAgentId = await resolveSelectedAgentId(
|
|
837
|
-
workspace: this.workspace,
|
|
838
|
-
input: options.input,
|
|
839
|
-
requestedAgentId: options.agentId,
|
|
840
|
-
threadId: options.threadId,
|
|
841
|
-
preferredHostAgentId: AgentHarnessRuntime.DEFAULT_HOST_AGENT_ID,
|
|
842
|
-
getThreadSummary: (threadId) => this.getSession(threadId),
|
|
843
|
-
});
|
|
789
|
+
const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
|
|
844
790
|
const binding = this.workspace.bindings.get(selectedAgentId);
|
|
845
791
|
if (!binding) {
|
|
846
792
|
const result = await this.run(options);
|
|
@@ -857,268 +803,268 @@ export class AgentHarnessRuntime {
|
|
|
857
803
|
}
|
|
858
804
|
let emitted = false;
|
|
859
805
|
let streamActivityObserved = false;
|
|
860
|
-
const
|
|
861
|
-
|
|
862
|
-
const { threadId, runId, isNewThread } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, runRequest, options.threadId);
|
|
863
|
-
const priorHistoryPromise = Promise.resolve(isNewThread ? [] : undefined).then((historyHint) => historyHint ?? this.loadPriorHistory(threadId, runId));
|
|
864
|
-
const runCreatedEventPromise = this.emitRunCreated(threadId, runId, {
|
|
865
|
-
agentId: selectedAgentId,
|
|
806
|
+
const { threadId, runId, isNewThread, runCreatedEventPromise, releaseRunSlotPromise, } = await this.prepareRunStart(options, invocation, (_binding, activeSelectedAgentId) => ({
|
|
807
|
+
agentId: activeSelectedAgentId,
|
|
866
808
|
requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
|
|
867
|
-
selectedAgentId,
|
|
809
|
+
selectedAgentId: activeSelectedAgentId,
|
|
868
810
|
input: options.input,
|
|
869
811
|
state: "running",
|
|
870
|
-
});
|
|
812
|
+
}));
|
|
813
|
+
const priorHistoryPromise = Promise.resolve(isNewThread ? [] : undefined).then((historyHint) => historyHint ?? this.loadPriorHistory(threadId, runId));
|
|
871
814
|
yield { type: "event", event: await runCreatedEventPromise };
|
|
872
|
-
const releaseRunSlotPromise = this.acquireRunSlot(threadId, runId, "running", priority);
|
|
873
815
|
let releaseRunSlot = async () => undefined;
|
|
816
|
+
const emitOutputDelta = (content) => emitStreamingOutputDeltaAndCreateItem((currentThreadId, currentRunId, sequence, eventType, payload) => this.emit(currentThreadId, currentRunId, sequence, eventType, payload), threadId, runId, selectedAgentId, content);
|
|
874
817
|
try {
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
? chunk.
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
result: {
|
|
927
|
-
threadId,
|
|
928
|
-
runId,
|
|
929
|
-
agentId: selectedAgentId,
|
|
930
|
-
state: "waiting_for_approval",
|
|
931
|
-
output: assistantOutput,
|
|
932
|
-
finalMessageText: assistantOutput,
|
|
933
|
-
interruptContent: normalizedChunk.content,
|
|
934
|
-
approvalId: approvalRequest.approval.approvalId,
|
|
935
|
-
pendingActionId: approvalRequest.approval.pendingActionId,
|
|
936
|
-
},
|
|
937
|
-
};
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
if (normalizedChunk.kind === "reasoning") {
|
|
941
|
-
await this.emit(threadId, runId, 3, "reasoning.delta", {
|
|
942
|
-
content: normalizedChunk.content,
|
|
943
|
-
});
|
|
944
|
-
yield {
|
|
945
|
-
type: "reasoning",
|
|
818
|
+
const [priorHistory, acquiredReleaseRunSlot] = await Promise.all([
|
|
819
|
+
priorHistoryPromise,
|
|
820
|
+
releaseRunSlotPromise,
|
|
821
|
+
]).then(([loadedPriorHistory, resolvedReleaseRunSlot]) => [loadedPriorHistory, resolvedReleaseRunSlot]);
|
|
822
|
+
releaseRunSlot = acquiredReleaseRunSlot;
|
|
823
|
+
let assistantOutput = "";
|
|
824
|
+
const toolErrors = [];
|
|
825
|
+
let lastToolResultKey = null;
|
|
826
|
+
for await (const chunk of this.runtimeAdapter.stream(binding, options.input, threadId, priorHistory, {
|
|
827
|
+
context: invocation.context,
|
|
828
|
+
state: invocation.state,
|
|
829
|
+
files: invocation.files,
|
|
830
|
+
runId,
|
|
831
|
+
})) {
|
|
832
|
+
if (chunk) {
|
|
833
|
+
streamActivityObserved = true;
|
|
834
|
+
const normalizedChunk = typeof chunk === "string"
|
|
835
|
+
? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
|
|
836
|
+
? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
|
|
837
|
+
: { kind: "content", content: chunk }
|
|
838
|
+
: chunk;
|
|
839
|
+
if (normalizedChunk.kind === "upstream-event") {
|
|
840
|
+
yield {
|
|
841
|
+
type: "upstream-event",
|
|
842
|
+
threadId,
|
|
843
|
+
runId,
|
|
844
|
+
agentId: selectedAgentId,
|
|
845
|
+
event: normalizedChunk.event.format === "langgraph-v2"
|
|
846
|
+
? normalizedChunk.event
|
|
847
|
+
: normalizeUpstreamRuntimeEvent(normalizedChunk.event.raw ?? normalizedChunk.event),
|
|
848
|
+
};
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
if (normalizedChunk.kind === "interrupt") {
|
|
852
|
+
const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
|
|
853
|
+
const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 6, "waiting_for_approval", {
|
|
854
|
+
previousState: "running",
|
|
855
|
+
checkpointRef,
|
|
856
|
+
});
|
|
857
|
+
const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 7);
|
|
858
|
+
yield {
|
|
859
|
+
type: "event",
|
|
860
|
+
event: waitingEvent,
|
|
861
|
+
};
|
|
862
|
+
yield {
|
|
863
|
+
type: "event",
|
|
864
|
+
event: approvalRequest.event,
|
|
865
|
+
};
|
|
866
|
+
yield {
|
|
867
|
+
type: "result",
|
|
868
|
+
result: {
|
|
946
869
|
threadId,
|
|
947
870
|
runId,
|
|
948
871
|
agentId: selectedAgentId,
|
|
949
|
-
|
|
950
|
-
|
|
872
|
+
state: "waiting_for_approval",
|
|
873
|
+
output: assistantOutput,
|
|
874
|
+
finalMessageText: assistantOutput,
|
|
875
|
+
interruptContent: normalizedChunk.content,
|
|
876
|
+
approvalId: approvalRequest.approval.approvalId,
|
|
877
|
+
pendingActionId: approvalRequest.approval.pendingActionId,
|
|
878
|
+
},
|
|
879
|
+
};
|
|
880
|
+
return;
|
|
881
|
+
}
|
|
882
|
+
if (normalizedChunk.kind === "reasoning") {
|
|
883
|
+
await this.emit(threadId, runId, 3, "reasoning.delta", {
|
|
884
|
+
content: normalizedChunk.content,
|
|
885
|
+
});
|
|
886
|
+
yield {
|
|
887
|
+
type: "reasoning",
|
|
888
|
+
threadId,
|
|
889
|
+
runId,
|
|
890
|
+
agentId: selectedAgentId,
|
|
891
|
+
content: normalizedChunk.content,
|
|
892
|
+
};
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
if (normalizedChunk.kind === "step") {
|
|
896
|
+
yield {
|
|
897
|
+
type: "step",
|
|
898
|
+
threadId,
|
|
899
|
+
runId,
|
|
900
|
+
agentId: selectedAgentId,
|
|
901
|
+
content: normalizedChunk.content,
|
|
902
|
+
};
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
if (normalizedChunk.kind === "tool-result") {
|
|
906
|
+
const toolResultKey = createStreamingToolResultKey(normalizedChunk.toolName, normalizedChunk.output, normalizedChunk.isError);
|
|
907
|
+
if (toolResultKey === lastToolResultKey) {
|
|
951
908
|
continue;
|
|
952
909
|
}
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
threadId,
|
|
957
|
-
runId,
|
|
958
|
-
agentId: selectedAgentId,
|
|
959
|
-
content: normalizedChunk.content,
|
|
960
|
-
};
|
|
961
|
-
continue;
|
|
910
|
+
lastToolResultKey = toolResultKey;
|
|
911
|
+
if (normalizedChunk.isError) {
|
|
912
|
+
toolErrors.push(renderToolFailure(normalizedChunk.toolName, normalizedChunk.output));
|
|
962
913
|
}
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
type: "tool-result",
|
|
974
|
-
threadId,
|
|
975
|
-
runId,
|
|
976
|
-
agentId: selectedAgentId,
|
|
977
|
-
toolName: normalizedChunk.toolName,
|
|
978
|
-
output: normalizedChunk.output,
|
|
979
|
-
isError: normalizedChunk.isError,
|
|
980
|
-
};
|
|
981
|
-
continue;
|
|
982
|
-
}
|
|
983
|
-
emitted = true;
|
|
984
|
-
assistantOutput += normalizedChunk.content;
|
|
985
|
-
yield await this.emitOutputDeltaAndCreateItem(threadId, runId, selectedAgentId, normalizedChunk.content);
|
|
914
|
+
yield {
|
|
915
|
+
type: "tool-result",
|
|
916
|
+
threadId,
|
|
917
|
+
runId,
|
|
918
|
+
agentId: selectedAgentId,
|
|
919
|
+
toolName: normalizedChunk.toolName,
|
|
920
|
+
output: normalizedChunk.output,
|
|
921
|
+
isError: normalizedChunk.isError,
|
|
922
|
+
};
|
|
923
|
+
continue;
|
|
986
924
|
}
|
|
987
|
-
}
|
|
988
|
-
if (!assistantOutput && toolErrors.length > 0) {
|
|
989
|
-
assistantOutput = toolErrors.join("\n\n");
|
|
990
925
|
emitted = true;
|
|
991
|
-
|
|
926
|
+
assistantOutput += normalizedChunk.content;
|
|
927
|
+
yield await emitOutputDelta(normalizedChunk.content);
|
|
992
928
|
}
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
929
|
+
}
|
|
930
|
+
if (!assistantOutput && toolErrors.length > 0) {
|
|
931
|
+
assistantOutput = toolErrors.join("\n\n");
|
|
932
|
+
emitted = true;
|
|
933
|
+
yield await emitOutputDelta(assistantOutput);
|
|
934
|
+
}
|
|
935
|
+
if (!assistantOutput) {
|
|
936
|
+
const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
|
|
937
|
+
if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
|
|
938
|
+
yield createStreamingContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
|
|
1003
939
|
}
|
|
1004
|
-
|
|
940
|
+
if (actual.output) {
|
|
941
|
+
assistantOutput = actual.output;
|
|
942
|
+
emitted = true;
|
|
943
|
+
yield await emitOutputDelta(actual.output);
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
await appendLifecycleAssistantMessage(this.persistence, threadId, runId, assistantOutput);
|
|
947
|
+
yield {
|
|
948
|
+
type: "result",
|
|
949
|
+
result: {
|
|
950
|
+
threadId,
|
|
951
|
+
runId,
|
|
952
|
+
agentId: selectedAgentId,
|
|
953
|
+
state: "completed",
|
|
954
|
+
output: assistantOutput,
|
|
955
|
+
finalMessageText: assistantOutput,
|
|
956
|
+
},
|
|
957
|
+
};
|
|
958
|
+
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
|
|
959
|
+
previousState: "running",
|
|
960
|
+
}) };
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
catch (error) {
|
|
964
|
+
if (emitted || streamActivityObserved) {
|
|
965
|
+
const runtimeFailure = renderRuntimeFailure(error);
|
|
966
|
+
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
967
|
+
previousState: "running",
|
|
968
|
+
error: error instanceof Error ? error.message : String(error),
|
|
969
|
+
}) };
|
|
970
|
+
yield {
|
|
971
|
+
type: "content",
|
|
972
|
+
threadId,
|
|
973
|
+
runId,
|
|
974
|
+
agentId: selectedAgentId,
|
|
975
|
+
content: runtimeFailure,
|
|
976
|
+
};
|
|
1005
977
|
yield {
|
|
1006
978
|
type: "result",
|
|
1007
979
|
result: {
|
|
1008
980
|
threadId,
|
|
1009
981
|
runId,
|
|
1010
982
|
agentId: selectedAgentId,
|
|
1011
|
-
state: "
|
|
1012
|
-
output:
|
|
1013
|
-
finalMessageText:
|
|
983
|
+
state: "failed",
|
|
984
|
+
output: runtimeFailure,
|
|
985
|
+
finalMessageText: runtimeFailure,
|
|
1014
986
|
},
|
|
1015
987
|
};
|
|
1016
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
|
|
1017
|
-
previousState: "running",
|
|
1018
|
-
}) };
|
|
1019
988
|
return;
|
|
1020
989
|
}
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
990
|
+
if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
|
|
991
|
+
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
992
|
+
previousState: "running",
|
|
993
|
+
error: error.message,
|
|
994
|
+
}) };
|
|
995
|
+
yield {
|
|
996
|
+
type: "content",
|
|
997
|
+
threadId,
|
|
998
|
+
runId,
|
|
999
|
+
agentId: selectedAgentId,
|
|
1000
|
+
content: renderRuntimeFailure(error),
|
|
1001
|
+
};
|
|
1002
|
+
yield {
|
|
1003
|
+
type: "result",
|
|
1004
|
+
result: {
|
|
1030
1005
|
threadId,
|
|
1031
1006
|
runId,
|
|
1032
1007
|
agentId: selectedAgentId,
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
};
|
|
1046
|
-
return;
|
|
1008
|
+
state: "failed",
|
|
1009
|
+
output: renderRuntimeFailure(error),
|
|
1010
|
+
finalMessageText: renderRuntimeFailure(error),
|
|
1011
|
+
},
|
|
1012
|
+
};
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
try {
|
|
1016
|
+
const actual = await this.invokeWithHistory(binding, options.input, threadId, runId);
|
|
1017
|
+
await appendLifecycleAssistantMessage(this.persistence, threadId, runId, actual.output);
|
|
1018
|
+
if (Array.isArray(actual.contentBlocks) && actual.contentBlocks.length > 0) {
|
|
1019
|
+
yield createStreamingContentBlocksItem(threadId, runId, selectedAgentId, actual.contentBlocks);
|
|
1047
1020
|
}
|
|
1048
|
-
if (
|
|
1049
|
-
yield
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1021
|
+
if (actual.output) {
|
|
1022
|
+
yield await emitOutputDelta(actual.output);
|
|
1023
|
+
}
|
|
1024
|
+
yield {
|
|
1025
|
+
type: "result",
|
|
1026
|
+
result: {
|
|
1027
|
+
...actual,
|
|
1055
1028
|
threadId,
|
|
1056
1029
|
runId,
|
|
1057
1030
|
agentId: selectedAgentId,
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
threadId,
|
|
1087
|
-
runId,
|
|
1088
|
-
agentId: selectedAgentId,
|
|
1089
|
-
},
|
|
1090
|
-
};
|
|
1091
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
|
|
1092
|
-
previousState: "running",
|
|
1093
|
-
}) };
|
|
1094
|
-
return;
|
|
1095
|
-
}
|
|
1096
|
-
catch (invokeError) {
|
|
1097
|
-
await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
|
|
1098
|
-
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
1099
|
-
previousState: "running",
|
|
1100
|
-
error: invokeError instanceof Error ? invokeError.message : String(invokeError),
|
|
1101
|
-
}) };
|
|
1102
|
-
yield {
|
|
1103
|
-
type: "content",
|
|
1031
|
+
},
|
|
1032
|
+
};
|
|
1033
|
+
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
|
|
1034
|
+
previousState: "running",
|
|
1035
|
+
}) };
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
catch (invokeError) {
|
|
1039
|
+
await emitSyntheticFallbackEvent({
|
|
1040
|
+
persistence: this.persistence,
|
|
1041
|
+
publishEvent: (event) => this.eventBus.publish(event),
|
|
1042
|
+
trackBackgroundTask: (task) => this.trackBackgroundTask(task),
|
|
1043
|
+
backgroundEventTypes: AgentHarnessRuntime.BACKGROUND_EVENT_TYPES,
|
|
1044
|
+
}, threadId, runId, selectedAgentId, invokeError);
|
|
1045
|
+
yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
|
|
1046
|
+
previousState: "running",
|
|
1047
|
+
error: invokeError instanceof Error ? invokeError.message : String(invokeError),
|
|
1048
|
+
}) };
|
|
1049
|
+
yield {
|
|
1050
|
+
type: "content",
|
|
1051
|
+
threadId,
|
|
1052
|
+
runId,
|
|
1053
|
+
agentId: selectedAgentId,
|
|
1054
|
+
content: renderRuntimeFailure(invokeError),
|
|
1055
|
+
};
|
|
1056
|
+
yield {
|
|
1057
|
+
type: "result",
|
|
1058
|
+
result: {
|
|
1104
1059
|
threadId,
|
|
1105
1060
|
runId,
|
|
1106
1061
|
agentId: selectedAgentId,
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
runId,
|
|
1114
|
-
agentId: selectedAgentId,
|
|
1115
|
-
state: "failed",
|
|
1116
|
-
output: renderRuntimeFailure(invokeError),
|
|
1117
|
-
finalMessageText: renderRuntimeFailure(invokeError),
|
|
1118
|
-
},
|
|
1119
|
-
};
|
|
1120
|
-
return;
|
|
1121
|
-
}
|
|
1062
|
+
state: "failed",
|
|
1063
|
+
output: renderRuntimeFailure(invokeError),
|
|
1064
|
+
finalMessageText: renderRuntimeFailure(invokeError),
|
|
1065
|
+
},
|
|
1066
|
+
};
|
|
1067
|
+
return;
|
|
1122
1068
|
}
|
|
1123
1069
|
}
|
|
1124
1070
|
finally {
|
|
@@ -1136,14 +1082,14 @@ export class AgentHarnessRuntime {
|
|
|
1136
1082
|
if (!thread) {
|
|
1137
1083
|
throw new Error("resume requires either threadId or approvalId");
|
|
1138
1084
|
}
|
|
1139
|
-
const approval = approvalById ?? await this.
|
|
1085
|
+
const approval = approvalById ?? await resolveHarnessApprovalRecord(this.persistence, options, thread);
|
|
1140
1086
|
const threadId = approval.threadId;
|
|
1141
1087
|
const runId = approval.runId;
|
|
1142
1088
|
const binding = this.workspace.bindings.get(thread.agentId);
|
|
1143
1089
|
if (!binding) {
|
|
1144
1090
|
throw new Error(`Unknown agent ${thread.agentId}`);
|
|
1145
1091
|
}
|
|
1146
|
-
const resumePayload =
|
|
1092
|
+
const resumePayload = buildHarnessResumePayload(binding, approval, options);
|
|
1147
1093
|
const cancellation = await this.getRunCancellation(runId);
|
|
1148
1094
|
if (cancellation.requested) {
|
|
1149
1095
|
return this.finalizeCancelledRun(threadId, runId, thread.status, cancellation.reason);
|
|
@@ -1178,7 +1124,7 @@ export class AgentHarnessRuntime {
|
|
|
1178
1124
|
const startedAt = Date.now();
|
|
1179
1125
|
try {
|
|
1180
1126
|
const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
|
|
1181
|
-
this.
|
|
1127
|
+
this.recordLlmSuccess(startedAt);
|
|
1182
1128
|
const cancelledAfterInvoke = await this.getRunCancellation(runId);
|
|
1183
1129
|
if (cancelledAfterInvoke.requested) {
|
|
1184
1130
|
return this.finalizeCancelledRun(threadId, runId, "resuming", cancelledAfterInvoke.reason);
|
|
@@ -1196,7 +1142,7 @@ export class AgentHarnessRuntime {
|
|
|
1196
1142
|
};
|
|
1197
1143
|
}
|
|
1198
1144
|
catch (error) {
|
|
1199
|
-
this.
|
|
1145
|
+
this.recordLlmFailure(startedAt);
|
|
1200
1146
|
throw error;
|
|
1201
1147
|
}
|
|
1202
1148
|
}
|
|
@@ -1204,9 +1150,6 @@ export class AgentHarnessRuntime {
|
|
|
1204
1150
|
await releaseRunSlot();
|
|
1205
1151
|
}
|
|
1206
1152
|
}
|
|
1207
|
-
buildResumePayload(binding, approval, options) {
|
|
1208
|
-
return buildHarnessResumePayload(binding, approval, options);
|
|
1209
|
-
}
|
|
1210
1153
|
async restartConversation(options) {
|
|
1211
1154
|
const thread = await this.getSession(options.threadId);
|
|
1212
1155
|
if (!thread) {
|
|
@@ -1233,12 +1176,12 @@ export class AgentHarnessRuntime {
|
|
|
1233
1176
|
};
|
|
1234
1177
|
}
|
|
1235
1178
|
async close() {
|
|
1236
|
-
await this.healthMonitor
|
|
1179
|
+
await this.healthMonitor?.stop();
|
|
1237
1180
|
await this.checkpointMaintenance?.stop();
|
|
1238
1181
|
await this.runtimeRecordMaintenance?.stop();
|
|
1239
1182
|
this.unregisterThreadMemorySync();
|
|
1240
1183
|
await Promise.allSettled(Array.from(this.backgroundTasks));
|
|
1241
|
-
await this.threadMemorySync
|
|
1184
|
+
await this.threadMemorySync?.close();
|
|
1242
1185
|
}
|
|
1243
1186
|
async stop() {
|
|
1244
1187
|
await this.close();
|
|
@@ -1282,136 +1225,14 @@ export class AgentHarnessRuntime {
|
|
|
1282
1225
|
}
|
|
1283
1226
|
await this.reclaimExpiredClaimedRuns();
|
|
1284
1227
|
const threads = await this.persistence.listSessions();
|
|
1228
|
+
const recoveryContext = this.createStartupRecoveryContext();
|
|
1285
1229
|
for (const thread of threads) {
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
continue;
|
|
1291
|
-
}
|
|
1292
|
-
const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
|
|
1293
|
-
if (!request) {
|
|
1294
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1295
|
-
previousState: "queued",
|
|
1296
|
-
error: "missing persisted run request for queued run recovery",
|
|
1297
|
-
});
|
|
1298
|
-
continue;
|
|
1299
|
-
}
|
|
1300
|
-
const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
|
|
1301
|
-
try {
|
|
1302
|
-
await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
|
|
1303
|
-
context: request.invocation?.context,
|
|
1304
|
-
state: request.invocation?.inputs,
|
|
1305
|
-
files: request.invocation?.attachments,
|
|
1306
|
-
previousState: "queued",
|
|
1307
|
-
stateSequence: 103,
|
|
1308
|
-
approvalSequence: 104,
|
|
1309
|
-
});
|
|
1310
|
-
}
|
|
1311
|
-
finally {
|
|
1312
|
-
await releaseRunSlot();
|
|
1313
|
-
}
|
|
1314
|
-
continue;
|
|
1315
|
-
}
|
|
1316
|
-
if (thread.status === "running") {
|
|
1317
|
-
const isStale = await this.isStaleRunningRun(thread);
|
|
1318
|
-
if (!isStale) {
|
|
1319
|
-
continue;
|
|
1320
|
-
}
|
|
1321
|
-
const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
|
|
1322
|
-
const binding = this.workspace.bindings.get(runMeta.agentId);
|
|
1323
|
-
if (!binding) {
|
|
1324
|
-
continue;
|
|
1325
|
-
}
|
|
1326
|
-
if (!this.supportsRunningReplay(binding)) {
|
|
1327
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1328
|
-
previousState: "running",
|
|
1329
|
-
error: "stale running run cannot be replayed safely",
|
|
1330
|
-
});
|
|
1331
|
-
await this.persistence.releaseRunClaim(thread.latestRunId);
|
|
1332
|
-
continue;
|
|
1333
|
-
}
|
|
1334
|
-
const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
|
|
1335
|
-
if (!request) {
|
|
1336
|
-
await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
|
|
1337
|
-
previousState: "running",
|
|
1338
|
-
error: "missing persisted run request for stale running run recovery",
|
|
1339
|
-
});
|
|
1340
|
-
await this.persistence.releaseRunClaim(thread.latestRunId);
|
|
1341
|
-
continue;
|
|
1342
|
-
}
|
|
1343
|
-
const releaseRunSlot = await this.acquireRunSlot(thread.threadId, thread.latestRunId, "running", normalizeRunPriority(request.priority));
|
|
1344
|
-
try {
|
|
1345
|
-
await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
|
|
1346
|
-
resumeKind: "startup-running-recovery",
|
|
1347
|
-
state: "running",
|
|
1348
|
-
});
|
|
1349
|
-
await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
|
|
1350
|
-
context: request.invocation?.context,
|
|
1351
|
-
state: request.invocation?.inputs,
|
|
1352
|
-
files: request.invocation?.attachments,
|
|
1353
|
-
previousState: "running",
|
|
1354
|
-
stateSequence: 103,
|
|
1355
|
-
approvalSequence: 104,
|
|
1356
|
-
});
|
|
1357
|
-
}
|
|
1358
|
-
finally {
|
|
1359
|
-
await releaseRunSlot();
|
|
1360
|
-
}
|
|
1361
|
-
continue;
|
|
1362
|
-
}
|
|
1363
|
-
if (thread.status !== "resuming" || !this.recoveryConfig.resumeResumingRunsOnStartup) {
|
|
1364
|
-
continue;
|
|
1365
|
-
}
|
|
1366
|
-
const binding = this.workspace.bindings.get(thread.agentId);
|
|
1367
|
-
if (!binding) {
|
|
1230
|
+
const handled = await recoverQueuedStartupRun(recoveryContext, thread) ||
|
|
1231
|
+
await recoverRunningStartupRun(recoveryContext, thread) ||
|
|
1232
|
+
await recoverResumingStartupRun(recoveryContext, thread);
|
|
1233
|
+
if (handled) {
|
|
1368
1234
|
continue;
|
|
1369
1235
|
}
|
|
1370
|
-
const recoveryIntent = await this.persistence.getRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1371
|
-
if (!recoveryIntent || recoveryIntent.kind !== "approval-decision") {
|
|
1372
|
-
continue;
|
|
1373
|
-
}
|
|
1374
|
-
if (recoveryIntent.attempts >= this.recoveryConfig.maxRecoveryAttempts) {
|
|
1375
|
-
await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
|
|
1376
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1377
|
-
continue;
|
|
1378
|
-
}
|
|
1379
|
-
await this.persistence.saveRecoveryIntent(thread.threadId, thread.latestRunId, {
|
|
1380
|
-
...recoveryIntent,
|
|
1381
|
-
attempts: recoveryIntent.attempts + 1,
|
|
1382
|
-
});
|
|
1383
|
-
await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
|
|
1384
|
-
resumeKind: "startup-recovery",
|
|
1385
|
-
checkpointRef: recoveryIntent.checkpointRef,
|
|
1386
|
-
state: "resuming",
|
|
1387
|
-
});
|
|
1388
|
-
const history = await this.persistence.listThreadMessages(thread.threadId);
|
|
1389
|
-
const priorHistory = history.filter((message) => message.runId !== thread.latestRunId);
|
|
1390
|
-
const runInput = await this.loadRunInput(thread.threadId, thread.latestRunId);
|
|
1391
|
-
const startedAt = Date.now();
|
|
1392
|
-
try {
|
|
1393
|
-
const actual = await this.runtimeAdapter.invoke(binding, "", thread.threadId, thread.latestRunId, recoveryIntent.resumePayload, priorHistory);
|
|
1394
|
-
this.healthMonitor.recordLlmSuccess(Date.now() - startedAt);
|
|
1395
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1396
|
-
await this.finalizeContinuedRun(binding, thread.threadId, thread.latestRunId, runInput, actual, {
|
|
1397
|
-
previousState: "resuming",
|
|
1398
|
-
stateSequence: 101,
|
|
1399
|
-
approvalSequence: 102,
|
|
1400
|
-
});
|
|
1401
|
-
}
|
|
1402
|
-
catch (error) {
|
|
1403
|
-
this.healthMonitor.recordLlmFailure(Date.now() - startedAt);
|
|
1404
|
-
if (recoveryIntent.attempts + 1 >= this.recoveryConfig.maxRecoveryAttempts) {
|
|
1405
|
-
await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
|
|
1406
|
-
await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
|
|
1407
|
-
await this.emit(thread.threadId, thread.latestRunId, 101, "run.state.changed", {
|
|
1408
|
-
previousState: "resuming",
|
|
1409
|
-
state: "failed",
|
|
1410
|
-
checkpointRef: recoveryIntent.checkpointRef,
|
|
1411
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1412
|
-
});
|
|
1413
|
-
}
|
|
1414
|
-
}
|
|
1415
1236
|
}
|
|
1416
1237
|
}
|
|
1417
1238
|
async reclaimExpiredClaimedRuns(nowIso = new Date().toISOString()) {
|
|
@@ -1459,4 +1280,3 @@ export class AgentHarnessRuntime {
|
|
|
1459
1280
|
return nowMs - heartbeatAtMs >= this.concurrencyConfig.heartbeatTimeoutMs;
|
|
1460
1281
|
}
|
|
1461
1282
|
}
|
|
1462
|
-
export { AgentHarnessRuntime as AgentHarness };
|