@assemblyline-agents/runtime 6.1.5 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/background-review.d.ts.map +1 -1
- package/dist/background-review.js +10 -1
- package/dist/background-review.js.map +1 -1
- package/dist/background-subagents.d.ts +9 -11
- package/dist/background-subagents.d.ts.map +1 -1
- package/dist/background-subagents.js +67 -126
- package/dist/background-subagents.js.map +1 -1
- package/dist/background-work-summary.d.ts +21 -0
- package/dist/background-work-summary.d.ts.map +1 -0
- package/dist/background-work-summary.js +94 -0
- package/dist/background-work-summary.js.map +1 -0
- package/dist/background-work-tool.d.ts +7 -0
- package/dist/background-work-tool.d.ts.map +1 -0
- package/dist/background-work-tool.js +73 -0
- package/dist/background-work-tool.js.map +1 -0
- package/dist/builtin-tool-executors.d.ts.map +1 -1
- package/dist/builtin-tool-executors.js +4 -0
- package/dist/builtin-tool-executors.js.map +1 -1
- package/dist/builtin-tools.d.ts +1 -1
- package/dist/builtin-tools.d.ts.map +1 -1
- package/dist/builtin-tools.js +6 -0
- package/dist/builtin-tools.js.map +1 -1
- package/dist/channel.d.ts +3 -1
- package/dist/channel.d.ts.map +1 -1
- package/dist/connection-wait-recovery.d.ts +5 -0
- package/dist/connection-wait-recovery.d.ts.map +1 -0
- package/dist/connection-wait-recovery.js +106 -0
- package/dist/connection-wait-recovery.js.map +1 -0
- package/dist/conversation-turn-dispatch.d.ts.map +1 -1
- package/dist/conversation-turn-dispatch.js +2 -1
- package/dist/conversation-turn-dispatch.js.map +1 -1
- package/dist/delivery-queue.d.ts.map +1 -1
- package/dist/delivery-queue.js +2 -0
- package/dist/delivery-queue.js.map +1 -1
- package/dist/delivery-send.d.ts.map +1 -1
- package/dist/delivery-send.js +8 -6
- package/dist/delivery-send.js.map +1 -1
- package/dist/harness-execution-context.d.ts +2 -0
- package/dist/harness-execution-context.d.ts.map +1 -1
- package/dist/harness-execution-context.js +29 -3
- package/dist/harness-execution-context.js.map +1 -1
- package/dist/harness-tool-execution.d.ts.map +1 -1
- package/dist/harness-tool-execution.js +45 -72
- package/dist/harness-tool-execution.js.map +1 -1
- package/dist/harness-turn-execution.d.ts.map +1 -1
- package/dist/harness-turn-execution.js +8 -1
- package/dist/harness-turn-execution.js.map +1 -1
- package/dist/in-memory-run-store.d.ts +5 -0
- package/dist/in-memory-run-store.d.ts.map +1 -1
- package/dist/in-memory-run-store.js +34 -82
- package/dist/in-memory-run-store.js.map +1 -1
- package/dist/in-memory-run-transactions.d.ts +14 -0
- package/dist/in-memory-run-transactions.d.ts.map +1 -0
- package/dist/in-memory-run-transactions.js +163 -0
- package/dist/in-memory-run-transactions.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/operator-controls.d.ts.map +1 -1
- package/dist/operator-controls.js +10 -21
- package/dist/operator-controls.js.map +1 -1
- package/dist/read-tool-result.d.ts +3 -0
- package/dist/read-tool-result.d.ts.map +1 -0
- package/dist/read-tool-result.js +25 -0
- package/dist/read-tool-result.js.map +1 -0
- package/dist/replay-resume-authorization.d.ts.map +1 -1
- package/dist/replay-resume-authorization.js +4 -14
- package/dist/replay-resume-authorization.js.map +1 -1
- package/dist/replay-resume-connection.d.ts.map +1 -1
- package/dist/replay-resume-connection.js +3 -1
- package/dist/replay-resume-connection.js.map +1 -1
- package/dist/replay-resume-controls.d.ts.map +1 -1
- package/dist/replay-resume-controls.js +43 -27
- package/dist/replay-resume-controls.js.map +1 -1
- package/dist/replay-resume-recovery.d.ts.map +1 -1
- package/dist/replay-resume-recovery.js +5 -1
- package/dist/replay-resume-recovery.js.map +1 -1
- package/dist/replay-resume-shared.d.ts +3 -4
- package/dist/replay-resume-shared.d.ts.map +1 -1
- package/dist/replay-resume-shared.js +4 -13
- package/dist/replay-resume-shared.js.map +1 -1
- package/dist/replay-resume-types.d.ts +3 -1
- package/dist/replay-resume-types.d.ts.map +1 -1
- package/dist/run-command-settlement.d.ts +6 -0
- package/dist/run-command-settlement.d.ts.map +1 -0
- package/dist/run-command-settlement.js +35 -0
- package/dist/run-command-settlement.js.map +1 -0
- package/dist/run-commands.d.ts +16 -0
- package/dist/run-commands.d.ts.map +1 -0
- package/dist/run-commands.js +195 -0
- package/dist/run-commands.js.map +1 -0
- package/dist/run-execution-host.d.ts +1 -0
- package/dist/run-execution-host.d.ts.map +1 -1
- package/dist/run-execution-lease.d.ts +15 -0
- package/dist/run-execution-lease.d.ts.map +1 -0
- package/dist/run-execution-lease.js +160 -0
- package/dist/run-execution-lease.js.map +1 -0
- package/dist/run-execution.d.ts.map +1 -1
- package/dist/run-execution.js +2 -0
- package/dist/run-execution.js.map +1 -1
- package/dist/run-heartbeat.d.ts.map +1 -1
- package/dist/run-heartbeat.js +25 -0
- package/dist/run-heartbeat.js.map +1 -1
- package/dist/run-initialization.d.ts.map +1 -1
- package/dist/run-initialization.js +7 -0
- package/dist/run-initialization.js.map +1 -1
- package/dist/run-outcome.d.ts.map +1 -1
- package/dist/run-outcome.js +5 -7
- package/dist/run-outcome.js.map +1 -1
- package/dist/run-recovery.d.ts +2 -2
- package/dist/run-recovery.d.ts.map +1 -1
- package/dist/run-recovery.js +73 -212
- package/dist/run-recovery.js.map +1 -1
- package/dist/run-stream.d.ts.map +1 -1
- package/dist/run-stream.js +9 -3
- package/dist/run-stream.js.map +1 -1
- package/dist/run-tree-cancellation.d.ts +9 -0
- package/dist/run-tree-cancellation.d.ts.map +1 -0
- package/dist/run-tree-cancellation.js +166 -0
- package/dist/run-tree-cancellation.js.map +1 -0
- package/dist/run-tree.d.ts +9 -0
- package/dist/run-tree.d.ts.map +1 -0
- package/dist/run-tree.js +37 -0
- package/dist/run-tree.js.map +1 -0
- package/dist/runtime-engine-base.d.ts +1 -0
- package/dist/runtime-engine-base.d.ts.map +1 -1
- package/dist/runtime-engine-base.js +11 -1
- package/dist/runtime-engine-base.js.map +1 -1
- package/dist/runtime-engine-capabilities.d.ts.map +1 -1
- package/dist/runtime-engine-capabilities.js +5 -2
- package/dist/runtime-engine-capabilities.js.map +1 -1
- package/dist/runtime-engine-config.d.ts.map +1 -1
- package/dist/runtime-engine-config.js +3 -2
- package/dist/runtime-engine-config.js.map +1 -1
- package/dist/runtime-engine-operations.d.ts +12 -2
- package/dist/runtime-engine-operations.d.ts.map +1 -1
- package/dist/runtime-engine-operations.js +52 -5
- package/dist/runtime-engine-operations.js.map +1 -1
- package/dist/runtime-engine-run.d.ts.map +1 -1
- package/dist/runtime-engine-run.js +3 -0
- package/dist/runtime-engine-run.js.map +1 -1
- package/dist/runtime-types.d.ts +7 -1
- package/dist/runtime-types.d.ts.map +1 -1
- package/dist/state/run-lifecycle.d.ts +111 -0
- package/dist/state/run-lifecycle.d.ts.map +1 -0
- package/dist/state/run-lifecycle.js +70 -0
- package/dist/state/run-lifecycle.js.map +1 -0
- package/dist/state/run-state.d.ts +1 -0
- package/dist/state/run-state.d.ts.map +1 -1
- package/dist/state/run-state.js.map +1 -1
- package/dist/state/store-contracts.d.ts +3 -0
- package/dist/state/store-contracts.d.ts.map +1 -1
- package/dist/state/store-contracts.js +1 -1
- package/dist/state/store-contracts.js.map +1 -1
- package/dist/state/task-budget.d.ts +39 -0
- package/dist/state/task-budget.d.ts.map +1 -0
- package/dist/state/task-budget.js +2 -0
- package/dist/state/task-budget.js.map +1 -0
- package/dist/state-stores.d.ts.map +1 -1
- package/dist/state-stores.js +1 -0
- package/dist/state-stores.js.map +1 -1
- package/dist/state.d.ts +2 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +2 -0
- package/dist/state.js.map +1 -1
- package/dist/storage.d.ts +3 -0
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +6 -0
- package/dist/storage.js.map +1 -1
- package/dist/subagent-background-resume.d.ts +2 -2
- package/dist/subagent-background-resume.d.ts.map +1 -1
- package/dist/subagent-background-resume.js +11 -41
- package/dist/subagent-background-resume.js.map +1 -1
- package/dist/subagent-delegate-tool.d.ts +1 -1
- package/dist/subagent-delegate-tool.d.ts.map +1 -1
- package/dist/subagent-delegate-tool.js +20 -4
- package/dist/subagent-delegate-tool.js.map +1 -1
- package/dist/subagent-execution.d.ts +3 -1
- package/dist/subagent-execution.d.ts.map +1 -1
- package/dist/subagent-execution.js +61 -21
- package/dist/subagent-execution.js.map +1 -1
- package/dist/subagent-settlement.d.ts +1 -1
- package/dist/subagent-settlement.d.ts.map +1 -1
- package/dist/subagent-settlement.js +67 -43
- package/dist/subagent-settlement.js.map +1 -1
- package/dist/task-budget-accounting.d.ts +13 -0
- package/dist/task-budget-accounting.d.ts.map +1 -0
- package/dist/task-budget-accounting.js +35 -0
- package/dist/task-budget-accounting.js.map +1 -0
- package/dist/task-budget-config.d.ts +5 -0
- package/dist/task-budget-config.d.ts.map +1 -0
- package/dist/task-budget-config.js +55 -0
- package/dist/task-budget-config.js.map +1 -0
- package/dist/task-budget-harness.d.ts +5 -0
- package/dist/task-budget-harness.d.ts.map +1 -0
- package/dist/task-budget-harness.js +44 -0
- package/dist/task-budget-harness.js.map +1 -0
- package/dist/task-budget.d.ts +14 -0
- package/dist/task-budget.d.ts.map +1 -0
- package/dist/task-budget.js +212 -0
- package/dist/task-budget.js.map +1 -0
- package/dist/task-deadline.d.ts +4 -0
- package/dist/task-deadline.d.ts.map +1 -0
- package/dist/task-deadline.js +36 -0
- package/dist/task-deadline.js.map +1 -0
- package/dist/tool-execution-host.d.ts +4 -2
- package/dist/tool-execution-host.d.ts.map +1 -1
- package/dist/tool-execution.d.ts.map +1 -1
- package/dist/tool-execution.js +24 -9
- package/dist/tool-execution.js.map +1 -1
- package/dist/tool-result-output.d.ts +14 -0
- package/dist/tool-result-output.d.ts.map +1 -0
- package/dist/tool-result-output.js +66 -0
- package/dist/tool-result-output.js.map +1 -0
- package/package.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-initialization.d.ts","sourceRoot":"","sources":["../src/run-initialization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-initialization.d.ts","sourceRoot":"","sources":["../src/run-initialization.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,cAAc,EACd,eAAe,EAEhB,MAAM,2BAA2B,CAAC;AAgBnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAG1E,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,eAAe,GAAG,SAAS,CAAC;IAC/C,SAAS,EAAE,cAAc,GAAG,SAAS,CAAC;IACtC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,GAAG,EAAE,SAAS,CAAC;CAChB;AAED,MAAM,MAAM,uBAAuB,GAC/B,iBAAiB,GACjB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,cAAc,CAAA;CAAE,CAAC;AAEjD,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,gBAAgB,EACtB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,uBAAuB,CAAC,CAkMlC"}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { createTaskBudget, taskBudgetLimits } from "./task-budget-config.js";
|
|
2
|
+
import { initialRunLifecycle } from "./state.js";
|
|
1
3
|
import { durableAgentSurfaceScope, surfaceForPath } from "./agent-surface.js";
|
|
2
4
|
import { normalizeAgentAudience } from "./audience.js";
|
|
3
5
|
import { acceptBackgroundSubagentHandoff } from "./subagent-handoff.js";
|
|
@@ -60,6 +62,7 @@ export async function initializeRunExecution(host, options) {
|
|
|
60
62
|
});
|
|
61
63
|
const runInput = {
|
|
62
64
|
message: options.message,
|
|
65
|
+
...(options.target ? { target: safeJson(options.target) } : {}),
|
|
63
66
|
agentSurface: surfacePath,
|
|
64
67
|
channel,
|
|
65
68
|
eventId: options.turn?.eventId ?? options.eventId ?? null,
|
|
@@ -102,8 +105,11 @@ export async function initializeRunExecution(host, options) {
|
|
|
102
105
|
? { defaultChannel: executionSurface.defaultOutboundChannel }
|
|
103
106
|
: {}),
|
|
104
107
|
}, options, runInput);
|
|
108
|
+
const newBudget = !options.existingRun || (options.existingRun.status === "created" && !options.existingRun.input.parentRunId && !options.existingRun.lifecycle?.taskBudget);
|
|
109
|
+
const taskBudget = newBudget ? createTaskBudget(taskBudgetLimits(host.taskBudget, options.taskBudget), options.existingRun ? new Date(options.existingRun.createdAt) : host.now()) : undefined;
|
|
105
110
|
let run = options.existingRun
|
|
106
111
|
? await host.state.updateRun(options.existingRun.id, {
|
|
112
|
+
...(taskBudget ? { lifecycle: { ...(options.existingRun.lifecycle ?? initialRunLifecycle()), taskBudget } } : {}),
|
|
107
113
|
input: {
|
|
108
114
|
...options.existingRun.input,
|
|
109
115
|
...runInput,
|
|
@@ -113,6 +119,7 @@ export async function initializeRunExecution(host, options) {
|
|
|
113
119
|
agentRevision: host.manifest.agentRevision,
|
|
114
120
|
status: "created",
|
|
115
121
|
input: runInput,
|
|
122
|
+
lifecycle: { ...initialRunLifecycle(), ...(taskBudget ? { taskBudget } : {}) },
|
|
116
123
|
});
|
|
117
124
|
try {
|
|
118
125
|
await options.onRunCreated?.(run);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-initialization.js","sourceRoot":"","sources":["../src/run-initialization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-initialization.js","sourceRoot":"","sources":["../src/run-initialization.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAMjD,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,+BAA+B,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAIzD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAgB9C,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAsB,EACtB,OAAwB;IAExB,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;IACvC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAChC,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,SAAS,EAAE,CAAC;IAClC,MAAM,WAAW,GACf,CAAC,OAAO,CAAC,WAAW;QACpB,OAAO,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,KAAK,QAAQ;QACxD,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY;QACxC,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,CAAC,YAAY;QACpB,MAAM,CAAC;IACT,MAAM,gBAAgB,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACpE,MAAM,OAAO,GACX,CAAC,OAAO,CAAC,WAAW;QACpB,OAAO,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ;QACnD,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO;QACnC,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,CAAC,IAAI,EAAE,OAAO;QACrB,OAAO,CAAC,OAAO;QACf,OAAO,CAAC;IACV,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CACtD,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,OAAO,CAC1C,CAAC;IACF,MAAM,sBAAsB,GAC1B,CAAC,OAAO,CAAC,WAAW;QACpB,OAAO,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,KAAK,QAAQ;QAC1D,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc;QAC1C,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,CAAC,IAAI,EAAE,cAAc;QAC5B,OAAO,CAAC,cAAc,CAAC;IACzB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;IAC5D,MAAM,SAAS,GACb,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC;QACrE,oBAAoB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GACb,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC;QACrE,SAAS,CAAC;IACZ,yEAAyE;IACzE,wEAAwE;IACxE,yDAAyD;IACzD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,WAAW;QACnC,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5D,CAAC,CAAC,SAAS,CAAC;QACZ,sBAAsB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI;QACpE,OAAO,EAAE,IAAI;KACd,CAAC;IACJ,MAAM,WAAW,GACf,OAAO,CAAC,WAAW,IAAI,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC;QACxE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW;QACvC,CAAC,CAAE,QAAQ,CAAC;YACR,OAAO;YACP,cAAc,EAAE,sBAAsB,IAAI,IAAI;YAC9C,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI;SAC7D,CAAgB,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,YAAY,EAAE,WAAW;QACzB,OAAO;QACP,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,IAAI;QACzD,cAAc,EACZ,sBAAsB;YACtB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,cAAc;YACzC,IAAI;QACN,gBAAgB,EAAE,sBAAsB;YACtC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,OAAO,CAAC,WAAW;gBACrB,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI;gBACrD,CAAC,CAAC,IAAI;QACR,MAAM,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI,YAAY,IAAI,IAAI;QAC1D,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;QACjD,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;QACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;QAC5B,SAAS,EACP,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;QACnE,WAAW,EACT,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;QACvE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;QACzE,QAAQ,EAAE,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAC1E,WAAW;QACX,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE;QAC1D,kBAAkB,EAChB,OAAO,CAAC,kBAAkB;YAC1B,CAAC,OAAO,CAAC,WAAW;gBACpB,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACxD,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,kBAAkB;gBAC9C,CAAC,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;YAC9B,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAe,EAAE;YACpD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS;YAC5B,CAAC,CAAC,EAAE,IAAI,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAA0B,EAAE;YACzE,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IACF,MAAM,4BAA4B,CAChC;QACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,wBAAwB,CAClC,IAAI,CAAC,iBAAiB,EAAE,EACxB,WAAW,CACZ;QACD,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;QACf,GAAG,CAAC,gBAAgB,CAAC,sBAAsB;YACzC,CAAC,CAAC,EAAE,cAAc,EAAE,gBAAgB,CAAC,sBAAsB,EAAE;YAC7D,CAAC,CAAC,EAAE,CAAC;KACR,EACD,OAAO,EACP,QAAQ,CACT,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAC7K,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/L,IAAI,GAAG,GAAG,OAAO,CAAC,WAAW;QAC3B,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE;YACjD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjH,KAAK,EAAE;gBACL,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK;gBAC5B,GAAG,QAAQ;aACZ;SACF,CAAC;QACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YACzB,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa;YAC1C,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,QAAQ;YACf,SAAS,EAAE,EAAE,GAAG,mBAAmB,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;SAC/E,CAAC,CAAC;IACP,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;YAC3C,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC,CAAC;IACL,CAAC;IACD,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,cAAc,EAAE;QACnD,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa;KAC3C,CAAC,CAAC;IACH,IAAI,CAAC;QACH,MAAM,+BAA+B,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,GAAG,GAAG,MAAM,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE;YACrD,MAAM,EAAE,yBAAyB;YACjC,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QACH,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,MAAM,IAAI,CAAC,MAAM,CACvB,GAAG,EACH,4BAA4B,OAAO,EAAE,EACrC,KAAK,EACL,KAAK,EACL,KAAK,CACN;SACF,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,MAAM;QAChB,MAAM,sBAAsB,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE,IACE,IAAI,CAAC,8BAA8B,EAAE;QACrC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAC7D,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,sBAAsB,EAAE;YAC3D,MAAM,EAAE,kBAAkB;YAC1B,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa;YAClC,OAAO,EACL,sFAAsF;SACzF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrD,IAAI,YAAY,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;QACzC,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,MAAM,IAAI,CAAC,MAAM,CACvB,YAAY,EACZ,6CAA6C,CAC9C;SACF,CAAC;IACJ,CAAC;IACD,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAEhE,OAAO;QACL,IAAI,EAAE,OAAO;QACb,aAAa;QACb,OAAO;QACP,iBAAiB;QACjB,SAAS;QACT,YAAY;QACZ,GAAG;KACJ,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-outcome.d.ts","sourceRoot":"","sources":["../src/run-outcome.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-outcome.d.ts","sourceRoot":"","sources":["../src/run-outcome.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAG5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE5E;;;;GAIG;AAEH,MAAM,MAAM,gBAAgB,GACxB,8BAA8B,GAC9B,wBAAwB,GACxB,0BAA0B,GAC1B,2BAA2B,GAC3B,sBAAsB,GACtB,6BAA6B,GAC7B,aAAa,GACb,6BAA6B,GAC7B,oBAAoB,GACpB,oBAAoB,GACpB,uBAAuB,GACvB,wBAAwB,GACxB,wBAAwB,GACxB,sBAAsB,GACtB,yBAAyB,GACzB,yBAAyB,GACzB,0BAA0B,GAC1B,6BAA6B,GAC7B,cAAc,GACd,sBAAsB,CAAC;AAE3B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,gBAAgB,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,wEAAwE;IACxE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,iFAAiF;AACjF,wBAAsB,OAAO,CAC3B,IAAI,EAAE;IAAE,KAAK,EAAE,YAAY,CAAA;CAAE,EAC7B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,SAAS,CAAC,CAUpB;AAED,6EAA6E;AAC7E,wBAAsB,mCAAmC,CACvD,IAAI,EAAE;IAAE,KAAK,EAAE,YAAY,CAAC;IAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAA;CAAE,EAC/E,GAAG,EAAE,SAAS,GACb,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA2BhC;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACxC,IAAI,CASN;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE;IAAE,KAAK,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,aAAa,CAAC;IAAC,wBAAwB,EAAE,MAAM,CAAC;IAAC,GAAG,IAAI,IAAI,CAAA;CAAE,EACnG,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC,CA8Cf"}
|
package/dist/run-outcome.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { settleStoppedRunCommands } from "./run-command-settlement.js";
|
|
1
2
|
import { modelErrorClassification } from "@assemblyline-agents/core";
|
|
2
3
|
import { enqueueRunFailureNotice } from "./delivery-queue.js";
|
|
3
4
|
/** Mark a run failed with a durable reason and append the `run.failed` event. */
|
|
@@ -9,13 +10,9 @@ export async function failRun(deps, runId, input) {
|
|
|
9
10
|
};
|
|
10
11
|
if (input.clearControlIntent)
|
|
11
12
|
patch.controlIntent = undefined;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
reason: input.reason,
|
|
16
|
-
...(input.data ?? {})
|
|
17
|
-
});
|
|
18
|
-
return run;
|
|
13
|
+
return (await deps.state.transactRun(runId, ({ run }) => run.status === "cancelled" ? undefined : {
|
|
14
|
+
patch, events: [{ type: "run.failed", data: { error: input.error, reason: input.reason, ...(input.data ?? {}) } }]
|
|
15
|
+
})).run;
|
|
19
16
|
}
|
|
20
17
|
/** Settle a control request that arrived during a forced direct-tool run. */
|
|
21
18
|
export async function settleCompletionBoundaryControlImpl(deps, run) {
|
|
@@ -25,6 +22,7 @@ export async function settleCompletionBoundaryControlImpl(deps, run) {
|
|
|
25
22
|
if (current.status === "cancelled") {
|
|
26
23
|
const cleared = await deps.state.updateRun(run.id, { controlIntent: undefined });
|
|
27
24
|
deps.runControlIntents.delete(run.id);
|
|
25
|
+
await settleStoppedRunCommands(deps.state, run.id);
|
|
28
26
|
return cleared;
|
|
29
27
|
}
|
|
30
28
|
if (!deps.state.settleRunControl) {
|
package/dist/run-outcome.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-outcome.js","sourceRoot":"","sources":["../src/run-outcome.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-outcome.js","sourceRoot":"","sources":["../src/run-outcome.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAyC9D,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAA6B,EAC7B,KAAa,EACb,KAAsB;IAEtB,MAAM,KAAK,GAAiD;QAC1D,MAAM,EAAE,QAAQ;QAChB,cAAc,EAAE,KAAK,CAAC,MAAM;QAC5B,aAAa,EAAE,KAAK,CAAC,KAAK;KAC3B,CAAC;IACF,IAAI,KAAK,CAAC,kBAAkB;QAAE,KAAK,CAAC,aAAa,GAAG,SAAS,CAAC;IAC9D,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAChG,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;KACnH,CAAC,CAAC,CAAC,GAAG,CAAC;AACV,CAAC;AAED,6EAA6E;AAC7E,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,IAA+E,EAC/E,GAAc;IAEd,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO,EAAE,aAAa;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1D,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QACrG,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,oBAAoB,EAAE,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QACvH,OAAO,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE;YAC5C,MAAM,EAAE,oBAAoB;YAC5B,KAAK,EAAE,6FAA6F;YACpG,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAqB,EACrB,KAAa,EACb,KAAyC;IAEzC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC;IACzC,MAAM,MAAM,GAAe,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9C,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;IACzE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAmG,EACnG,KAAa,EACb,KAAc;IAEd,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,KAAK,YAAY,KAAK,IAAI,CAC5B,KAAK,CAAC,IAAI,KAAK,0BAA0B,IAAI,KAAK,CAAC,IAAI,KAAK,+BAA+B,CAC5F;QAAE,OAAO;IACV,IAAI,wBAAwB,CAAC,KAAK,CAAC,KAAK,OAAO,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC9F,MAAM,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;gBAC9C,KAAK;gBACL,KAAK,EAAE,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACnE,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;QACL,CAAC;QACD,OAAO;IACT,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,uBAAuB,CAAC,CAAC;QAC3F,MAAM,WAAW,GAAG,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACvF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,sBAAsB,CAAC,CAAC;QAC/F,IAAI,CAAC,mBAAmB,IAAI,CAAC,SAAS,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC/G,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBACnG,MAAM,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,OAAO;QACT,CAAC;IACH,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE;YAC7D,KAAK;YACL,KAAK,EAAE,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACnE,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;IACL,CAAC;IACD,MAAM,MAAM,GAAe,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACxE,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IACtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACpG,CAAC;IAAC,MAAM,CAAC;QACP,+EAA+E;IACjF,CAAC;AACH,CAAC"}
|
package/dist/run-recovery.d.ts
CHANGED
|
@@ -8,13 +8,13 @@ import type { ConnectionAuthorizationSessionStore } from "./connections.js";
|
|
|
8
8
|
* Orphaned-run recovery: a staleness-guarded sweep that repairs runs left in
|
|
9
9
|
* `created`/`running` after a crash. Live runs are protected by a heartbeat
|
|
10
10
|
* that bumps `updatedAt`; multi-replica safety comes from claiming each
|
|
11
|
-
* candidate through
|
|
11
|
+
* candidate through a renewable, fenced execution lease.
|
|
12
12
|
*/
|
|
13
13
|
export interface RunRecoveryOptions {
|
|
14
14
|
now?: Date;
|
|
15
15
|
/** Only runs idle longer than this are candidates. Default max(5min, 4x heartbeat). */
|
|
16
16
|
staleAfterMs?: number;
|
|
17
|
-
/** In-place continuation resume attempts
|
|
17
|
+
/** In-place continuation resume attempts without checkpoint progress before failing. Default 5. */
|
|
18
18
|
maxResumeAttempts?: number;
|
|
19
19
|
}
|
|
20
20
|
export interface RunRecoveryHost {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-recovery.d.ts","sourceRoot":"","sources":["../src/run-recovery.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-recovery.d.ts","sourceRoot":"","sources":["../src/run-recovery.ts"],"names":[],"mappings":"AAMA,OAAO,EAAsB,KAAK,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEhF,OAAO,KAAK,EAAY,SAAS,EAAE,YAAY,EAAkB,MAAM,YAAY,CAAC;AAIpF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,mCAAmC,EAAE,MAAM,kBAAkB,CAAC;AAI5E;;;;;GAKG;AAEH,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mGAAmG;IACnG,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,GAAG,IAAI,IAAI,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,wBAAwB,EAAE,MAAM,CAAC;IACjC,mCAAmC,EAAE,mCAAmC,CAAC;IACzE,gFAAgF;IAChF,0BAA0B,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IAClE;;;;;;;OAOG;IACH,yBAAyB,CACvB,GAAG,EAAE,SAAS,EACd,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAA;KAAE,GAClD,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IAClC,uEAAuE;IACvE,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACpE,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CAChE;AAED,2FAA2F;AAC3F,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,sFAAsF;IACtF,WAAW,EAAE,OAAO,CAAC;CACtB;AA4BD,wBAAgB,sBAAsB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAErE;AAED,wBAAsB,yBAAyB,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAqC7H;AA8aD;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EAChG,mBAAmB,EAAE,MAAM,GAAG,SAAS,EACvC,gBAAgB,EAAE,KAAK,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,GAChE,MAAM,GAAG,SAAS,CAyBpB;AAsID,MAAM,WAAW,uBAAuB;IACtC,cAAc,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5F,iBAAiB,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/F,sBAAsB,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,wBAAwB,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7D,WAAW,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACvE,sBAAsB,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAChG,qBAAqB,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACpG;AAED,MAAM,WAAW,2BAA2B;IAC1C,kDAAkD;IAClD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,IAAI,IAAI,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,EAAE,aAAa,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,mBAAmB,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,GAAG,SAAS,CAAC;IAC9H,sBAAsB,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,GAAG,SAAS,CAAC;IACjI,2BAA2B,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,CAAC;IACjF,6BAA6B,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,CAAC;IACnF,2BAA2B,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,GAAG,SAAS,CAAC;IAClI,0BAA0B,CAAC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,IAAI,IAAI,CAAA;KAAE,CAAC;IACzH,qBAAqB,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1E,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAClC;AAYD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,oBAAoB,EAAE,OAAO,GAAE,uBAA4B,GAAG,2BAA2B,CA6DzI"}
|
package/dist/run-recovery.js
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { recoverRunCancellations } from "./run-tree-cancellation.js";
|
|
2
|
+
import { recoverTaskBudgets } from "./task-budget.js";
|
|
3
|
+
import { settleStoppedRunCommands } from "./run-command-settlement.js";
|
|
4
|
+
import { recoverStuckConnectionWaits } from "./connection-wait-recovery.js";
|
|
5
|
+
import { withRunExecutionLease } from "./run-execution-lease.js";
|
|
6
|
+
import { DEFAULT_RUN_RECOVERY_ATTEMPTS } from "./state/run-lifecycle.js";
|
|
1
7
|
import { DELEGATE_TOOL_NAME } from "@assemblyline-agents/core";
|
|
2
8
|
import { assistantTextFromContinuation } from "@assemblyline-agents/pi";
|
|
3
9
|
import { enqueueRunFailureNotice, finalDeliveryIdempotencyKey } from "./delivery-queue.js";
|
|
@@ -29,6 +35,8 @@ export function defaultRunStaleAfterMs(runHeartbeatMs) {
|
|
|
29
35
|
}
|
|
30
36
|
export async function recoverIncompleteRunsImpl(host, options = {}) {
|
|
31
37
|
const { state } = host;
|
|
38
|
+
await recoverTaskBudgets(state);
|
|
39
|
+
await recoverRunCancellations(state);
|
|
32
40
|
const now = options.now ?? host.now();
|
|
33
41
|
await recoverSandboxSyncState(host, now);
|
|
34
42
|
const staleAfterMs = options.staleAfterMs ?? defaultRunStaleAfterMs(host.runHeartbeatMs);
|
|
@@ -37,12 +45,17 @@ export async function recoverIncompleteRunsImpl(host, options = {}) {
|
|
|
37
45
|
const candidates = state.listRunsByStatus
|
|
38
46
|
? await state.listRunsByStatus(["created", "running"], { updatedBefore: cutoffIso })
|
|
39
47
|
: await state.listRuns();
|
|
48
|
+
const expiredChildren = await state.queryRuns({ statuses: ["running"], background: true, leaseAvailable: true });
|
|
49
|
+
const selected = new Map(candidates.map(run => [run.id, run]));
|
|
50
|
+
for (const run of expiredChildren)
|
|
51
|
+
if (run.lifecycle?.lease)
|
|
52
|
+
selected.set(run.id, run);
|
|
40
53
|
const recovered = [];
|
|
41
|
-
const sweep = { maxResumeAttempts: options.maxResumeAttempts ??
|
|
42
|
-
for (const run of
|
|
54
|
+
const sweep = { maxResumeAttempts: options.maxResumeAttempts ?? DEFAULT_RUN_RECOVERY_ATTEMPTS, cutoff };
|
|
55
|
+
for (const run of selected.values()) {
|
|
43
56
|
if (!ORPHAN_CANDIDATE_STATUSES.has(run.status))
|
|
44
57
|
continue;
|
|
45
|
-
if (Date.parse(run.updatedAt) > cutoff)
|
|
58
|
+
if (Date.parse(run.updatedAt) > cutoff && !(run.lifecycle?.lease && Date.parse(run.lifecycle.lease.expiresAt) <= Date.now()))
|
|
46
59
|
continue;
|
|
47
60
|
try {
|
|
48
61
|
recovered.push(...await claimAndRecover(host, run, sweep));
|
|
@@ -56,227 +69,62 @@ export async function recoverIncompleteRunsImpl(host, options = {}) {
|
|
|
56
69
|
}
|
|
57
70
|
}
|
|
58
71
|
await recoverStuckConnectionWaits(host, cutoff, now, recovered);
|
|
59
|
-
await
|
|
60
|
-
return recovered;
|
|
61
|
-
}
|
|
62
|
-
const SETTLEMENT_SWEEP_SCOPE = "run-recovery";
|
|
63
|
-
const SETTLEMENT_WATERMARK_KEY = "subagent-settlement-watermark";
|
|
64
|
-
const SETTLEMENT_SWEEP_LIMIT = 200;
|
|
65
|
-
const TERMINAL_STATUS_LIST = ["completed", "failed", "cancelled"];
|
|
66
|
-
const SETTLEMENT_MARKER_EVENTS = new Set(["run.completed", "run.failed", "run.cancelled", "run.recovered"]);
|
|
67
|
-
/**
|
|
68
|
-
* A child that reached a terminal status but whose process died before
|
|
69
|
-
* settlement reported it (subagent-settlement.ts) is never `created`/`running`
|
|
70
|
-
* again, so the orphan sweep alone would miss it. Walk terminal runs newer
|
|
71
|
-
* than a durable watermark and settle the child runs that carry no
|
|
72
|
-
* `subagent.settled` after their latest terminal transition. The watermark
|
|
73
|
-
* advances to the newest run of a full page (so a backlog is paged through
|
|
74
|
-
* across sweeps) and otherwise trails `now` by one staleness window, so a
|
|
75
|
-
* terminal write racing this listing is seen by the next sweep; settlement's
|
|
76
|
-
* reservation makes the overlap idempotent.
|
|
77
|
-
*/
|
|
78
|
-
async function settleUnsettledChildRuns(host, now, staleAfterMs) {
|
|
79
|
-
const { state } = host;
|
|
80
|
-
const stored = (await state.getSetting(SETTLEMENT_SWEEP_SCOPE, SETTLEMENT_WATERMARK_KEY))?.value;
|
|
81
|
-
const watermark = typeof stored === "string" ? stored : undefined;
|
|
82
|
-
const candidates = state.listRunsByStatus
|
|
83
|
-
? await state.listRunsByStatus(TERMINAL_STATUS_LIST, {
|
|
84
|
-
...(watermark ? { updatedAfter: watermark } : {}),
|
|
85
|
-
limit: SETTLEMENT_SWEEP_LIMIT
|
|
86
|
-
})
|
|
87
|
-
: (await state.listRuns())
|
|
88
|
-
.filter((run) => TERMINAL_STATUS_LIST.includes(run.status) && (!watermark || run.updatedAt > watermark))
|
|
89
|
-
.sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))
|
|
90
|
-
.slice(0, SETTLEMENT_SWEEP_LIMIT);
|
|
91
|
-
for (const run of candidates) {
|
|
92
|
-
if (!isSubagentRun(run))
|
|
93
|
-
continue;
|
|
72
|
+
for (const run of await state.queryRuns({ pendingCommands: true, statuses: ["suspended", "completed", "failed", "cancelled"] })) {
|
|
94
73
|
try {
|
|
95
|
-
|
|
96
|
-
continue;
|
|
97
|
-
await host.settleSubagentRun(await host.replay(run.id));
|
|
74
|
+
await settleStoppedRunCommands(state, run.id);
|
|
98
75
|
}
|
|
99
76
|
catch (error) {
|
|
100
|
-
host.logger.warn("
|
|
77
|
+
host.logger.warn("command receipt recovery failed", { runId: run.id, error: errorMessage(error) });
|
|
101
78
|
}
|
|
102
79
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
: new Date(now.getTime() - staleAfterMs).toISOString();
|
|
106
|
-
if (!watermark || next > watermark)
|
|
107
|
-
await state.setSetting(SETTLEMENT_SWEEP_SCOPE, SETTLEMENT_WATERMARK_KEY, next);
|
|
108
|
-
}
|
|
109
|
-
function settledAfterTerminalTransition(events) {
|
|
110
|
-
let marker = 0;
|
|
111
|
-
let settled = 0;
|
|
112
|
-
for (const event of events) {
|
|
113
|
-
if (SETTLEMENT_MARKER_EVENTS.has(event.type))
|
|
114
|
-
marker = Math.max(marker, event.sequence);
|
|
115
|
-
if (event.type === "subagent.settled")
|
|
116
|
-
settled = Math.max(settled, event.sequence);
|
|
117
|
-
}
|
|
118
|
-
return settled > marker;
|
|
80
|
+
await settleUnsettledChildRuns(host);
|
|
81
|
+
return recovered;
|
|
119
82
|
}
|
|
120
|
-
/**
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const candidates = state.listRunsByStatus
|
|
130
|
-
? await state.listRunsByStatus(["waiting_for_connection"], { updatedBefore: cutoffIso })
|
|
131
|
-
: (await state.listRuns()).filter((run) => run.status === "waiting_for_connection");
|
|
132
|
-
for (const run of candidates) {
|
|
133
|
-
if (run.status !== "waiting_for_connection" || Date.parse(run.updatedAt) > cutoff)
|
|
134
|
-
continue;
|
|
135
|
-
try {
|
|
136
|
-
let current = await state.getRun(run.id);
|
|
137
|
-
if (!current || current.status !== "waiting_for_connection" || Date.parse(current.updatedAt) > cutoff)
|
|
138
|
-
continue;
|
|
139
|
-
const events = await state.listEvents(run.id);
|
|
140
|
-
// A live OAuth session has a callback resume path and human latency
|
|
141
|
-
// applies. Expired/deleted sessions do not: terminalize them so a legacy
|
|
142
|
-
// parked run cannot retain its conversation mailbox forever.
|
|
143
|
-
const authRequired = events.filter((event) => event.type === "authorization.required").length;
|
|
144
|
-
const authCompleted = events.filter((event) => event.type === "authorization.completed").length;
|
|
145
|
-
if (authRequired > authCompleted) {
|
|
146
|
-
const requiredEvents = events.filter((event) => event.type === "authorization.required");
|
|
147
|
-
const sessions = await Promise.all(requiredEvents.map(async (event) => {
|
|
148
|
-
const challenge = event.data.challenge;
|
|
149
|
-
const sessionId = challenge && typeof challenge === "object" && !Array.isArray(challenge)
|
|
150
|
-
&& typeof challenge.sessionId === "string"
|
|
151
|
-
? String(challenge.sessionId)
|
|
152
|
-
: undefined;
|
|
153
|
-
return sessionId
|
|
154
|
-
? host.connectionAuthorizationSessionStore.getSession(sessionId)
|
|
155
|
-
: undefined;
|
|
156
|
-
}));
|
|
157
|
-
const live = sessions.some((session) => session?.status === "pending" && session.expiresAt > recoveryNow.getTime());
|
|
158
|
-
if (live)
|
|
159
|
-
continue;
|
|
160
|
-
const claimed = await claimConnectionWaitRecovery(state, current, cutoff);
|
|
161
|
-
if (!claimed)
|
|
162
|
-
continue;
|
|
163
|
-
current = claimed;
|
|
164
|
-
for (const session of sessions) {
|
|
165
|
-
if (session?.status === "pending" && session.expiresAt <= recoveryNow.getTime()) {
|
|
166
|
-
await host.connectionAuthorizationSessionStore.updateSession(session.id, { status: "expired" });
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
const message = "Connection authorization expired before completion. Re-run the request to start a fresh authorization flow.";
|
|
170
|
-
const failed = await failRun({ state }, run.id, {
|
|
171
|
-
reason: "connection.unavailable",
|
|
172
|
-
error: message,
|
|
173
|
-
data: { recovered: true, authorizationExpired: true }
|
|
174
|
-
});
|
|
175
|
-
await enqueueRunFailureNotice(host, current, message);
|
|
176
|
-
await state.appendEvent(run.id, "run.recovered", {
|
|
177
|
-
strategy: "expired-authorization-marked-failed"
|
|
178
|
-
});
|
|
179
|
-
recovered.push(failed);
|
|
180
|
-
continue;
|
|
83
|
+
/** Outstanding notifications remain queryable until reporting succeeds. */
|
|
84
|
+
async function settleUnsettledChildRuns(host) {
|
|
85
|
+
const updatedBefore = new Date().toISOString();
|
|
86
|
+
let after;
|
|
87
|
+
for (;;) {
|
|
88
|
+
const candidates = await host.state.queryRuns({ pendingNotification: true, updatedBefore, ...(after ? { after } : {}), limit: 200 });
|
|
89
|
+
for (const run of candidates) {
|
|
90
|
+
try {
|
|
91
|
+
await host.settleSubagentRun(await host.replay(run.id));
|
|
181
92
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
continue;
|
|
185
|
-
current = claimed;
|
|
186
|
-
if (state.touchRun)
|
|
187
|
-
await state.touchRun(run.id);
|
|
188
|
-
else
|
|
189
|
-
await state.updateRun(run.id, {});
|
|
190
|
-
const missing = lastMissingConnections(events);
|
|
191
|
-
if (missing.length > 0 && host.missingRequiredConnections(missing).length === 0) {
|
|
192
|
-
try {
|
|
193
|
-
const resumed = await host.resumeRunFromContinuation(current);
|
|
194
|
-
if (resumed) {
|
|
195
|
-
await state.appendEvent(resumed.id, "run.recovered", {
|
|
196
|
-
strategy: "connections-available-resume",
|
|
197
|
-
status: resumed.status
|
|
198
|
-
});
|
|
199
|
-
recovered.push(resumed);
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
catch (error) {
|
|
204
|
-
await state.appendEvent(run.id, "run.recovery_resume_failed", { error: errorMessage(error) });
|
|
205
|
-
host.logger.warn("connection-wait continuation resume failed", { runId: run.id, error: errorMessage(error) });
|
|
206
|
-
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
host.logger.warn("subagent settlement sweep failed for run", { runId: run.id, error: errorMessage(error) });
|
|
207
95
|
}
|
|
208
|
-
const message = `Required connection${missing.length === 1 ? "" : "s"} unavailable: ${missing.join(", ") || "unknown"}. Configure the connection and re-send the request.`;
|
|
209
|
-
const failed = await failRun({ state }, run.id, {
|
|
210
|
-
reason: "connection.unavailable",
|
|
211
|
-
error: message,
|
|
212
|
-
data: { recovered: true, missingConnections: missing }
|
|
213
|
-
});
|
|
214
|
-
await enqueueRunFailureNotice(host, current, message);
|
|
215
|
-
await state.appendEvent(run.id, "run.recovered", { strategy: "connection-unavailable-marked-failed" });
|
|
216
|
-
recovered.push(failed);
|
|
217
96
|
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
}
|
|
223
|
-
async function claimConnectionWaitRecovery(state, candidate, cutoff) {
|
|
224
|
-
const claimed = await state.reserveIdempotencyKey("run:connection-recovery", `${candidate.id}:${candidate.updatedAt}`, candidate.id);
|
|
225
|
-
if (!claimed)
|
|
226
|
-
return undefined;
|
|
227
|
-
const current = await state.getRun(candidate.id);
|
|
228
|
-
if (current?.status === "waiting_for_connection"
|
|
229
|
-
&& current.updatedAt === candidate.updatedAt
|
|
230
|
-
&& Date.parse(current.updatedAt) <= cutoff) {
|
|
231
|
-
return current;
|
|
232
|
-
}
|
|
233
|
-
await state.releaseIdempotencyKey?.("run:connection-recovery", `${candidate.id}:${candidate.updatedAt}`);
|
|
234
|
-
return undefined;
|
|
235
|
-
}
|
|
236
|
-
function lastMissingConnections(events) {
|
|
237
|
-
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
238
|
-
const event = events[index];
|
|
239
|
-
if (event?.type !== "connection.required")
|
|
240
|
-
continue;
|
|
241
|
-
const list = event.data.missingConnections;
|
|
242
|
-
return Array.isArray(list) ? list.filter((value) => typeof value === "string") : [];
|
|
97
|
+
if (candidates.length < 200)
|
|
98
|
+
break;
|
|
99
|
+
const last = candidates.at(-1);
|
|
100
|
+
after = { updatedAt: last.updatedAt, id: last.id };
|
|
243
101
|
}
|
|
244
|
-
return [];
|
|
245
102
|
}
|
|
246
103
|
/**
|
|
247
104
|
* Multi-replica claim: exactly one sweeper acts on this (run, updatedAt)
|
|
248
105
|
* generation. Returns the recovered run plus any parent it woke.
|
|
249
106
|
*/
|
|
250
107
|
async function claimAndRecover(host, run, sweep) {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
if (!isSubagentRun(outcome) || !isTerminalRunStatus(outcome.status))
|
|
272
|
-
return [outcome];
|
|
273
|
-
const settled = await host.settleSubagentRun(await host.replay(outcome.id));
|
|
274
|
-
if (isBackgroundSubagentRun(settled.run))
|
|
275
|
-
return [settled.run];
|
|
276
|
-
// A foreground child never outlives its parent's process, so a recovered
|
|
277
|
-
// child means the parent is waiting on it: wake the parent with the real
|
|
278
|
-
// delegate result instead of an "interrupted" one.
|
|
279
|
-
return [settled.run, ...await wakeForegroundParent(host, settled.run, sweep)];
|
|
108
|
+
return await withRunExecutionLease(host.state, run.id, "execution", ["created", "running"], async () => {
|
|
109
|
+
const { state } = host;
|
|
110
|
+
const current = await state.getRun(run.id);
|
|
111
|
+
// Claiming renews updatedAt; validate the snapshot's status and progress
|
|
112
|
+
// inside the lease rather than permanently reserving a timestamp key.
|
|
113
|
+
if (!current || !ORPHAN_CANDIDATE_STATUSES.has(current.status))
|
|
114
|
+
return [];
|
|
115
|
+
const outcome = await recoverOrphanedRun(host, current, sweep);
|
|
116
|
+
if (!outcome)
|
|
117
|
+
return [];
|
|
118
|
+
if (!isSubagentRun(outcome) || !isTerminalRunStatus(outcome.status))
|
|
119
|
+
return [outcome];
|
|
120
|
+
const settled = await host.settleSubagentRun(await host.replay(outcome.id));
|
|
121
|
+
if (isBackgroundSubagentRun(settled.run))
|
|
122
|
+
return [settled.run];
|
|
123
|
+
// A foreground child never outlives its parent's process, so a recovered
|
|
124
|
+
// child means the parent is waiting on it: wake the parent with the real
|
|
125
|
+
// delegate result instead of an "interrupted" one.
|
|
126
|
+
return [settled.run, ...await wakeForegroundParent(host, settled.run, sweep)];
|
|
127
|
+
}, current => Date.parse(current.updatedAt) <= sweep.cutoff || Boolean(current.lifecycle?.lease && Date.parse(current.lifecycle.lease.expiresAt) <= Date.now())) ?? [];
|
|
280
128
|
}
|
|
281
129
|
function isSubagentRun(run) {
|
|
282
130
|
return typeof run.input.parentRunId === "string" && typeof run.input.subagent === "string";
|
|
@@ -306,14 +154,15 @@ async function recoveryDeferredOnChild(state, parentRunId, childRunId) {
|
|
|
306
154
|
async function recoverOrphanedRun(host, run, sweep) {
|
|
307
155
|
const { state } = host;
|
|
308
156
|
const { maxResumeAttempts } = sweep;
|
|
309
|
-
const
|
|
157
|
+
const history = await state.listEvents(run.id);
|
|
158
|
+
const generationStart = [...history].reverse().find(event => event.type === "run.generation_started" && event.data.generation === run.lifecycle?.generation)?.sequence ?? 0;
|
|
159
|
+
const events = history.filter(event => event.sequence >= generationStart);
|
|
310
160
|
const eventTypes = new Set(events.map((event) => event.type));
|
|
311
161
|
await state.appendEvent(run.id, "run.recovery_started", {
|
|
312
162
|
previousStatus: run.status,
|
|
313
163
|
staleSince: run.updatedAt
|
|
314
164
|
});
|
|
315
165
|
if (isQueuedBackgroundSubagent(run)) {
|
|
316
|
-
await state.releaseIdempotencyKey?.("background-subagent:execute", `${run.id}:${run.updatedAt}`);
|
|
317
166
|
const requeued = await state.updateRun(run.id, {});
|
|
318
167
|
await state.appendEvent(run.id, "run.recovered", {
|
|
319
168
|
strategy: "background-subagent-requeued"
|
|
@@ -321,6 +170,15 @@ async function recoverOrphanedRun(host, run, sweep) {
|
|
|
321
170
|
return requeued;
|
|
322
171
|
}
|
|
323
172
|
await recoverOutstandingUsageObservations(host, run, events);
|
|
173
|
+
const pendingCommand = (await state.listRunCommands(run.id)).find(command => command.id === run.lifecycle?.resumeCommandId && command.status === "accepted");
|
|
174
|
+
if (pendingCommand && ["answer", "approve"].includes(pendingCommand.operation) && (run.lifecycle?.recoveryAttempts ?? 0) < maxResumeAttempts) {
|
|
175
|
+
const action = (await state.listToolCalls(run.id)).find(call => call.id === pendingCommand.pendingActionId);
|
|
176
|
+
if (action?.status === "waiting_for_approval") {
|
|
177
|
+
return (await state.transactRun(run.id, ({ run: current }) => ({ patch: { status: "created", lifecycle: {
|
|
178
|
+
...current.lifecycle, recoveryAttempts: current.lifecycle.recoveryAttempts + 1
|
|
179
|
+
} }, events: [{ type: "run.recovered", data: { strategy: "pending-command-requeued", commandId: pendingCommand.id } }] }))).run;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
324
182
|
if (eventTypes.has("delivery.sent")) {
|
|
325
183
|
const completed = await state.updateRun(run.id, { status: "completed" });
|
|
326
184
|
await state.appendEvent(run.id, "run.recovered", {
|
|
@@ -356,9 +214,12 @@ async function recoverOrphanedRun(host, run, sweep) {
|
|
|
356
214
|
return undefined;
|
|
357
215
|
}
|
|
358
216
|
const { toolOutcomes } = settlement;
|
|
359
|
-
const resumeAttempts =
|
|
217
|
+
const resumeAttempts = run.lifecycle?.recoveryAttempts ?? 0;
|
|
360
218
|
if (resumeAttempts < maxResumeAttempts) {
|
|
361
219
|
try {
|
|
220
|
+
await state.transactRun(run.id, ({ run: latest }) => latest.lifecycle ? { patch: {
|
|
221
|
+
lifecycle: { ...latest.lifecycle, recoveryAttempts: latest.lifecycle.recoveryAttempts + 1 }
|
|
222
|
+
} } : undefined);
|
|
362
223
|
const resumed = await host.resumeRunFromContinuation(run, toolOutcomes.length > 0 ? { toolOutcomes } : undefined);
|
|
363
224
|
if (resumed) {
|
|
364
225
|
await state.appendEvent(resumed.id, "run.recovered", {
|
|
@@ -379,7 +240,7 @@ async function recoverOrphanedRun(host, run, sweep) {
|
|
|
379
240
|
// is either already settled or already owned by the durable queue —
|
|
380
241
|
// creating a second one here would double-send. The real response text
|
|
381
242
|
// rides on that already-created obligation, so this path is always safe.
|
|
382
|
-
const existing = await state.listDeliveries(run.id);
|
|
243
|
+
const existing = (await state.listDeliveries(run.id)).filter(delivery => delivery.idempotencyKey === finalDeliveryIdempotencyKey(run, typeof run.input.channel === "string" ? run.input.channel : "local"));
|
|
383
244
|
if (existing.length > 0) {
|
|
384
245
|
const completed = await state.updateRun(run.id, { status: "completed" });
|
|
385
246
|
await state.appendEvent(run.id, "run.recovered", {
|