@caupulican/pi-adaptative 0.86.3 → 0.86.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/dist/core/agent-paths.d.ts +13 -0
- package/dist/core/agent-paths.d.ts.map +1 -1
- package/dist/core/agent-paths.js +23 -0
- package/dist/core/agent-paths.js.map +1 -1
- package/dist/core/autonomy/bounded-completion.d.ts +7 -0
- package/dist/core/autonomy/bounded-completion.d.ts.map +1 -1
- package/dist/core/autonomy/bounded-completion.js +12 -4
- package/dist/core/autonomy/bounded-completion.js.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.d.ts +2 -0
- package/dist/core/autonomy/lane-tool-surface.d.ts.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.js +6 -3
- package/dist/core/autonomy/lane-tool-surface.js.map +1 -1
- package/dist/core/autonomy/subagent-prompt.js +1 -1
- package/dist/core/autonomy/subagent-prompt.js.map +1 -1
- package/dist/core/background-lane-controller.d.ts +25 -2
- package/dist/core/background-lane-controller.d.ts.map +1 -1
- package/dist/core/background-lane-controller.js +67 -0
- package/dist/core/background-lane-controller.js.map +1 -1
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts +4 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts.map +1 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js +20 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts +25 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.js +299 -0
- package/dist/core/delegation/sanitized-context-fork.js.map +1 -0
- package/dist/core/delegation/session-root-mailbox.d.ts +75 -0
- package/dist/core/delegation/session-root-mailbox.d.ts.map +1 -0
- package/dist/core/delegation/session-root-mailbox.js +627 -0
- package/dist/core/delegation/session-root-mailbox.js.map +1 -0
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts +102 -9
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control-coordinator.js +1307 -125
- package/dist/core/delegation/worker-agent-control-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-agent-control.d.ts +183 -8
- package/dist/core/delegation/worker-agent-control.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control.js +778 -48
- package/dist/core/delegation/worker-agent-control.js.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.d.ts +9 -0
- package/dist/core/delegation/worker-attempt-executor.d.ts.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.js +479 -214
- package/dist/core/delegation/worker-attempt-executor.js.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.d.ts +9 -0
- package/dist/core/delegation/worker-authority-resolver.d.ts.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.js +21 -2
- package/dist/core/delegation/worker-authority-resolver.js.map +1 -1
- package/dist/core/delegation/worker-context-fork-store.d.ts +71 -0
- package/dist/core/delegation/worker-context-fork-store.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-fork-store.js +437 -0
- package/dist/core/delegation/worker-context-fork-store.js.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts +18 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js +28 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts +31 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.js +203 -0
- package/dist/core/delegation/worker-conversation-revision.js.map +1 -0
- package/dist/core/delegation/worker-conversation-store.d.ts +130 -18
- package/dist/core/delegation/worker-conversation-store.d.ts.map +1 -1
- package/dist/core/delegation/worker-conversation-store.js +1263 -79
- package/dist/core/delegation/worker-conversation-store.js.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.d.ts +30 -2
- package/dist/core/delegation/worker-delegation-controller.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.js +675 -111
- package/dist/core/delegation/worker-delegation-controller.js.map +1 -1
- package/dist/core/delegation/worker-delegation-request.d.ts +2 -0
- package/dist/core/delegation/worker-delegation-request.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-request.js.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts +20 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.js +210 -37
- package/dist/core/delegation/worker-dispatch-scheduler.js.map +1 -1
- package/dist/core/delegation/worker-execution-policy.d.ts.map +1 -1
- package/dist/core/delegation/worker-execution-policy.js +14 -2
- package/dist/core/delegation/worker-execution-policy.js.map +1 -1
- package/dist/core/delegation/worker-fleet-limits.d.ts +38 -0
- package/dist/core/delegation/worker-fleet-limits.d.ts.map +1 -0
- package/dist/core/delegation/worker-fleet-limits.js +113 -0
- package/dist/core/delegation/worker-fleet-limits.js.map +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts.map +1 -1
- package/dist/core/delegation/worker-lane-projection.js +6 -2
- package/dist/core/delegation/worker-lane-projection.js.map +1 -1
- package/dist/core/delegation/worker-lifecycle.d.ts +20 -3
- package/dist/core/delegation/worker-lifecycle.d.ts.map +1 -1
- package/dist/core/delegation/worker-lifecycle.js +42 -18
- package/dist/core/delegation/worker-lifecycle.js.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.d.ts +4 -0
- package/dist/core/delegation/worker-notification-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.js +45 -4
- package/dist/core/delegation/worker-notification-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts +50 -0
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts.map +1 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js +185 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js.map +1 -0
- package/dist/core/delegation/worker-recovery-coordinator.d.ts +45 -4
- package/dist/core/delegation/worker-recovery-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-recovery-coordinator.js +223 -17
- package/dist/core/delegation/worker-recovery-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-retry-policy.d.ts +36 -0
- package/dist/core/delegation/worker-retry-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-retry-policy.js +41 -0
- package/dist/core/delegation/worker-retry-policy.js.map +1 -0
- package/dist/core/delegation/worker-runner.d.ts +3 -1
- package/dist/core/delegation/worker-runner.d.ts.map +1 -1
- package/dist/core/delegation/worker-runner.js +4 -2
- package/dist/core/delegation/worker-runner.js.map +1 -1
- package/dist/core/delegation/worker-task-view.d.ts +43 -0
- package/dist/core/delegation/worker-task-view.d.ts.map +1 -0
- package/dist/core/delegation/worker-task-view.js +194 -0
- package/dist/core/delegation/worker-task-view.js.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts +55 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js +179 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js.map +1 -0
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts +2 -2
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-tree-budget-coordinator.js +12 -46
- package/dist/core/delegation/worker-tree-budget-coordinator.js.map +1 -1
- package/dist/core/goals/goal-continuation-controller.d.ts.map +1 -1
- package/dist/core/goals/goal-continuation-controller.js +15 -0
- package/dist/core/goals/goal-continuation-controller.js.map +1 -1
- package/dist/core/goals/goal-state.d.ts +6 -0
- package/dist/core/goals/goal-state.d.ts.map +1 -1
- package/dist/core/goals/goal-state.js +1 -0
- package/dist/core/goals/goal-state.js.map +1 -1
- package/dist/core/goals/goal-tool-core.d.ts +1 -0
- package/dist/core/goals/goal-tool-core.d.ts.map +1 -1
- package/dist/core/goals/goal-tool-core.js +25 -3
- package/dist/core/goals/goal-tool-core.js.map +1 -1
- package/dist/core/keybindings.d.ts +10 -0
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +2 -0
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/orchestration/attempt-ordering.d.ts +9 -0
- package/dist/core/orchestration/attempt-ordering.d.ts.map +1 -0
- package/dist/core/orchestration/attempt-ordering.js +29 -0
- package/dist/core/orchestration/attempt-ordering.js.map +1 -0
- package/dist/core/orchestration/capability-gateway.d.ts +19 -0
- package/dist/core/orchestration/capability-gateway.d.ts.map +1 -1
- package/dist/core/orchestration/capability-gateway.js +19 -2
- package/dist/core/orchestration/capability-gateway.js.map +1 -1
- package/dist/core/orchestration/contracts.d.ts +43 -3
- package/dist/core/orchestration/contracts.d.ts.map +1 -1
- package/dist/core/orchestration/contracts.js +24 -0
- package/dist/core/orchestration/contracts.js.map +1 -1
- package/dist/core/orchestration/delegation-ledger.d.ts +8 -3
- package/dist/core/orchestration/delegation-ledger.d.ts.map +1 -1
- package/dist/core/orchestration/delegation-ledger.js +148 -55
- package/dist/core/orchestration/delegation-ledger.js.map +1 -1
- package/dist/core/orchestration/event-store.d.ts +6 -3
- package/dist/core/orchestration/event-store.d.ts.map +1 -1
- package/dist/core/orchestration/event-store.js +228 -30
- package/dist/core/orchestration/event-store.js.map +1 -1
- package/dist/core/orchestration/task-runtime-codecs.d.ts +31 -0
- package/dist/core/orchestration/task-runtime-codecs.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-codecs.js +1471 -0
- package/dist/core/orchestration/task-runtime-codecs.js.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts +30 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.js +340 -0
- package/dist/core/orchestration/task-runtime-projection.js.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts +38 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.js +1270 -0
- package/dist/core/orchestration/task-runtime-reducer.js.map +1 -0
- package/dist/core/orchestration/task-runtime-state.d.ts +142 -0
- package/dist/core/orchestration/task-runtime-state.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-state.js +14 -0
- package/dist/core/orchestration/task-runtime-state.js.map +1 -0
- package/dist/core/orchestration/task-runtime.d.ts +44 -98
- package/dist/core/orchestration/task-runtime.d.ts.map +1 -1
- package/dist/core/orchestration/task-runtime.js +357 -1113
- package/dist/core/orchestration/task-runtime.js.map +1 -1
- package/dist/core/orchestration/worker-context-fork-reference.d.ts +20 -0
- package/dist/core/orchestration/worker-context-fork-reference.d.ts.map +1 -0
- package/dist/core/orchestration/worker-context-fork-reference.js +53 -0
- package/dist/core/orchestration/worker-context-fork-reference.js.map +1 -0
- package/dist/core/orchestration/worker-execution-contract.d.ts.map +1 -1
- package/dist/core/orchestration/worker-execution-contract.js +4 -2
- package/dist/core/orchestration/worker-execution-contract.js.map +1 -1
- package/dist/core/research/research-runner.d.ts +2 -0
- package/dist/core/research/research-runner.d.ts.map +1 -1
- package/dist/core/research/research-runner.js +1 -0
- package/dist/core/research/research-runner.js.map +1 -1
- package/dist/core/runtime-builder.d.ts.map +1 -1
- package/dist/core/runtime-builder.js +8 -0
- package/dist/core/runtime-builder.js.map +1 -1
- package/dist/core/secrets/credential-project.d.ts.map +1 -1
- package/dist/core/secrets/credential-project.js +24 -2
- package/dist/core/secrets/credential-project.js.map +1 -1
- package/dist/core/security/secret-text.d.ts +3 -0
- package/dist/core/security/secret-text.d.ts.map +1 -1
- package/dist/core/security/secret-text.js +15 -0
- package/dist/core/security/secret-text.js.map +1 -1
- package/dist/core/settings-manager.d.ts +3 -0
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +160 -8
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/tools/delegate.d.ts +33 -15
- package/dist/core/tools/delegate.d.ts.map +1 -1
- package/dist/core/tools/delegate.js +763 -115
- package/dist/core/tools/delegate.js.map +1 -1
- package/dist/core/tools/goal.d.ts +1 -0
- package/dist/core/tools/goal.d.ts.map +1 -1
- package/dist/core/tools/goal.js +9 -1
- package/dist/core/tools/goal.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +1 -0
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/activity-lane.d.ts +5 -0
- package/dist/modes/interactive/components/activity-lane.d.ts.map +1 -1
- package/dist/modes/interactive/components/activity-lane.js +133 -49
- package/dist/modes/interactive/components/activity-lane.js.map +1 -1
- package/dist/modes/interactive/components/agents-overlay.d.ts +40 -0
- package/dist/modes/interactive/components/agents-overlay.d.ts.map +1 -0
- package/dist/modes/interactive/components/agents-overlay.js +153 -0
- package/dist/modes/interactive/components/agents-overlay.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +18 -13
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.js +6 -3
- package/dist/modes/interactive/interactive-event-controller.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +5 -0
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +51 -0
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/key-handlers.d.ts +1 -0
- package/dist/modes/interactive/key-handlers.d.ts.map +1 -1
- package/dist/modes/interactive/key-handlers.js +1 -0
- package/dist/modes/interactive/key-handlers.js.map +1 -1
- package/docs/harness-architecture.md +61 -17
- package/docs/settings.md +17 -9
- package/docs/worker-profiles.md +6 -2
- package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package-lock.json +2 -2
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package-lock.json +2 -2
- package/examples/extensions/with-deps/package.json +1 -1
- package/npm-shrinkwrap.json +12 -12
- package/package.json +4 -4
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { addUsage, createEmptyUsage } from "@caupulican/pi-agent-core/usage";
|
|
2
|
+
export class WorkerCompletionProtocolError extends Error {
|
|
3
|
+
constructor(message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = "WorkerCompletionProtocolError";
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
function usageDelta(usage) {
|
|
9
|
+
return {
|
|
10
|
+
inputTokens: usage.input,
|
|
11
|
+
outputTokens: usage.output,
|
|
12
|
+
cacheReadTokens: usage.cacheRead,
|
|
13
|
+
cacheWriteTokens: usage.cacheWrite,
|
|
14
|
+
totalTokens: usage.totalTokens,
|
|
15
|
+
costUsd: usage.cost.total,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function positiveUsageDelta(reported, accounted) {
|
|
19
|
+
return {
|
|
20
|
+
inputTokens: Math.max(0, reported.input - accounted.input),
|
|
21
|
+
outputTokens: Math.max(0, reported.output - accounted.output),
|
|
22
|
+
cacheReadTokens: Math.max(0, reported.cacheRead - accounted.cacheRead),
|
|
23
|
+
cacheWriteTokens: Math.max(0, reported.cacheWrite - accounted.cacheWrite),
|
|
24
|
+
totalTokens: Math.max(0, reported.totalTokens - accounted.totalTokens),
|
|
25
|
+
costUsd: Math.max(0, reported.cost.total - accounted.cost.total),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function hasUsage(delta) {
|
|
29
|
+
return (delta.inputTokens !== 0 ||
|
|
30
|
+
delta.outputTokens !== 0 ||
|
|
31
|
+
delta.cacheReadTokens !== 0 ||
|
|
32
|
+
delta.cacheWriteTokens !== 0 ||
|
|
33
|
+
delta.totalTokens !== 0 ||
|
|
34
|
+
delta.costUsd !== 0);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Owns one provider request's reservation epochs and usage evidence. The caller remains responsible
|
|
38
|
+
* for persisting the assistant and checkpoint between accounting and consuming an epoch.
|
|
39
|
+
*/
|
|
40
|
+
export class WorkerProviderTurnProtocol {
|
|
41
|
+
constructor(options) {
|
|
42
|
+
this.callbackAccountedUsage = createEmptyUsage();
|
|
43
|
+
this.heldConsumed = false;
|
|
44
|
+
this.assistantUsageAccounted = false;
|
|
45
|
+
this.inFlight = false;
|
|
46
|
+
this.generation = 0;
|
|
47
|
+
this.closed = false;
|
|
48
|
+
this.successfulPreflights = 0;
|
|
49
|
+
this.consumedPreflights = 0;
|
|
50
|
+
this.acquireReservation = options.acquireReservation;
|
|
51
|
+
this.recordUsage = options.recordUsage;
|
|
52
|
+
this.signal = options.signal;
|
|
53
|
+
this.onFailure = options.onFailure;
|
|
54
|
+
}
|
|
55
|
+
async requestPreflight() {
|
|
56
|
+
if (this.closed || this.inFlight || (this.held && !this.heldConsumed)) {
|
|
57
|
+
const error = new WorkerCompletionProtocolError("Worker completion attempted an overlapping or out-of-order provider preflight.");
|
|
58
|
+
this.generation += 1;
|
|
59
|
+
this.onFailure(error);
|
|
60
|
+
this.releaseHeldReservation();
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
// A terminal assistant retains capacity until either its result is reconciled or the adapter
|
|
64
|
+
// proves that another provider turn is starting.
|
|
65
|
+
this.releaseHeldReservation();
|
|
66
|
+
this.inFlight = true;
|
|
67
|
+
const generation = ++this.generation;
|
|
68
|
+
let acquired;
|
|
69
|
+
try {
|
|
70
|
+
acquired = await this.acquireReservation();
|
|
71
|
+
this.signal.throwIfAborted();
|
|
72
|
+
if (this.closed || generation !== this.generation) {
|
|
73
|
+
throw new WorkerCompletionProtocolError("Worker completion provider preflight resolved after its ownership fence closed.");
|
|
74
|
+
}
|
|
75
|
+
this.held = acquired;
|
|
76
|
+
this.heldConsumed = false;
|
|
77
|
+
this.assistantUsageAccounted = false;
|
|
78
|
+
acquired = undefined;
|
|
79
|
+
this.successfulPreflights += 1;
|
|
80
|
+
return { maxTokens: this.held.maxTokens };
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
this.onFailure(error);
|
|
84
|
+
acquired?.release();
|
|
85
|
+
this.releaseHeldReservation();
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
this.inFlight = false;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
accountAssistantUsage(usage) {
|
|
93
|
+
this.assertOutstandingReservation();
|
|
94
|
+
if (this.assistantUsageAccounted) {
|
|
95
|
+
this.fail(new WorkerCompletionProtocolError("Worker completion attempted to account more than one assistant under one provider preflight."));
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
this.recordUsage?.(usageDelta(usage));
|
|
99
|
+
addUsage(this.callbackAccountedUsage, usage);
|
|
100
|
+
this.assistantUsageAccounted = true;
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
this.onFailure(error);
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** Account usage before releasing a tool-request turn for nested or subsequent provider work. */
|
|
108
|
+
consumeToolAssistantAndRelease() {
|
|
109
|
+
this.consumeAccountedAssistant();
|
|
110
|
+
this.releaseHeldReservation();
|
|
111
|
+
}
|
|
112
|
+
/** Account usage while retaining a terminal turn's remaining capacity through result verification. */
|
|
113
|
+
consumeTerminalAssistantAndHold() {
|
|
114
|
+
this.consumeAccountedAssistant();
|
|
115
|
+
}
|
|
116
|
+
assertEverySuccessfulPreflightConsumed() {
|
|
117
|
+
if (!this.inFlight &&
|
|
118
|
+
(!this.held || this.heldConsumed) &&
|
|
119
|
+
this.consumedPreflights === this.successfulPreflights) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
this.fail(new WorkerCompletionProtocolError("Worker completion returned with a provider preflight authority epoch that no assistant consumed."));
|
|
123
|
+
}
|
|
124
|
+
assertProviderOutputPreflight(kind = "completion") {
|
|
125
|
+
if (this.successfulPreflights > 0)
|
|
126
|
+
return;
|
|
127
|
+
this.fail(new WorkerCompletionProtocolError(kind === "compaction"
|
|
128
|
+
? "Worker compaction returned provider output without a successful authority preflight."
|
|
129
|
+
: "Worker completion returned provider output without a successful authority preflight."));
|
|
130
|
+
}
|
|
131
|
+
hasSuccessfulPreflight() {
|
|
132
|
+
return this.successfulPreflights > 0;
|
|
133
|
+
}
|
|
134
|
+
hasOutstandingAssistantReservation() {
|
|
135
|
+
return this.held !== undefined && !this.heldConsumed;
|
|
136
|
+
}
|
|
137
|
+
/** Conservatively charge only provider-result usage not already evidenced by assistant callbacks. */
|
|
138
|
+
accountUnverifiedResultUsageDelta(reported) {
|
|
139
|
+
const delta = positiveUsageDelta(reported, this.callbackAccountedUsage);
|
|
140
|
+
let recorded = false;
|
|
141
|
+
if (this.recordUsage && hasUsage(delta)) {
|
|
142
|
+
try {
|
|
143
|
+
this.recordUsage(delta);
|
|
144
|
+
recorded = true;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
this.onFailure(error);
|
|
148
|
+
throw error;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return recorded;
|
|
152
|
+
}
|
|
153
|
+
close() {
|
|
154
|
+
if (!this.closed) {
|
|
155
|
+
this.closed = true;
|
|
156
|
+
this.generation += 1;
|
|
157
|
+
}
|
|
158
|
+
this.releaseHeldReservation();
|
|
159
|
+
}
|
|
160
|
+
consumeAccountedAssistant() {
|
|
161
|
+
this.assertOutstandingReservation();
|
|
162
|
+
if (!this.assistantUsageAccounted) {
|
|
163
|
+
this.fail(new WorkerCompletionProtocolError("Worker completion attempted to consume an assistant before its provider usage was accounted."));
|
|
164
|
+
}
|
|
165
|
+
this.consumedPreflights += 1;
|
|
166
|
+
this.heldConsumed = true;
|
|
167
|
+
}
|
|
168
|
+
assertOutstandingReservation() {
|
|
169
|
+
if (this.held && !this.heldConsumed)
|
|
170
|
+
return;
|
|
171
|
+
this.fail(new WorkerCompletionProtocolError("Worker completion attempted to persist an assistant without a held provider reservation."));
|
|
172
|
+
}
|
|
173
|
+
releaseHeldReservation() {
|
|
174
|
+
const held = this.held;
|
|
175
|
+
this.held = undefined;
|
|
176
|
+
this.heldConsumed = false;
|
|
177
|
+
this.assistantUsageAccounted = false;
|
|
178
|
+
held?.release();
|
|
179
|
+
}
|
|
180
|
+
fail(error) {
|
|
181
|
+
this.onFailure(error);
|
|
182
|
+
throw error;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=worker-provider-turn-protocol.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-provider-turn-protocol.js","sourceRoot":"","sources":["../../../src/core/delegation/worker-provider-turn-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAI7E,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACvD,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAC7C,CAAC;CACD;AAED,SAAS,UAAU,CAAC,KAAY;IAC/B,OAAO;QACN,WAAW,EAAE,KAAK,CAAC,KAAK;QACxB,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,eAAe,EAAE,KAAK,CAAC,SAAS;QAChC,gBAAgB,EAAE,KAAK,CAAC,UAAU;QAClC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;KACzB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAe,EAAE,SAAgB;IAC5D,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;QAC1D,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAC7D,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;QACtE,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;QACzE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;QACtE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;KAChE,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAkC;IACnD,OAAO,CACN,KAAK,CAAC,WAAW,KAAK,CAAC;QACvB,KAAK,CAAC,YAAY,KAAK,CAAC;QACxB,KAAK,CAAC,eAAe,KAAK,CAAC;QAC3B,KAAK,CAAC,gBAAgB,KAAK,CAAC;QAC5B,KAAK,CAAC,WAAW,KAAK,CAAC;QACvB,KAAK,CAAC,OAAO,KAAK,CAAC,CACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,0BAA0B;IAetC,YAAY,OAKX;QAfgB,2BAAsB,GAAG,gBAAgB,EAAE,CAAC;QAErD,iBAAY,GAAG,KAAK,CAAC;QACrB,4BAAuB,GAAG,KAAK,CAAC;QAChC,aAAQ,GAAG,KAAK,CAAC;QACjB,eAAU,GAAG,CAAC,CAAC;QACf,WAAM,GAAG,KAAK,CAAC;QACf,yBAAoB,GAAG,CAAC,CAAC;QACzB,uBAAkB,GAAG,CAAC,CAAC;QAQ9B,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,gBAAgB;QACrB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,IAAI,6BAA6B,CAC9C,gFAAgF,CAChF,CAAC;YACF,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YACrB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC;QACb,CAAC;QACD,6FAA6F;QAC7F,iDAAiD;QACjD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC;QACrC,IAAI,QAA+C,CAAC;QACpD,IAAI,CAAC;YACJ,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnD,MAAM,IAAI,6BAA6B,CACtC,iFAAiF,CACjF,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;YACrB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;YACrC,QAAQ,GAAG,SAAS,CAAC;YACrB,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;YAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC;QACb,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACvB,CAAC;IACF,CAAC;IAED,qBAAqB,CAAC,KAAY;QACjC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,8FAA8F,CAC9F,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC;YACJ,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAED,iGAAiG;IACjG,8BAA8B;QAC7B,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC/B,CAAC;IAED,sGAAsG;IACtG,+BAA+B;QAC9B,IAAI,CAAC,yBAAyB,EAAE,CAAC;IAClC,CAAC;IAED,sCAAsC;QACrC,IACC,CAAC,IAAI,CAAC,QAAQ;YACd,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC;YACjC,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,oBAAoB,EACpD,CAAC;YACF,OAAO;QACR,CAAC;QACD,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,kGAAkG,CAClG,CACD,CAAC;IACH,CAAC;IAED,6BAA6B,CAAC,IAAI,GAAgC,YAAY;QAC7E,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC;YAAE,OAAO;QAC1C,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,IAAI,KAAK,YAAY;YACpB,CAAC,CAAC,sFAAsF;YACxF,CAAC,CAAC,sFAAsF,CACzF,CACD,CAAC;IACH,CAAC;IAED,sBAAsB;QACrB,OAAO,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,kCAAkC;QACjC,OAAO,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,qGAAqG;IACrG,iCAAiC,CAAC,QAAe;QAChD,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACxE,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC;gBACJ,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACxB,QAAQ,GAAG,IAAI,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACtB,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;QACD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC/B,CAAC;IAEO,yBAAyB;QAChC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,8FAA8F,CAC9F,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,4BAA4B;QACnC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAC5C,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,0FAA0F,CAC1F,CACD,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACrC,IAAI,EAAE,OAAO,EAAE,CAAC;IACjB,CAAC;IAEO,IAAI,CAAC,KAAoC;QAChD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtB,MAAM,KAAK,CAAC;IACb,CAAC;CACD","sourcesContent":["import { addUsage, createEmptyUsage } from \"@caupulican/pi-agent-core/usage\";\nimport type { Usage } from \"@caupulican/pi-ai\";\nimport type { GatewayUsageDelta, ProviderBudgetReservation } from \"../orchestration/capability-gateway.ts\";\n\nexport class WorkerCompletionProtocolError extends Error {\n\tconstructor(message: string) {\n\t\tsuper(message);\n\t\tthis.name = \"WorkerCompletionProtocolError\";\n\t}\n}\n\nfunction usageDelta(usage: Usage): Required<GatewayUsageDelta> {\n\treturn {\n\t\tinputTokens: usage.input,\n\t\toutputTokens: usage.output,\n\t\tcacheReadTokens: usage.cacheRead,\n\t\tcacheWriteTokens: usage.cacheWrite,\n\t\ttotalTokens: usage.totalTokens,\n\t\tcostUsd: usage.cost.total,\n\t};\n}\n\nfunction positiveUsageDelta(reported: Usage, accounted: Usage): Required<GatewayUsageDelta> {\n\treturn {\n\t\tinputTokens: Math.max(0, reported.input - accounted.input),\n\t\toutputTokens: Math.max(0, reported.output - accounted.output),\n\t\tcacheReadTokens: Math.max(0, reported.cacheRead - accounted.cacheRead),\n\t\tcacheWriteTokens: Math.max(0, reported.cacheWrite - accounted.cacheWrite),\n\t\ttotalTokens: Math.max(0, reported.totalTokens - accounted.totalTokens),\n\t\tcostUsd: Math.max(0, reported.cost.total - accounted.cost.total),\n\t};\n}\n\nfunction hasUsage(delta: Required<GatewayUsageDelta>): boolean {\n\treturn (\n\t\tdelta.inputTokens !== 0 ||\n\t\tdelta.outputTokens !== 0 ||\n\t\tdelta.cacheReadTokens !== 0 ||\n\t\tdelta.cacheWriteTokens !== 0 ||\n\t\tdelta.totalTokens !== 0 ||\n\t\tdelta.costUsd !== 0\n\t);\n}\n\n/**\n * Owns one provider request's reservation epochs and usage evidence. The caller remains responsible\n * for persisting the assistant and checkpoint between accounting and consuming an epoch.\n */\nexport class WorkerProviderTurnProtocol {\n\tprivate readonly acquireReservation: () => Promise<ProviderBudgetReservation>;\n\tprivate readonly recordUsage: ((delta: GatewayUsageDelta) => void) | undefined;\n\tprivate readonly signal: AbortSignal;\n\tprivate readonly onFailure: (error: unknown) => void;\n\tprivate readonly callbackAccountedUsage = createEmptyUsage();\n\tprivate held: ProviderBudgetReservation | undefined;\n\tprivate heldConsumed = false;\n\tprivate assistantUsageAccounted = false;\n\tprivate inFlight = false;\n\tprivate generation = 0;\n\tprivate closed = false;\n\tprivate successfulPreflights = 0;\n\tprivate consumedPreflights = 0;\n\n\tconstructor(options: {\n\t\tacquireReservation(): Promise<ProviderBudgetReservation>;\n\t\trecordUsage?(delta: GatewayUsageDelta): void;\n\t\tsignal: AbortSignal;\n\t\tonFailure(error: unknown): void;\n\t}) {\n\t\tthis.acquireReservation = options.acquireReservation;\n\t\tthis.recordUsage = options.recordUsage;\n\t\tthis.signal = options.signal;\n\t\tthis.onFailure = options.onFailure;\n\t}\n\n\tasync requestPreflight(): Promise<{ maxTokens: number }> {\n\t\tif (this.closed || this.inFlight || (this.held && !this.heldConsumed)) {\n\t\t\tconst error = new WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion attempted an overlapping or out-of-order provider preflight.\",\n\t\t\t);\n\t\t\tthis.generation += 1;\n\t\t\tthis.onFailure(error);\n\t\t\tthis.releaseHeldReservation();\n\t\t\tthrow error;\n\t\t}\n\t\t// A terminal assistant retains capacity until either its result is reconciled or the adapter\n\t\t// proves that another provider turn is starting.\n\t\tthis.releaseHeldReservation();\n\t\tthis.inFlight = true;\n\t\tconst generation = ++this.generation;\n\t\tlet acquired: ProviderBudgetReservation | undefined;\n\t\ttry {\n\t\t\tacquired = await this.acquireReservation();\n\t\t\tthis.signal.throwIfAborted();\n\t\t\tif (this.closed || generation !== this.generation) {\n\t\t\t\tthrow new WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion provider preflight resolved after its ownership fence closed.\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tthis.held = acquired;\n\t\t\tthis.heldConsumed = false;\n\t\t\tthis.assistantUsageAccounted = false;\n\t\t\tacquired = undefined;\n\t\t\tthis.successfulPreflights += 1;\n\t\t\treturn { maxTokens: this.held.maxTokens };\n\t\t} catch (error) {\n\t\t\tthis.onFailure(error);\n\t\t\tacquired?.release();\n\t\t\tthis.releaseHeldReservation();\n\t\t\tthrow error;\n\t\t} finally {\n\t\t\tthis.inFlight = false;\n\t\t}\n\t}\n\n\taccountAssistantUsage(usage: Usage): void {\n\t\tthis.assertOutstandingReservation();\n\t\tif (this.assistantUsageAccounted) {\n\t\t\tthis.fail(\n\t\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion attempted to account more than one assistant under one provider preflight.\",\n\t\t\t\t),\n\t\t\t);\n\t\t}\n\t\ttry {\n\t\t\tthis.recordUsage?.(usageDelta(usage));\n\t\t\taddUsage(this.callbackAccountedUsage, usage);\n\t\t\tthis.assistantUsageAccounted = true;\n\t\t} catch (error) {\n\t\t\tthis.onFailure(error);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t/** Account usage before releasing a tool-request turn for nested or subsequent provider work. */\n\tconsumeToolAssistantAndRelease(): void {\n\t\tthis.consumeAccountedAssistant();\n\t\tthis.releaseHeldReservation();\n\t}\n\n\t/** Account usage while retaining a terminal turn's remaining capacity through result verification. */\n\tconsumeTerminalAssistantAndHold(): void {\n\t\tthis.consumeAccountedAssistant();\n\t}\n\n\tassertEverySuccessfulPreflightConsumed(): void {\n\t\tif (\n\t\t\t!this.inFlight &&\n\t\t\t(!this.held || this.heldConsumed) &&\n\t\t\tthis.consumedPreflights === this.successfulPreflights\n\t\t) {\n\t\t\treturn;\n\t\t}\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion returned with a provider preflight authority epoch that no assistant consumed.\",\n\t\t\t),\n\t\t);\n\t}\n\n\tassertProviderOutputPreflight(kind: \"completion\" | \"compaction\" = \"completion\"): void {\n\t\tif (this.successfulPreflights > 0) return;\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\tkind === \"compaction\"\n\t\t\t\t\t? \"Worker compaction returned provider output without a successful authority preflight.\"\n\t\t\t\t\t: \"Worker completion returned provider output without a successful authority preflight.\",\n\t\t\t),\n\t\t);\n\t}\n\n\thasSuccessfulPreflight(): boolean {\n\t\treturn this.successfulPreflights > 0;\n\t}\n\n\thasOutstandingAssistantReservation(): boolean {\n\t\treturn this.held !== undefined && !this.heldConsumed;\n\t}\n\n\t/** Conservatively charge only provider-result usage not already evidenced by assistant callbacks. */\n\taccountUnverifiedResultUsageDelta(reported: Usage): boolean {\n\t\tconst delta = positiveUsageDelta(reported, this.callbackAccountedUsage);\n\t\tlet recorded = false;\n\t\tif (this.recordUsage && hasUsage(delta)) {\n\t\t\ttry {\n\t\t\t\tthis.recordUsage(delta);\n\t\t\t\trecorded = true;\n\t\t\t} catch (error) {\n\t\t\t\tthis.onFailure(error);\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\treturn recorded;\n\t}\n\n\tclose(): void {\n\t\tif (!this.closed) {\n\t\t\tthis.closed = true;\n\t\t\tthis.generation += 1;\n\t\t}\n\t\tthis.releaseHeldReservation();\n\t}\n\n\tprivate consumeAccountedAssistant(): void {\n\t\tthis.assertOutstandingReservation();\n\t\tif (!this.assistantUsageAccounted) {\n\t\t\tthis.fail(\n\t\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion attempted to consume an assistant before its provider usage was accounted.\",\n\t\t\t\t),\n\t\t\t);\n\t\t}\n\t\tthis.consumedPreflights += 1;\n\t\tthis.heldConsumed = true;\n\t}\n\n\tprivate assertOutstandingReservation(): void {\n\t\tif (this.held && !this.heldConsumed) return;\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion attempted to persist an assistant without a held provider reservation.\",\n\t\t\t),\n\t\t);\n\t}\n\n\tprivate releaseHeldReservation(): void {\n\t\tconst held = this.held;\n\t\tthis.held = undefined;\n\t\tthis.heldConsumed = false;\n\t\tthis.assistantUsageAccounted = false;\n\t\theld?.release();\n\t}\n\n\tprivate fail(error: WorkerCompletionProtocolError): never {\n\t\tthis.onFailure(error);\n\t\tthrow error;\n\t}\n}\n"]}
|
|
@@ -13,14 +13,29 @@ export type WorkerRecoveryDispatchResult = {
|
|
|
13
13
|
started: false;
|
|
14
14
|
skipReason: string;
|
|
15
15
|
};
|
|
16
|
+
export type WorkerRetryScheduleResult = {
|
|
17
|
+
scheduled: true;
|
|
18
|
+
record: LaneRecord;
|
|
19
|
+
retriesUsed: number;
|
|
20
|
+
notBefore: string;
|
|
21
|
+
} | {
|
|
22
|
+
scheduled: false;
|
|
23
|
+
reason: string;
|
|
24
|
+
};
|
|
16
25
|
export interface WorkerRecoveryCoordinatorOptions {
|
|
17
26
|
lifecycle: WorkerLifecycle;
|
|
18
|
-
scheduler: Pick<WorkerDispatchScheduler, "enqueue"
|
|
27
|
+
scheduler: Pick<WorkerDispatchScheduler, "enqueue"> & Partial<Pick<WorkerDispatchScheduler, "onQueueCapacityAvailable">>;
|
|
19
28
|
recoverWriteReservations(): void;
|
|
20
29
|
publishTerminalRecord(record: LaneRecord): void;
|
|
30
|
+
publishTerminalRecords?(records: readonly LaneRecord[]): void;
|
|
21
31
|
dispatchVerification(recovery: Extract<PendingVerificationRecovery, {
|
|
22
32
|
action: "dispatch";
|
|
23
33
|
}>): WorkerRecoveryDispatchResult;
|
|
34
|
+
recoverTaskBearingMailboxTurns(): void;
|
|
35
|
+
recoverSessionRootReplies(): void;
|
|
36
|
+
/** Wake the scheduler and UI after a retained retry deadline becomes eligible. */
|
|
37
|
+
retryReady?(): void;
|
|
38
|
+
now?(): number;
|
|
24
39
|
warn(message: string): void;
|
|
25
40
|
}
|
|
26
41
|
/**
|
|
@@ -33,21 +48,47 @@ export declare class WorkerRecoveryCoordinator {
|
|
|
33
48
|
private queueRecovered;
|
|
34
49
|
private recoveringDurableState;
|
|
35
50
|
private readonly verificationRecoveryFailures;
|
|
51
|
+
private readonly retryTimers;
|
|
52
|
+
private readonly unsubscribeQueueCapacity;
|
|
36
53
|
constructor(options: WorkerRecoveryCoordinatorOptions);
|
|
37
|
-
/** Idempotently rebuild in-process
|
|
54
|
+
/** Idempotently rebuild in-process queues, terminal outboxes, and task-bearing mailboxes. */
|
|
38
55
|
recover(): void;
|
|
56
|
+
/** Evaluate, persist, and arm one retry through the same durable path used by restart recovery. */
|
|
57
|
+
scheduleAttemptRetry(args: {
|
|
58
|
+
laneId: string;
|
|
59
|
+
agentId: string;
|
|
60
|
+
ownerId: string;
|
|
61
|
+
request: WorkerDelegationRequest;
|
|
62
|
+
outcome: {
|
|
63
|
+
laneStatus: string;
|
|
64
|
+
reasonCode: string;
|
|
65
|
+
reasonDetail?: string;
|
|
66
|
+
};
|
|
67
|
+
provider: string;
|
|
68
|
+
maxAttempts?: number;
|
|
69
|
+
}): WorkerRetryScheduleResult;
|
|
70
|
+
/** Cancel only process-local alarms; the retained deadline remains authoritative on disk. */
|
|
71
|
+
dispose(): void;
|
|
72
|
+
clearScheduledRetry(laneId: string): void;
|
|
73
|
+
/** Keep an externally requested wake queued while the durable retry deadline is still retained. */
|
|
74
|
+
deferRetryIfNeeded(record: LaneRecord, request: WorkerDelegationRequest): boolean;
|
|
75
|
+
private enqueueOrDeferRetry;
|
|
76
|
+
private onRetryTimer;
|
|
77
|
+
private tryEnqueueDueRetry;
|
|
78
|
+
private flushDueRetries;
|
|
79
|
+
private now;
|
|
39
80
|
/** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */
|
|
40
81
|
recoveredRequest(attempt: AttemptRuntimeState, verificationOfTaskId?: string): WorkerDelegationRequest;
|
|
41
82
|
/** Append deterministic unknown outcomes only for tool calls not durably followed by a result. */
|
|
42
83
|
repairInterruptedToolResults(conversation: WorkerConversation): void;
|
|
43
84
|
/** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */
|
|
44
|
-
recoveredTerminalCompletion(conversation: WorkerConversation): {
|
|
85
|
+
recoveredTerminalCompletion(conversation: WorkerConversation, attemptId: string): {
|
|
45
86
|
text: string;
|
|
46
87
|
usage: Usage;
|
|
47
88
|
stopReason: string;
|
|
48
89
|
} | undefined;
|
|
49
90
|
/** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */
|
|
50
|
-
initialUsage(conversation: WorkerConversation, checkpointUsage: AttemptUsageSnapshot | undefined): AttemptUsageSnapshot;
|
|
91
|
+
initialUsage(conversation: WorkerConversation, checkpointUsage: AttemptUsageSnapshot | undefined, attemptId: string): AttemptUsageSnapshot;
|
|
51
92
|
/** Fail closed when the current owner profile no longer permits a persisted grant. */
|
|
52
93
|
durableGrantIsStillPermitted(grant: ExecutionGrant, plan: WorkerExecutionPlan, selectedResources: readonly ResourcePointer[]): boolean;
|
|
53
94
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-recovery-coordinator.d.ts","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAc,MAAM,+BAA+B,CAAC;AACvH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAAE,2BAA2B,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-recovery-coordinator.d.ts","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAc,MAAM,+BAA+B,CAAC;AACvH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAAE,2BAA2B,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IAAE,OAAO,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtG,MAAM,MAAM,yBAAyB,GAClC;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,SAAS,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAUxC,MAAM,WAAW,gCAAgC;IAChD,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,EAAE,IAAI,CAAC,uBAAuB,EAAE,SAAS,CAAC,GAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;IACpE,wBAAwB,IAAI,IAAI,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAChD,sBAAsB,CAAC,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI,CAAC;IAC9D,oBAAoB,CACnB,QAAQ,EAAE,OAAO,CAAC,2BAA2B,EAAE;QAAE,MAAM,EAAE,UAAU,CAAA;KAAE,CAAC,GACpE,4BAA4B,CAAC;IAChC,8BAA8B,IAAI,IAAI,CAAC;IACvC,yBAAyB,IAAI,IAAI,CAAC;IAClC,kFAAkF;IAClF,UAAU,CAAC,IAAI,IAAI,CAAC;IACpB,GAAG,CAAC,IAAI,MAAM,CAAC;IACf,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAcD;;;;GAIG;AACH,qBAAa,yBAAyB;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAA6B;IAC1E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2C;IACvE,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAA2B;IAEpE,YAAY,OAAO,EAAE,gCAAgC,EAQpD;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAyGd;IAED,mGAAmG;IACnG,oBAAoB,CAAC,IAAI,EAAE;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,uBAAuB,CAAC;QACjC,OAAO,EAAE;YAAE,UAAU,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,YAAY,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3E,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,yBAAyB,CAoC5B;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAMd;IAED,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAMxC;IAED,mGAAmG;IACnG,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAYhF;IAED,OAAO,CAAC,mBAAmB;IAqC3B,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,kBAAkB;IAqC1B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,GAAG;IAIX,wGAAwG;IACxG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAmBrG;IAED,kGAAkG;IAClG,4BAA4B,CAAC,YAAY,EAAE,kBAAkB,GAAG,IAAI,CA0BnE;IAED,uGAAuG;IACvG,2BAA2B,CAC1B,YAAY,EAAE,kBAAkB,EAChC,SAAS,EAAE,MAAM,GACf;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAahE;IAED,+GAA+G;IAC/G,YAAY,CACX,YAAY,EAAE,kBAAkB,EAChC,eAAe,EAAE,oBAAoB,GAAG,SAAS,EACjD,SAAS,EAAE,MAAM,GACf,oBAAoB,CAGtB;IAED,sFAAsF;IACtF,4BAA4B,CAC3B,KAAK,EAAE,cAAc,EACrB,IAAI,EAAE,mBAAmB,EACzB,iBAAiB,EAAE,SAAS,eAAe,EAAE,GAC3C,OAAO,CAqCT;CACD"}
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
import { isPathWithinScope } from "../autonomy/path-scope.js";
|
|
2
2
|
import { reconcileAttemptUsage } from "../orchestration/attempt-usage.js";
|
|
3
|
+
import { evaluateWorkerRetry } from "./worker-retry-policy.js";
|
|
4
|
+
function restartResumableSuspension(attempt, newlySuspendedAttemptIds) {
|
|
5
|
+
return (newlySuspendedAttemptIds.has(attempt.attemptId) ||
|
|
6
|
+
attempt.reasonCode === "agent_process_recovered_after_owner_exit" ||
|
|
7
|
+
attempt.reasonCode === "agent_process_interrupted" ||
|
|
8
|
+
attempt.reasonCode?.startsWith("retry_scheduled:") === true);
|
|
9
|
+
}
|
|
3
10
|
/**
|
|
4
11
|
* Owns restart-only worker state repair. It has no provider or tool execution authority: it
|
|
5
12
|
* reconstructs bounded durable requests, requeues them through the owner scheduler, and supplies
|
|
@@ -10,17 +17,24 @@ export class WorkerRecoveryCoordinator {
|
|
|
10
17
|
this.queueRecovered = false;
|
|
11
18
|
this.recoveringDurableState = false;
|
|
12
19
|
this.verificationRecoveryFailures = new Map();
|
|
20
|
+
this.retryTimers = new Map();
|
|
13
21
|
this.options = options;
|
|
22
|
+
this.unsubscribeQueueCapacity = options.scheduler.onQueueCapacityAvailable?.(() => {
|
|
23
|
+
this.flushDueRetries();
|
|
24
|
+
// Mandatory verifier dispatches remain derivable from durable subject state even after the
|
|
25
|
+
// ordinary queue was fully recovered. Every released slot must replay that retained demand.
|
|
26
|
+
this.recover();
|
|
27
|
+
});
|
|
14
28
|
}
|
|
15
|
-
/** Idempotently rebuild in-process
|
|
29
|
+
/** Idempotently rebuild in-process queues, terminal outboxes, and task-bearing mailboxes. */
|
|
16
30
|
recover() {
|
|
17
31
|
if (this.recoveringDurableState)
|
|
18
32
|
return;
|
|
19
33
|
this.recoveringDurableState = true;
|
|
20
|
-
const { lifecycle
|
|
34
|
+
const { lifecycle } = this.options;
|
|
21
35
|
try {
|
|
22
36
|
if (!this.queueRecovered) {
|
|
23
|
-
const
|
|
37
|
+
const newlySuspendedAttemptIds = new Set(lifecycle.suspendBoundInProcessAttemptsForRestart());
|
|
24
38
|
try {
|
|
25
39
|
this.options.recoverWriteReservations();
|
|
26
40
|
}
|
|
@@ -29,10 +43,13 @@ export class WorkerRecoveryCoordinator {
|
|
|
29
43
|
}
|
|
30
44
|
const queued = lifecycle.recoverQueued();
|
|
31
45
|
const snapshot = lifecycle.getTaskRuntimeSnapshot();
|
|
32
|
-
for (const
|
|
33
|
-
|
|
34
|
-
|
|
46
|
+
for (const attempt of Object.values(snapshot.attempts)) {
|
|
47
|
+
if (attempt.status !== "suspended" ||
|
|
48
|
+
!attempt.agentId ||
|
|
49
|
+
attempt.dispatch.executionKind === "managed-process" ||
|
|
50
|
+
!restartResumableSuspension(attempt, newlySuspendedAttemptIds)) {
|
|
35
51
|
continue;
|
|
52
|
+
}
|
|
36
53
|
const record = lifecycle.getRecord(attempt.taskId);
|
|
37
54
|
const task = snapshot.tasks[attempt.taskId]?.task;
|
|
38
55
|
if (!record || !task)
|
|
@@ -43,10 +60,23 @@ export class WorkerRecoveryCoordinator {
|
|
|
43
60
|
...(task.verificationOfTaskId ? { verificationOfTaskId: task.verificationOfTaskId } : {}),
|
|
44
61
|
});
|
|
45
62
|
}
|
|
63
|
+
let recoveredAll = true;
|
|
46
64
|
for (const { attempt, record, verificationOfTaskId } of queued) {
|
|
47
|
-
|
|
65
|
+
let retained;
|
|
66
|
+
try {
|
|
67
|
+
retained = this.enqueueOrDeferRetry(record, attempt, this.recoveredRequest(attempt, verificationOfTaskId), verificationOfTaskId !== undefined);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
recoveredAll = false;
|
|
71
|
+
this.options.warn(`Failed to recover queued worker ${record.laneId}; retaining durable dispatch: ${error instanceof Error ? error.message : String(error)}`);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (retained)
|
|
75
|
+
continue;
|
|
76
|
+
recoveredAll = false;
|
|
77
|
+
break;
|
|
48
78
|
}
|
|
49
|
-
this.queueRecovered =
|
|
79
|
+
this.queueRecovered = recoveredAll;
|
|
50
80
|
}
|
|
51
81
|
for (const recovery of lifecycle.getPendingVerificationRecoveries()) {
|
|
52
82
|
if (recovery.action === "reconcile") {
|
|
@@ -59,7 +89,19 @@ export class WorkerRecoveryCoordinator {
|
|
|
59
89
|
}
|
|
60
90
|
continue;
|
|
61
91
|
}
|
|
62
|
-
|
|
92
|
+
let started;
|
|
93
|
+
try {
|
|
94
|
+
started = this.options.dispatchVerification(recovery);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
98
|
+
const failureKey = `dispatch_threw:${reason}`;
|
|
99
|
+
if (this.verificationRecoveryFailures.get(recovery.subjectTaskId) !== failureKey) {
|
|
100
|
+
this.verificationRecoveryFailures.set(recovery.subjectTaskId, failureKey);
|
|
101
|
+
this.options.warn(`Recovered verification for ${recovery.subjectTaskId} threw during dispatch: ${reason}`);
|
|
102
|
+
}
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
63
105
|
if (started.started) {
|
|
64
106
|
this.verificationRecoveryFailures.delete(recovery.subjectTaskId);
|
|
65
107
|
continue;
|
|
@@ -69,22 +111,186 @@ export class WorkerRecoveryCoordinator {
|
|
|
69
111
|
this.verificationRecoveryFailures.set(recovery.subjectTaskId, started.skipReason);
|
|
70
112
|
this.options.warn(`Recovered verification for ${recovery.subjectTaskId} did not start: ${started.skipReason}`);
|
|
71
113
|
}
|
|
72
|
-
|
|
73
|
-
|
|
114
|
+
const terminalRecords = lifecycle.getPendingTerminalNotifications().map((notification) => notification.record);
|
|
115
|
+
if (terminalRecords.length > 0) {
|
|
116
|
+
if (this.options.publishTerminalRecords)
|
|
117
|
+
this.options.publishTerminalRecords(terminalRecords);
|
|
118
|
+
else
|
|
119
|
+
for (const record of terminalRecords)
|
|
120
|
+
this.options.publishTerminalRecord(record);
|
|
74
121
|
}
|
|
122
|
+
this.options.recoverSessionRootReplies();
|
|
123
|
+
this.options.recoverTaskBearingMailboxTurns();
|
|
75
124
|
}
|
|
76
125
|
finally {
|
|
77
126
|
this.recoveringDurableState = false;
|
|
78
127
|
}
|
|
79
128
|
}
|
|
129
|
+
/** Evaluate, persist, and arm one retry through the same durable path used by restart recovery. */
|
|
130
|
+
scheduleAttemptRetry(args) {
|
|
131
|
+
const attempt = this.options.lifecycle.getActiveAttempt(args.laneId);
|
|
132
|
+
const record = this.options.lifecycle.getRecord(args.laneId);
|
|
133
|
+
if (!attempt || !record)
|
|
134
|
+
return { scheduled: false, reason: "attempt_missing" };
|
|
135
|
+
const retriesUsed = attempt.retry?.retriesUsed ?? 0;
|
|
136
|
+
const decision = evaluateWorkerRetry({
|
|
137
|
+
laneStatus: args.outcome.laneStatus,
|
|
138
|
+
reasonCode: args.outcome.reasonCode,
|
|
139
|
+
...(args.outcome.reasonDetail ? { reasonDetail: args.outcome.reasonDetail } : {}),
|
|
140
|
+
provider: args.provider,
|
|
141
|
+
retriesUsed,
|
|
142
|
+
...(args.maxAttempts !== undefined ? { maxAttempts: args.maxAttempts } : {}),
|
|
143
|
+
});
|
|
144
|
+
if (!decision.retry)
|
|
145
|
+
return { scheduled: false, reason: decision.reason };
|
|
146
|
+
const nextRetriesUsed = retriesUsed + 1;
|
|
147
|
+
const notBefore = new Date(this.now() + decision.delayMs).toISOString();
|
|
148
|
+
let suspended;
|
|
149
|
+
try {
|
|
150
|
+
suspended = this.options.lifecycle.scheduleAgentRetry({
|
|
151
|
+
laneId: args.laneId,
|
|
152
|
+
agentId: args.agentId,
|
|
153
|
+
ownerId: args.ownerId,
|
|
154
|
+
reasonCode: `retry_scheduled:${decision.reason}`,
|
|
155
|
+
retry: { retriesUsed: nextRetriesUsed, notBefore },
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
this.options.warn(`Worker ${args.laneId} retry suspension failed; terminalizing instead: ${error instanceof Error ? error.message : String(error)}`);
|
|
160
|
+
return { scheduled: false, reason: "retry_suspension_failed" };
|
|
161
|
+
}
|
|
162
|
+
this.options.warn(`Worker ${args.laneId} failed (${decision.reason}); retrying from the persisted transcript in ${Math.ceil(decision.delayMs / 1000)}s (attempt ${nextRetriesUsed + 1}).`);
|
|
163
|
+
this.enqueueOrDeferRetry(record, suspended, args.request, args.request.verificationOfTaskId !== undefined);
|
|
164
|
+
return { scheduled: true, record, retriesUsed: nextRetriesUsed, notBefore };
|
|
165
|
+
}
|
|
166
|
+
/** Cancel only process-local alarms; the retained deadline remains authoritative on disk. */
|
|
167
|
+
dispose() {
|
|
168
|
+
this.unsubscribeQueueCapacity?.();
|
|
169
|
+
for (const entry of this.retryTimers.values()) {
|
|
170
|
+
if (entry.timer)
|
|
171
|
+
clearTimeout(entry.timer);
|
|
172
|
+
}
|
|
173
|
+
this.retryTimers.clear();
|
|
174
|
+
}
|
|
175
|
+
clearScheduledRetry(laneId) {
|
|
176
|
+
for (const [attemptId, entry] of this.retryTimers) {
|
|
177
|
+
if (entry.laneId !== laneId)
|
|
178
|
+
continue;
|
|
179
|
+
if (entry.timer)
|
|
180
|
+
clearTimeout(entry.timer);
|
|
181
|
+
this.retryTimers.delete(attemptId);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/** Keep an externally requested wake queued while the durable retry deadline is still retained. */
|
|
185
|
+
deferRetryIfNeeded(record, request) {
|
|
186
|
+
const attempt = this.options.lifecycle.getActiveAttempt(record.laneId);
|
|
187
|
+
if (!attempt ||
|
|
188
|
+
attempt.status !== "suspended" ||
|
|
189
|
+
!attempt.retry ||
|
|
190
|
+
Date.parse(attempt.retry.notBefore) <= this.now()) {
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
this.enqueueOrDeferRetry(record, attempt, request, request.verificationOfTaskId !== undefined);
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
enqueueOrDeferRetry(record, attempt, request, priority) {
|
|
197
|
+
const retry = attempt.retry;
|
|
198
|
+
if (!retry) {
|
|
199
|
+
try {
|
|
200
|
+
this.options.scheduler.enqueue(record, request, true, priority);
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (error instanceof Error && error.message === "worker_dispatch_queue_full")
|
|
205
|
+
return false;
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const existing = this.retryTimers.get(attempt.attemptId);
|
|
210
|
+
if (existing?.notBefore === retry.notBefore)
|
|
211
|
+
return true;
|
|
212
|
+
if (existing?.timer)
|
|
213
|
+
clearTimeout(existing.timer);
|
|
214
|
+
const entry = {
|
|
215
|
+
laneId: record.laneId,
|
|
216
|
+
notBefore: retry.notBefore,
|
|
217
|
+
request,
|
|
218
|
+
priority,
|
|
219
|
+
};
|
|
220
|
+
this.retryTimers.set(attempt.attemptId, entry);
|
|
221
|
+
const remainingMs = Date.parse(retry.notBefore) - this.now();
|
|
222
|
+
if (remainingMs <= 0) {
|
|
223
|
+
this.tryEnqueueDueRetry(attempt.attemptId, entry);
|
|
224
|
+
return true;
|
|
225
|
+
}
|
|
226
|
+
const timer = setTimeout(() => this.onRetryTimer(attempt.attemptId), Math.min(remainingMs, 2_147_000_000));
|
|
227
|
+
if (typeof timer === "object" && "unref" in timer)
|
|
228
|
+
timer.unref();
|
|
229
|
+
entry.timer = timer;
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
onRetryTimer(attemptId) {
|
|
233
|
+
const entry = this.retryTimers.get(attemptId);
|
|
234
|
+
if (!entry)
|
|
235
|
+
return;
|
|
236
|
+
delete entry.timer;
|
|
237
|
+
this.tryEnqueueDueRetry(attemptId, entry);
|
|
238
|
+
}
|
|
239
|
+
tryEnqueueDueRetry(attemptId, entry) {
|
|
240
|
+
const attempt = this.options.lifecycle.getActiveAttempt(entry.laneId);
|
|
241
|
+
if (!attempt ||
|
|
242
|
+
attempt.attemptId !== attemptId ||
|
|
243
|
+
attempt.status !== "suspended" ||
|
|
244
|
+
attempt.retry?.notBefore !== entry.notBefore) {
|
|
245
|
+
this.retryTimers.delete(attemptId);
|
|
246
|
+
return "stale";
|
|
247
|
+
}
|
|
248
|
+
const record = this.options.lifecycle.getRecord(entry.laneId);
|
|
249
|
+
if (!record) {
|
|
250
|
+
this.retryTimers.delete(attemptId);
|
|
251
|
+
return "stale";
|
|
252
|
+
}
|
|
253
|
+
const remainingMs = Date.parse(entry.notBefore) - this.now();
|
|
254
|
+
if (remainingMs > 0) {
|
|
255
|
+
const timer = setTimeout(() => this.onRetryTimer(attemptId), Math.min(remainingMs, 2_147_000_000));
|
|
256
|
+
if (typeof timer === "object" && "unref" in timer)
|
|
257
|
+
timer.unref();
|
|
258
|
+
entry.timer = timer;
|
|
259
|
+
return "stale";
|
|
260
|
+
}
|
|
261
|
+
try {
|
|
262
|
+
this.options.scheduler.enqueue(record, entry.request, true, entry.priority);
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
if (error instanceof Error && error.message === "worker_dispatch_queue_full")
|
|
266
|
+
return "full";
|
|
267
|
+
this.options.warn(`Worker ${entry.laneId} retry enqueue failed; retaining the durable retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
268
|
+
return "full";
|
|
269
|
+
}
|
|
270
|
+
this.retryTimers.delete(attemptId);
|
|
271
|
+
this.options.retryReady?.();
|
|
272
|
+
return "enqueued";
|
|
273
|
+
}
|
|
274
|
+
flushDueRetries() {
|
|
275
|
+
for (const [attemptId, entry] of this.retryTimers) {
|
|
276
|
+
if (entry.timer || Date.parse(entry.notBefore) > this.now())
|
|
277
|
+
continue;
|
|
278
|
+
if (this.tryEnqueueDueRetry(attemptId, entry) === "full")
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
now() {
|
|
283
|
+
return this.options.now?.() ?? Date.now();
|
|
284
|
+
}
|
|
80
285
|
/** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */
|
|
81
286
|
recoveredRequest(attempt, verificationOfTaskId) {
|
|
82
287
|
const task = this.options.lifecycle.getTask(attempt.taskId)?.task;
|
|
288
|
+
const durableVerificationOfTaskId = verificationOfTaskId ?? task?.verificationOfTaskId;
|
|
83
289
|
return {
|
|
84
290
|
instructions: attempt.dispatch.instructions,
|
|
85
291
|
profileId: attempt.dispatch.profileId,
|
|
86
292
|
...(attempt.dispatch.parentAgentId ? { parentAgentId: attempt.dispatch.parentAgentId } : {}),
|
|
87
|
-
...(
|
|
293
|
+
...(durableVerificationOfTaskId ? { verificationOfTaskId: durableVerificationOfTaskId } : {}),
|
|
88
294
|
...(task
|
|
89
295
|
? {
|
|
90
296
|
taskContext: {
|
|
@@ -128,8 +334,8 @@ export class WorkerRecoveryCoordinator {
|
|
|
128
334
|
}
|
|
129
335
|
}
|
|
130
336
|
/** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */
|
|
131
|
-
recoveredTerminalCompletion(conversation) {
|
|
132
|
-
const last = conversation.
|
|
337
|
+
recoveredTerminalCompletion(conversation, attemptId) {
|
|
338
|
+
const last = conversation.getLastAttemptMessage(attemptId);
|
|
133
339
|
if (!last || last.role !== "assistant" || last.stopReason === "error" || last.stopReason === "aborted") {
|
|
134
340
|
return undefined;
|
|
135
341
|
}
|
|
@@ -139,11 +345,11 @@ export class WorkerRecoveryCoordinator {
|
|
|
139
345
|
.filter((content) => content.type === "text")
|
|
140
346
|
.map((content) => content.text)
|
|
141
347
|
.join("");
|
|
142
|
-
return
|
|
348
|
+
return { text, usage: last.usage, stopReason: last.stopReason };
|
|
143
349
|
}
|
|
144
350
|
/** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */
|
|
145
|
-
initialUsage(conversation, checkpointUsage) {
|
|
146
|
-
const transcriptUsage = conversation.getRawTranscriptUsage();
|
|
351
|
+
initialUsage(conversation, checkpointUsage, attemptId) {
|
|
352
|
+
const transcriptUsage = conversation.getRawTranscriptUsage(attemptId);
|
|
147
353
|
return checkpointUsage ? reconcileAttemptUsage(checkpointUsage, transcriptUsage) : transcriptUsage;
|
|
148
354
|
}
|
|
149
355
|
/** Fail closed when the current owner profile no longer permits a persisted grant. */
|