@namzu/sdk 6.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +677 -0
- package/dist/agents/ReactiveAgent.d.ts.map +1 -1
- package/dist/agents/ReactiveAgent.js +5 -0
- package/dist/agents/ReactiveAgent.js.map +1 -1
- package/dist/agents/SupervisorAgent.d.ts.map +1 -1
- package/dist/agents/SupervisorAgent.js +172 -158
- package/dist/agents/SupervisorAgent.js.map +1 -1
- package/dist/agents/__tests__/supervisor-inbox-scope.test.d.ts +2 -0
- package/dist/agents/__tests__/supervisor-inbox-scope.test.d.ts.map +1 -0
- package/dist/agents/__tests__/supervisor-inbox-scope.test.js +125 -0
- package/dist/agents/__tests__/supervisor-inbox-scope.test.js.map +1 -0
- package/dist/agents/runAgent.d.ts +19 -1
- package/dist/agents/runAgent.d.ts.map +1 -1
- package/dist/agents/runAgent.js +2 -0
- package/dist/agents/runAgent.js.map +1 -1
- package/dist/bridge/a2a/mapper.d.ts.map +1 -1
- package/dist/bridge/a2a/mapper.js +4 -0
- package/dist/bridge/a2a/mapper.js.map +1 -1
- package/dist/bridge/sse/mapper.d.ts.map +1 -1
- package/dist/bridge/sse/mapper.js +24 -0
- package/dist/bridge/sse/mapper.js.map +1 -1
- package/dist/connector/mcp/__tests__/positional-arrays.test.d.ts +2 -0
- package/dist/connector/mcp/__tests__/positional-arrays.test.d.ts.map +1 -0
- package/dist/connector/mcp/__tests__/positional-arrays.test.js +142 -0
- package/dist/connector/mcp/__tests__/positional-arrays.test.js.map +1 -0
- package/dist/connector/mcp/adapter.d.ts.map +1 -1
- package/dist/connector/mcp/adapter.js +123 -8
- package/dist/connector/mcp/adapter.js.map +1 -1
- package/dist/constants/agent/index.d.ts +5 -0
- package/dist/constants/agent/index.d.ts.map +1 -1
- package/dist/constants/agent/index.js +5 -0
- package/dist/constants/agent/index.js.map +1 -1
- package/dist/constants/plugin/index.d.ts +15 -0
- package/dist/constants/plugin/index.d.ts.map +1 -1
- package/dist/constants/plugin/index.js +15 -0
- package/dist/constants/plugin/index.js.map +1 -1
- package/dist/contracts/api.d.ts +1 -1
- package/dist/contracts/api.d.ts.map +1 -1
- package/dist/gateway/__tests__/completion-inbox.test.js +292 -2
- package/dist/gateway/__tests__/completion-inbox.test.js.map +1 -1
- package/dist/gateway/completion-inbox.d.ts +94 -6
- package/dist/gateway/completion-inbox.d.ts.map +1 -1
- package/dist/gateway/completion-inbox.js +235 -15
- package/dist/gateway/completion-inbox.js.map +1 -1
- package/dist/gateway/local.d.ts +11 -0
- package/dist/gateway/local.d.ts.map +1 -1
- package/dist/gateway/local.js +27 -1
- package/dist/gateway/local.js.map +1 -1
- package/dist/manager/agent/lifecycle.d.ts.map +1 -1
- package/dist/manager/agent/lifecycle.js +6 -0
- package/dist/manager/agent/lifecycle.js.map +1 -1
- package/dist/manager/run/persistence.d.ts +8 -0
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +12 -0
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/provider/thinking-support.d.ts +2 -1
- package/dist/provider/thinking-support.d.ts.map +1 -1
- package/dist/provider/thinking-support.js +14 -0
- package/dist/provider/thinking-support.js.map +1 -1
- package/dist/public-runtime.d.ts +1 -1
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +9 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/run/reporter.d.ts.map +1 -1
- package/dist/run/reporter.js +11 -0
- package/dist/run/reporter.js.map +1 -1
- package/dist/runtime/query/__tests__/completion-does-not-erase-the-answer.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/completion-does-not-erase-the-answer.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/completion-does-not-erase-the-answer.test.js +142 -0
- package/dist/runtime/query/__tests__/completion-does-not-erase-the-answer.test.js.map +1 -0
- package/dist/runtime/query/__tests__/completion-notification.test.js +414 -32
- package/dist/runtime/query/__tests__/completion-notification.test.js.map +1 -1
- package/dist/runtime/query/__tests__/context-size-on-the-wire.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/context-size-on-the-wire.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/context-size-on-the-wire.test.js +100 -0
- package/dist/runtime/query/__tests__/context-size-on-the-wire.test.js.map +1 -0
- package/dist/runtime/query/__tests__/context.test.js +18 -0
- package/dist/runtime/query/__tests__/context.test.js.map +1 -1
- package/dist/runtime/query/__tests__/effort-reaches-the-wire.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/effort-reaches-the-wire.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/effort-reaches-the-wire.test.js +118 -0
- package/dist/runtime/query/__tests__/effort-reaches-the-wire.test.js.map +1 -0
- package/dist/runtime/query/__tests__/tool-timeout.test.js +34 -0
- package/dist/runtime/query/__tests__/tool-timeout.test.js.map +1 -1
- package/dist/runtime/query/context.d.ts.map +1 -1
- package/dist/runtime/query/context.js +16 -1
- package/dist/runtime/query/context.js.map +1 -1
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +11 -1
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/guard.d.ts +28 -0
- package/dist/runtime/query/guard.d.ts.map +1 -1
- package/dist/runtime/query/guard.js +31 -0
- package/dist/runtime/query/guard.js.map +1 -1
- package/dist/runtime/query/iteration/__tests__/settle-grace.test.d.ts +2 -0
- package/dist/runtime/query/iteration/__tests__/settle-grace.test.d.ts.map +1 -0
- package/dist/runtime/query/iteration/__tests__/settle-grace.test.js +226 -0
- package/dist/runtime/query/iteration/__tests__/settle-grace.test.js.map +1 -0
- package/dist/runtime/query/iteration/index.d.ts +92 -0
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +818 -565
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/__tests__/compaction-declined.test.d.ts +2 -0
- package/dist/runtime/query/iteration/phases/__tests__/compaction-declined.test.d.ts.map +1 -0
- package/dist/runtime/query/iteration/phases/__tests__/compaction-declined.test.js +95 -0
- package/dist/runtime/query/iteration/phases/__tests__/compaction-declined.test.js.map +1 -0
- package/dist/runtime/query/iteration/phases/compaction.d.ts +34 -0
- package/dist/runtime/query/iteration/phases/compaction.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/compaction.js +61 -4
- package/dist/runtime/query/iteration/phases/compaction.js.map +1 -1
- package/dist/telemetry/__tests__/model-call-span.test.js +22 -4
- package/dist/telemetry/__tests__/model-call-span.test.js.map +1 -1
- package/dist/telemetry/__tests__/span-closure.test.js +12 -5
- package/dist/telemetry/__tests__/span-closure.test.js.map +1 -1
- package/dist/tools/__tests__/untrusted-envelope.test.js +16 -0
- package/dist/tools/__tests__/untrusted-envelope.test.js.map +1 -1
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js +117 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js.map +1 -1
- package/dist/tools/coordinator/__tests__/task-list.test.js +57 -0
- package/dist/tools/coordinator/__tests__/task-list.test.js.map +1 -1
- package/dist/tools/coordinator/__tests__/wait-with-idle-bound.test.d.ts +2 -0
- package/dist/tools/coordinator/__tests__/wait-with-idle-bound.test.d.ts.map +1 -0
- package/dist/tools/coordinator/__tests__/wait-with-idle-bound.test.js +193 -0
- package/dist/tools/coordinator/__tests__/wait-with-idle-bound.test.js.map +1 -0
- package/dist/tools/coordinator/index.d.ts +19 -0
- package/dist/tools/coordinator/index.d.ts.map +1 -1
- package/dist/tools/coordinator/index.js +191 -71
- package/dist/tools/coordinator/index.js.map +1 -1
- package/dist/tools/coordinator/wait-with-idle-bound.d.ts +66 -0
- package/dist/tools/coordinator/wait-with-idle-bound.d.ts.map +1 -0
- package/dist/tools/coordinator/wait-with-idle-bound.js +78 -0
- package/dist/tools/coordinator/wait-with-idle-bound.js.map +1 -0
- package/dist/tools/untrusted-envelope.d.ts.map +1 -1
- package/dist/tools/untrusted-envelope.js +9 -1
- package/dist/tools/untrusted-envelope.js.map +1 -1
- package/dist/types/agent/base.d.ts +16 -0
- package/dist/types/agent/base.d.ts.map +1 -1
- package/dist/types/agent/gateway.d.ts +41 -0
- package/dist/types/agent/gateway.d.ts.map +1 -1
- package/dist/types/agent/lifecycle-event.d.ts +9 -1
- package/dist/types/agent/lifecycle-event.d.ts.map +1 -1
- package/dist/types/agent/task.d.ts +5 -0
- package/dist/types/agent/task.d.ts.map +1 -1
- package/dist/types/hitl/index.d.ts +10 -0
- package/dist/types/hitl/index.d.ts.map +1 -1
- package/dist/types/hitl/index.js.map +1 -1
- package/dist/types/probe/registry.d.ts +6 -0
- package/dist/types/probe/registry.d.ts.map +1 -1
- package/dist/types/provider/interface.d.ts +35 -0
- package/dist/types/provider/interface.d.ts.map +1 -1
- package/dist/types/run/config.d.ts +25 -0
- package/dist/types/run/config.d.ts.map +1 -1
- package/dist/types/run/entity.d.ts +16 -0
- package/dist/types/run/entity.d.ts.map +1 -1
- package/dist/types/run/events.d.ts +75 -0
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/types/run/prepare-step.d.ts +17 -2
- package/dist/types/run/prepare-step.d.ts.map +1 -1
- package/dist/types/verification/index.d.ts +98 -0
- package/dist/types/verification/index.d.ts.map +1 -1
- package/dist/types/verification/index.js +10 -0
- package/dist/types/verification/index.js.map +1 -1
- package/dist/utils/__tests__/abort-reason.test.d.ts +2 -0
- package/dist/utils/__tests__/abort-reason.test.d.ts.map +1 -0
- package/dist/utils/__tests__/abort-reason.test.js +48 -0
- package/dist/utils/__tests__/abort-reason.test.js.map +1 -0
- package/dist/utils/abort.d.ts +26 -0
- package/dist/utils/abort.d.ts.map +1 -1
- package/dist/utils/abort.js +34 -0
- package/dist/utils/abort.js.map +1 -1
- package/dist/verification/__tests__/argument-pattern.test.d.ts +2 -0
- package/dist/verification/__tests__/argument-pattern.test.d.ts.map +1 -0
- package/dist/verification/__tests__/argument-pattern.test.js +122 -0
- package/dist/verification/__tests__/argument-pattern.test.js.map +1 -0
- package/dist/verification/__tests__/rule-order-and-reason.test.d.ts +2 -0
- package/dist/verification/__tests__/rule-order-and-reason.test.d.ts.map +1 -0
- package/dist/verification/__tests__/rule-order-and-reason.test.js +126 -0
- package/dist/verification/__tests__/rule-order-and-reason.test.js.map +1 -0
- package/dist/verification/gate.d.ts +17 -1
- package/dist/verification/gate.d.ts.map +1 -1
- package/dist/verification/gate.js +102 -2
- package/dist/verification/gate.js.map +1 -1
- package/dist/verification/index.d.ts +1 -1
- package/dist/verification/index.d.ts.map +1 -1
- package/dist/verification/index.js +1 -1
- package/dist/verification/index.js.map +1 -1
- package/dist/verification/rules.d.ts.map +1 -1
- package/dist/verification/rules.js +27 -0
- package/dist/verification/rules.js.map +1 -1
- package/package.json +1 -1
- package/src/agents/ReactiveAgent.ts +5 -0
- package/src/agents/SupervisorAgent.ts +175 -162
- package/src/agents/__tests__/supervisor-inbox-scope.test.ts +149 -0
- package/src/agents/runAgent.ts +22 -1
- package/src/bridge/a2a/mapper.ts +4 -0
- package/src/bridge/sse/mapper.ts +25 -0
- package/src/connector/mcp/__tests__/positional-arrays.test.ts +183 -0
- package/src/connector/mcp/adapter.ts +131 -7
- package/src/constants/agent/index.ts +5 -0
- package/src/constants/plugin/index.ts +15 -0
- package/src/contracts/api.ts +1 -0
- package/src/gateway/__tests__/completion-inbox.test.ts +348 -2
- package/src/gateway/completion-inbox.ts +248 -16
- package/src/gateway/local.ts +26 -1
- package/src/manager/agent/lifecycle.ts +6 -0
- package/src/manager/run/persistence.ts +12 -0
- package/src/provider/thinking-support.ts +19 -2
- package/src/public-runtime.ts +9 -0
- package/src/run/reporter.ts +12 -0
- package/src/runtime/query/__tests__/completion-does-not-erase-the-answer.test.ts +163 -0
- package/src/runtime/query/__tests__/completion-notification.test.ts +486 -34
- package/src/runtime/query/__tests__/context-size-on-the-wire.test.ts +122 -0
- package/src/runtime/query/__tests__/context.test.ts +24 -0
- package/src/runtime/query/__tests__/effort-reaches-the-wire.test.ts +135 -0
- package/src/runtime/query/__tests__/tool-timeout.test.ts +38 -0
- package/src/runtime/query/context.ts +16 -1
- package/src/runtime/query/executor.ts +11 -1
- package/src/runtime/query/guard.ts +32 -0
- package/src/runtime/query/iteration/__tests__/settle-grace.test.ts +265 -0
- package/src/runtime/query/iteration/index.ts +906 -635
- package/src/runtime/query/iteration/phases/__tests__/compaction-declined.test.ts +124 -0
- package/src/runtime/query/iteration/phases/compaction.ts +83 -10
- package/src/telemetry/__tests__/model-call-span.test.ts +22 -5
- package/src/telemetry/__tests__/span-closure.test.ts +12 -5
- package/src/tools/__tests__/untrusted-envelope.test.ts +23 -0
- package/src/tools/coordinator/__tests__/completion-delivery.test.ts +147 -0
- package/src/tools/coordinator/__tests__/task-list.test.ts +72 -0
- package/src/tools/coordinator/__tests__/wait-with-idle-bound.test.ts +247 -0
- package/src/tools/coordinator/index.ts +205 -78
- package/src/tools/coordinator/wait-with-idle-bound.ts +142 -0
- package/src/tools/untrusted-envelope.ts +9 -1
- package/src/types/agent/base.ts +17 -0
- package/src/types/agent/gateway.ts +42 -0
- package/src/types/agent/lifecycle-event.ts +7 -0
- package/src/types/agent/task.ts +5 -0
- package/src/types/hitl/index.ts +10 -0
- package/src/types/probe/registry.ts +6 -0
- package/src/types/provider/interface.ts +39 -0
- package/src/types/run/config.ts +26 -0
- package/src/types/run/entity.ts +17 -0
- package/src/types/run/events.ts +75 -0
- package/src/types/run/prepare-step.ts +17 -2
- package/src/types/verification/index.ts +61 -0
- package/src/utils/__tests__/abort-reason.test.ts +56 -0
- package/src/utils/abort.ts +34 -0
- package/src/verification/__tests__/argument-pattern.test.ts +158 -0
- package/src/verification/__tests__/rule-order-and-reason.test.ts +149 -0
- package/src/verification/gate.ts +106 -3
- package/src/verification/index.ts +1 -1
- package/src/verification/rules.ts +28 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Span, SpanStatusCode } from '@opentelemetry/api'
|
|
2
|
+
import { resolveContextWindow } from '../../../compaction/context-window.js'
|
|
2
3
|
import { extractFromAssistantMessage } from '../../../compaction/extractor.js'
|
|
3
4
|
import { AUTO_CONTINUATION_USER_MESSAGE } from '../../../constants/continuation.js'
|
|
4
5
|
import {
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
} from '../../../telemetry/attributes.js'
|
|
17
18
|
import { getTracer } from '../../../telemetry/runtime-accessors.js'
|
|
18
19
|
import { STRUCTURED_OUTPUT_TOOL_NAME } from '../../../tools/builtins/structuredOutput.js'
|
|
20
|
+
import { DELEGATION_TIMEOUT_MS } from '../../../tools/coordinator/index.js'
|
|
19
21
|
import type { CostInfo, TokenUsage } from '../../../types/common/index.js'
|
|
20
22
|
import type { MessageId } from '../../../types/ids/index.js'
|
|
21
23
|
import {
|
|
@@ -41,7 +43,7 @@ import type { ToolCallOutcome } from '../executor.js'
|
|
|
41
43
|
import { applyLifecycleHookResults } from '../plugin-hooks.js'
|
|
42
44
|
import { runAdvisoryPhase } from './phases/advisory.js'
|
|
43
45
|
import { runIterationCheckpoint } from './phases/checkpoint.js'
|
|
44
|
-
import { relieveOverflow, runCompactionCheck } from './phases/compaction.js'
|
|
46
|
+
import { measureContext, relieveOverflow, runCompactionCheck } from './phases/compaction.js'
|
|
45
47
|
import type { IterationContext } from './phases/index.js'
|
|
46
48
|
import { runPlanGate } from './phases/plan.js'
|
|
47
49
|
import { runToolReview } from './phases/tool-review.js'
|
|
@@ -62,20 +64,87 @@ export type { ToolReviewOutcome } from './phases/index.js'
|
|
|
62
64
|
*/
|
|
63
65
|
const DEFAULT_ANSWER_REVIEW_LIMIT = 3
|
|
64
66
|
|
|
67
|
+
/**
|
|
68
|
+
* The share of a run's REMAINING time a settle-hold may take.
|
|
69
|
+
*
|
|
70
|
+
* The rule is borrowed from `AGENT_MANAGER_DEFAULTS.maxBudgetFraction`, which
|
|
71
|
+
* gives a spawned child at most half of what its parent has left: one
|
|
72
|
+
* sub-activity may take a share of the remainder, never the remainder. The
|
|
73
|
+
* value is written out here rather than imported, because that field is a
|
|
74
|
+
* host-tunable knob about TOKEN allocation and coupling the two would let a
|
|
75
|
+
* host lowering one silently change the other.
|
|
76
|
+
*
|
|
77
|
+
* Half, specifically, because the hold is not the last thing the run does.
|
|
78
|
+
* Its whole purpose is to put a worker's result where the model can read it,
|
|
79
|
+
* and reading it costs a turn. A hold that spent everything remaining would
|
|
80
|
+
* deliver a notification into a run with no turn left to act on it — the same
|
|
81
|
+
* "the result exists and the model is never told" failure this mechanism was
|
|
82
|
+
* built to close, wearing a different costume.
|
|
83
|
+
*/
|
|
84
|
+
const SETTLE_GRACE_FRACTION = 0.5
|
|
85
|
+
|
|
65
86
|
/**
|
|
66
87
|
* How long a finishing run waits for a background worker it launched.
|
|
67
88
|
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
89
|
+
* Derived from the run rather than fixed, because a constant is wrong in both
|
|
90
|
+
* directions at once. The 120 seconds this replaces held a run configured for
|
|
91
|
+
* a twenty-second timeout open for 120,267 ms — six times its own budget, and
|
|
92
|
+
* unreachable by the guard, which only checks between iterations — while on an
|
|
93
|
+
* hour-long run it abandoned workers measured at 4m21s, 5m58s and 8m04s, all
|
|
94
|
+
* of them well inside the hour the delegation tools themselves declare.
|
|
95
|
+
*
|
|
96
|
+
* **Bounded by construction, and against the right boundary.** The input is
|
|
97
|
+
* time-to-FINALIZE, not time-to-deadline (see
|
|
98
|
+
* `GuardCoordinator.remainingBeforeFinalizeMs`). Measuring to the deadline was
|
|
99
|
+
* the first attempt and it was wrong in a way that looked safe: a hold cannot
|
|
100
|
+
* outlive the deadline either way, but half of the time-to-deadline started
|
|
101
|
+
* just under the warning threshold ends at 95% of the budget — so the slice
|
|
102
|
+
* that exists for the run to produce a closing answer is half spent waiting
|
|
103
|
+
* for the result that answer was supposed to use. Against the finalize point
|
|
104
|
+
* the hold cannot reach the reserve at all, which is what makes the guard's
|
|
105
|
+
* inability to interrupt a hold a non-issue rather than a smaller issue.
|
|
106
|
+
*
|
|
107
|
+
* **The floor of zero is a decision, not a clamp artefact.** A run with no
|
|
108
|
+
* time left before it must start finishing has no turn in which to read a
|
|
109
|
+
* notification, so waiting could only delay a stop that is already due.
|
|
110
|
+
* Nothing is lost by it: `CompletionInbox.waitForArrival` returns before it
|
|
111
|
+
* looks at its timer when a completion is already in hand, so a zero grace
|
|
112
|
+
* still delivers everything that has arrived. No minimum is invented on top,
|
|
113
|
+
* because zero is exactly what a run past the threshold should wait — and
|
|
114
|
+
* reading the remainder at hold time rather than trusting `forceFinalize`,
|
|
115
|
+
* which is sampled at the top of the iteration, is what makes a long iteration
|
|
116
|
+
* that crossed the line in between compute it.
|
|
117
|
+
*
|
|
118
|
+
* **The ceiling is the longest anything in this subsystem waits for a
|
|
119
|
+
* delegated worker.** It binds only for a host whose run timeout exceeds
|
|
120
|
+
* roughly two and a quarter hours; below that the fraction is smaller.
|
|
72
121
|
*/
|
|
73
|
-
|
|
122
|
+
export function settleGraceMs(remainingBeforeFinalizeMs: number): number {
|
|
123
|
+
return Math.min(
|
|
124
|
+
Math.floor(remainingBeforeFinalizeMs * SETTLE_GRACE_FRACTION),
|
|
125
|
+
DELEGATION_TIMEOUT_MS,
|
|
126
|
+
)
|
|
127
|
+
}
|
|
74
128
|
|
|
75
129
|
export class IterationOrchestrator {
|
|
76
130
|
private ctx: IterationContext
|
|
77
131
|
/** Rejections so far. See {@link DEFAULT_ANSWER_REVIEW_LIMIT}. */
|
|
78
132
|
private answerReviewAttempts = 0
|
|
133
|
+
/**
|
|
134
|
+
* The previous iteration held a `stopWhen` decision open for a worker.
|
|
135
|
+
*
|
|
136
|
+
* Set when the stop predicate fired and the run took one extra turn to
|
|
137
|
+
* read a delegated result, so the turn that then ends the run can report
|
|
138
|
+
* WHY it is over. Without it the outcome was right and the record was
|
|
139
|
+
* wrong: the run stopped because the host said so and reported `end_turn`,
|
|
140
|
+
* and this repo carries thirteen `StopReason` values precisely so that a
|
|
141
|
+
* run which ends for a nameable reason names it.
|
|
142
|
+
*
|
|
143
|
+
* Lives for exactly one iteration — see the read-and-clear at the top of
|
|
144
|
+
* the loop, which is the only site that touches it besides the one that
|
|
145
|
+
* sets it.
|
|
146
|
+
*/
|
|
147
|
+
private stopDeferredForOutstandingWork = false
|
|
79
148
|
|
|
80
149
|
constructor(ctx: IterationContext) {
|
|
81
150
|
this.ctx = ctx
|
|
@@ -111,452 +180,461 @@ export class IterationOrchestrator {
|
|
|
111
180
|
const planSignal = yield* runPlanGate(this.ctx)
|
|
112
181
|
if (planSignal === 'stop') return
|
|
113
182
|
|
|
114
|
-
|
|
115
|
-
|
|
183
|
+
// A `finally` rather than a line at each exit, for the reason written
|
|
184
|
+
// beside `iterSpan.end()` below: this loop leaves by eight `break`s,
|
|
185
|
+
// two `return`s and a `throw`, and a rule every future edit has to
|
|
186
|
+
// remember is a rule that gets forgotten — measured, it had been. Only
|
|
187
|
+
// the ordinary final-answer exit consulted the inbox, so a run that
|
|
188
|
+
// ended on a terminal tool, a structured output or the host's
|
|
189
|
+
// `stopWhen` settled over a finished worker's output and threw it away.
|
|
190
|
+
// A `finally` also covers a generator abandoned by its consumer, which
|
|
191
|
+
// no post-loop block reaches.
|
|
192
|
+
try {
|
|
193
|
+
while (true) {
|
|
194
|
+
// Read AND clear, in that order, in this one place.
|
|
195
|
+
//
|
|
196
|
+
// The flag is set by the previous iteration and read by this
|
|
197
|
+
// one, so a clear that ran before the read would wipe it
|
|
198
|
+
// before anything could use it — the obvious spelling of
|
|
199
|
+
// "clear it at the top" is the broken one. Taking the value
|
|
200
|
+
// into a local first gives the flag a lifetime of exactly one
|
|
201
|
+
// iteration, which is the property that makes this cheap: no
|
|
202
|
+
// path has to remember to clear it, because the next iteration
|
|
203
|
+
// does so whether or not anything read it, and there is no
|
|
204
|
+
// path by which a stale deferral can reach a later turn.
|
|
205
|
+
const stopWasDeferredForOutstandingWork = this.stopDeferredForOutstandingWork
|
|
206
|
+
this.stopDeferredForOutstandingWork = false
|
|
207
|
+
|
|
208
|
+
const guardResult = this.ctx.guard.beforeIteration(runMgr, this.ctx.abortController.signal)
|
|
209
|
+
|
|
210
|
+
if (guardResult.shouldStop) {
|
|
211
|
+
if (guardResult.isCancelled) {
|
|
212
|
+
this.ctx.log.info('Run cancelled by signal', { runId: runMgr.id })
|
|
213
|
+
runMgr.setStopReason('cancelled')
|
|
214
|
+
runMgr.markCancelled()
|
|
215
|
+
break
|
|
216
|
+
}
|
|
116
217
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
218
|
+
const stopReason = guardResult.stopReason ?? 'end_turn'
|
|
219
|
+
this.ctx.log.info('Guard enforcing stop', {
|
|
220
|
+
runId: runMgr.id,
|
|
221
|
+
stopReason,
|
|
222
|
+
iteration: runMgr.currentIteration,
|
|
223
|
+
inputTokens: runMgr.tokenUsage.promptTokens,
|
|
224
|
+
outputTokens: runMgr.tokenUsage.completionTokens,
|
|
225
|
+
})
|
|
226
|
+
await this.requestFinalResponse(model, stopReason)
|
|
227
|
+
yield* this.ctx.drainPending()
|
|
228
|
+
runMgr.setStopReason(stopReason)
|
|
122
229
|
break
|
|
123
230
|
}
|
|
124
231
|
|
|
125
|
-
const
|
|
126
|
-
|
|
232
|
+
const forceFinalize = guardResult.forceFinalize
|
|
233
|
+
const iterationNum = runMgr.incrementIteration()
|
|
234
|
+
this.ctx.log.debug('Iteration started', {
|
|
127
235
|
runId: runMgr.id,
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
236
|
+
iteration: iterationNum,
|
|
237
|
+
model,
|
|
238
|
+
forceFinalize,
|
|
239
|
+
messageCount: runMgr.messages.length,
|
|
132
240
|
})
|
|
133
|
-
await this.requestFinalResponse(model, stopReason)
|
|
134
|
-
yield* this.ctx.drainPending()
|
|
135
|
-
runMgr.setStopReason(stopReason)
|
|
136
|
-
break
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
const forceFinalize = guardResult.forceFinalize
|
|
140
|
-
const iterationNum = runMgr.incrementIteration()
|
|
141
|
-
this.ctx.log.debug('Iteration started', {
|
|
142
|
-
runId: runMgr.id,
|
|
143
|
-
iteration: iterationNum,
|
|
144
|
-
model,
|
|
145
|
-
forceFinalize,
|
|
146
|
-
messageCount: runMgr.messages.length,
|
|
147
|
-
})
|
|
148
241
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
})
|
|
153
|
-
if (iterationActivity) {
|
|
154
|
-
this.ctx.activityStore.start(iterationActivity.id)
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
// Parent explicitly: this body is an async generator, so the
|
|
158
|
-
// ambient context at resume time belongs to the CONSUMER, not to
|
|
159
|
-
// whoever created the run span. Without this every iteration
|
|
160
|
-
// emits as its own root and a 20-turn run shows up as 21
|
|
161
|
-
// disconnected traces.
|
|
162
|
-
const iterSpan = tracer.startSpan(
|
|
163
|
-
agentIterationSpanName(iterationNum),
|
|
164
|
-
{},
|
|
165
|
-
parentContext(this.ctx.rootSpan),
|
|
166
|
-
)
|
|
167
|
-
try {
|
|
168
|
-
// Tool spans for this turn belong under this iteration. Inside
|
|
169
|
-
// the try rather than before it: a throw from any of these left
|
|
170
|
-
// the span open, and an iteration span that never ends is a
|
|
171
|
-
// trace that never closes — the export is incomplete for exactly
|
|
172
|
-
// the run that failed.
|
|
173
|
-
this.ctx.toolExecutor.setParentSpan(iterSpan)
|
|
174
|
-
|
|
175
|
-
iterSpan.setAttributes({
|
|
176
|
-
[NAMZU.ITERATION]: iterationNum,
|
|
177
|
-
[NAMZU.RUN_ID]: runMgr.id,
|
|
178
|
-
[GENAI.REQUEST_MODEL]: model,
|
|
242
|
+
const iterationActivity = this.ctx.activityStore.create({
|
|
243
|
+
type: 'llm_turn',
|
|
244
|
+
description: `LLM iteration ${iterationNum}`,
|
|
179
245
|
})
|
|
246
|
+
if (iterationActivity) {
|
|
247
|
+
this.ctx.activityStore.start(iterationActivity.id)
|
|
248
|
+
}
|
|
180
249
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
250
|
+
// Parent explicitly: this body is an async generator, so the
|
|
251
|
+
// ambient context at resume time belongs to the CONSUMER, not to
|
|
252
|
+
// whoever created the run span. Without this every iteration
|
|
253
|
+
// emits as its own root and a 20-turn run shows up as 21
|
|
254
|
+
// disconnected traces.
|
|
255
|
+
const iterSpan = tracer.startSpan(
|
|
256
|
+
agentIterationSpanName(iterationNum),
|
|
257
|
+
{},
|
|
258
|
+
parentContext(this.ctx.rootSpan),
|
|
259
|
+
)
|
|
260
|
+
try {
|
|
261
|
+
// Tool spans for this turn belong under this iteration. Inside
|
|
262
|
+
// the try rather than before it: a throw from any of these left
|
|
263
|
+
// the span open, and an iteration span that never ends is a
|
|
264
|
+
// trace that never closes — the export is incomplete for exactly
|
|
265
|
+
// the run that failed.
|
|
266
|
+
this.ctx.toolExecutor.setParentSpan(iterSpan)
|
|
267
|
+
|
|
268
|
+
iterSpan.setAttributes({
|
|
269
|
+
[NAMZU.ITERATION]: iterationNum,
|
|
270
|
+
[NAMZU.RUN_ID]: runMgr.id,
|
|
271
|
+
[GENAI.REQUEST_MODEL]: model,
|
|
272
|
+
})
|
|
187
273
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
)
|
|
194
|
-
applyLifecycleHookResults('iteration_start', hookResults)
|
|
274
|
+
await this.ctx.emitEvent({
|
|
275
|
+
type: 'iteration_started',
|
|
276
|
+
runId: runMgr.id,
|
|
277
|
+
iteration: iterationNum,
|
|
278
|
+
})
|
|
195
279
|
yield* this.ctx.drainPending()
|
|
196
|
-
}
|
|
197
280
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
281
|
+
if (this.ctx.pluginManager) {
|
|
282
|
+
const hookResults = await this.ctx.pluginManager.executeHooks(
|
|
283
|
+
'iteration_start',
|
|
284
|
+
{ runId: runMgr.id, iteration: iterationNum },
|
|
285
|
+
this.ctx.emitEvent,
|
|
286
|
+
)
|
|
287
|
+
applyLifecycleHookResults('iteration_start', hookResults)
|
|
288
|
+
yield* this.ctx.drainPending()
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Re-pin the working-memory block from ground truth at the primacy
|
|
292
|
+
// edge BEFORE compaction runs (so the refreshed slot is what
|
|
293
|
+
// compaction preserves). No-op when no provider is configured.
|
|
294
|
+
await refreshWorkingMemory(this.ctx)
|
|
295
|
+
await runCompactionCheck(this.ctx)
|
|
296
|
+
|
|
297
|
+
// Cache discipline: keep the tools param byte-stable even on the
|
|
298
|
+
// forced-final iteration and forbid tool use via tool_choice
|
|
299
|
+
// 'none' instead. Dropping the tools array would invalidate the
|
|
300
|
+
// entire prompt-cache prefix (tools render at position 0) and
|
|
301
|
+
// risks a 400 because the history still carries
|
|
302
|
+
// tool_use/tool_result blocks.
|
|
303
|
+
// Snapshot the cumulative counters so the step can report ITS
|
|
304
|
+
// own usage rather than the run total.
|
|
305
|
+
const stepStartedAt = Date.now()
|
|
306
|
+
const usageBefore = { ...runMgr.tokenUsage }
|
|
307
|
+
const costBefore = { ...runMgr.costInfo }
|
|
308
|
+
|
|
309
|
+
// Shape this step before calling the model. `stopWhen` decides
|
|
310
|
+
// whether to keep going; this decides HOW. No-op when the host
|
|
311
|
+
// supplied no hook.
|
|
312
|
+
const step = await this.prepareStep(iterationNum)
|
|
313
|
+
|
|
314
|
+
const stepAllowedTools = step.allowedTools ?? this.ctx.allowedTools
|
|
315
|
+
const llmTools = this.ctx.tools.toLLMTools(stepAllowedTools)
|
|
316
|
+
// The same list the request was built from now also bounds what
|
|
317
|
+
// may run. Narrowing only the request left the restriction
|
|
318
|
+
// presentational — the model was shown fewer tools and could
|
|
319
|
+
// still call any of them by name.
|
|
320
|
+
this.ctx.toolExecutor.setStepAllowedTools(stepAllowedTools)
|
|
321
|
+
const enforceToolInputSchema = enforcedModelInputToolNames(this.ctx.tools, llmTools)
|
|
322
|
+
const stepModel = step.model ?? model
|
|
323
|
+
|
|
324
|
+
const baseMessages = forceFinalize
|
|
325
|
+
? [
|
|
326
|
+
...runMgr.messages,
|
|
327
|
+
createUserMessage(
|
|
328
|
+
'[SYSTEM] You are approaching your resource limits. Provide your final, comprehensive response now based on everything you have gathered so far. Do not request any more tool calls.',
|
|
329
|
+
),
|
|
330
|
+
]
|
|
331
|
+
: runMgr.messages
|
|
332
|
+
|
|
333
|
+
// Step guidance is appended to the REQUEST, never pushed onto
|
|
334
|
+
// the run's history: it applies to this step only, and pushing
|
|
335
|
+
// it would accumulate one stale instruction per iteration.
|
|
336
|
+
// Copy before it crosses the provider boundary. `runMgr.messages`
|
|
337
|
+
// is the LIVE run array, and the loop pushes onto it after the
|
|
338
|
+
// call returns — so a driver that retains what it was handed
|
|
339
|
+
// (to log it, cache it, or replay it on retry) watched its own
|
|
340
|
+
// input grow new turns underneath it. A capture provider in the
|
|
341
|
+
// estate recorded every turn as identical to the last for
|
|
342
|
+
// exactly this reason. Shallow is enough: the defect is array
|
|
343
|
+
// mutation, and per-iteration this is trivial next to the model
|
|
344
|
+
// call it precedes.
|
|
345
|
+
// A step's skills and its guidance ride the same ephemeral
|
|
346
|
+
// trailing system message. Appending leaves the cached prefix
|
|
347
|
+
// intact; rewriting the run's own prompt to carry a phase's
|
|
348
|
+
// skills would invalidate it on every iteration.
|
|
349
|
+
// `renderSkillsSection` already answers null for an empty list, so
|
|
350
|
+
// there is no length check here — a second guard for the same
|
|
351
|
+
// case is one more thing to keep in agreement with the first.
|
|
352
|
+
const stepSkills = step.skills ? renderSkillsSection([...step.skills]) : null
|
|
353
|
+
const stepPreamble = [step.system, stepSkills].filter(Boolean).join('\n\n')
|
|
354
|
+
const messages = stepPreamble
|
|
355
|
+
? [...baseMessages, createSystemMessage(stepPreamble)]
|
|
356
|
+
: [...baseMessages]
|
|
357
|
+
|
|
358
|
+
if (this.ctx.pluginManager) {
|
|
359
|
+
const hookResults = await this.ctx.pluginManager.executeHooks(
|
|
360
|
+
'pre_llm_call',
|
|
361
|
+
{
|
|
362
|
+
runId: runMgr.id,
|
|
363
|
+
iteration: iterationNum,
|
|
364
|
+
// Built inside the guard: a run with no plugins installed
|
|
365
|
+
// pays nothing for a projection nobody reads.
|
|
366
|
+
request: Object.freeze({
|
|
367
|
+
model: stepModel,
|
|
368
|
+
// Copied per turn, not handed over live: these are the
|
|
369
|
+
// run's own message objects, and a hook writing into
|
|
370
|
+
// one would edit the history the run is about to send.
|
|
371
|
+
messages: Object.freeze(messages.map((m) => Object.freeze({ ...m }))),
|
|
372
|
+
toolNames: Object.freeze(llmTools.map((t) => t.function.name)),
|
|
373
|
+
temperature: step.temperature ?? runConfig.temperature,
|
|
374
|
+
maxTokens: step.maxResponseTokens ?? runConfig.maxResponseTokens,
|
|
375
|
+
}),
|
|
376
|
+
},
|
|
377
|
+
this.ctx.emitEvent,
|
|
378
|
+
)
|
|
379
|
+
applyLifecycleHookResults('pre_llm_call', hookResults)
|
|
380
|
+
yield* this.ctx.drainPending()
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// Phase 4 (ses_001-tool-stream-events): consume the
|
|
384
|
+
// streaming response natively, emitting message and
|
|
385
|
+
// tool-input lifecycle events as deltas arrive. The
|
|
386
|
+
// helper yields RunEvents through drainPending() so SSE
|
|
387
|
+
// consumers see live progress; its return value is the
|
|
388
|
+
// aggregated `ChatCompletionResponse` for the legacy
|
|
389
|
+
// downstream paths (assistantMsg construction, working
|
|
390
|
+
// state extraction, telemetry attribute stamping).
|
|
391
|
+
const { response, messageId } = yield* streamProviderTurn(
|
|
392
|
+
this.ctx.provider,
|
|
268
393
|
{
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
394
|
+
model: stepModel,
|
|
395
|
+
messages,
|
|
396
|
+
tools: llmTools.length > 0 ? llmTools : undefined,
|
|
397
|
+
...(enforceToolInputSchema ? { enforceToolInputSchema } : {}),
|
|
398
|
+
// The forced-final turn wins: a step that asked to force a
|
|
399
|
+
// tool cannot override the loop's own decision to stop
|
|
400
|
+
// asking for them. Otherwise the step's choice applies —
|
|
401
|
+
// and only to this step, because the next one is prepared
|
|
402
|
+
// from scratch.
|
|
403
|
+
toolChoice:
|
|
404
|
+
forceFinalize && llmTools.length > 0
|
|
405
|
+
? 'none'
|
|
406
|
+
: llmTools.length > 0
|
|
407
|
+
? step.toolChoice
|
|
408
|
+
: undefined,
|
|
409
|
+
temperature: step.temperature ?? runConfig.temperature,
|
|
410
|
+
maxTokens: step.maxResponseTokens ?? runConfig.maxResponseTokens,
|
|
411
|
+
cacheControl: { type: 'auto' },
|
|
412
|
+
...(runConfig.thinking ? { thinking: runConfig.thinking } : {}),
|
|
413
|
+
...(runConfig.effort ? { effort: runConfig.effort } : {}),
|
|
414
|
+
// Thread the run abort into the model call so a Stop tears the
|
|
415
|
+
// in-flight turn down (provider passes it to fetch; the consumer
|
|
416
|
+
// also races it). Inert when never aborted.
|
|
417
|
+
signal: this.ctx.abortController.signal,
|
|
283
418
|
},
|
|
284
419
|
this.ctx.emitEvent,
|
|
420
|
+
this.ctx.drainPending,
|
|
421
|
+
runMgr.id,
|
|
422
|
+
iterationNum,
|
|
423
|
+
forceFinalize,
|
|
424
|
+
this.ctx.log,
|
|
425
|
+
iterSpan,
|
|
285
426
|
)
|
|
286
|
-
applyLifecycleHookResults('pre_llm_call', hookResults)
|
|
287
|
-
yield* this.ctx.drainPending()
|
|
288
|
-
}
|
|
289
427
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
// tool-input lifecycle events as deltas arrive. The
|
|
293
|
-
// helper yields RunEvents through drainPending() so SSE
|
|
294
|
-
// consumers see live progress; its return value is the
|
|
295
|
-
// aggregated `ChatCompletionResponse` for the legacy
|
|
296
|
-
// downstream paths (assistantMsg construction, working
|
|
297
|
-
// state extraction, telemetry attribute stamping).
|
|
298
|
-
const { response, messageId } = yield* streamProviderTurn(
|
|
299
|
-
this.ctx.provider,
|
|
300
|
-
{
|
|
301
|
-
model: stepModel,
|
|
302
|
-
messages,
|
|
303
|
-
tools: llmTools.length > 0 ? llmTools : undefined,
|
|
304
|
-
...(enforceToolInputSchema ? { enforceToolInputSchema } : {}),
|
|
305
|
-
// The forced-final turn wins: a step that asked to force a
|
|
306
|
-
// tool cannot override the loop's own decision to stop
|
|
307
|
-
// asking for them. Otherwise the step's choice applies —
|
|
308
|
-
// and only to this step, because the next one is prepared
|
|
309
|
-
// from scratch.
|
|
310
|
-
toolChoice:
|
|
311
|
-
forceFinalize && llmTools.length > 0
|
|
312
|
-
? 'none'
|
|
313
|
-
: llmTools.length > 0
|
|
314
|
-
? step.toolChoice
|
|
315
|
-
: undefined,
|
|
316
|
-
temperature: step.temperature ?? runConfig.temperature,
|
|
317
|
-
maxTokens: step.maxResponseTokens ?? runConfig.maxResponseTokens,
|
|
318
|
-
cacheControl: { type: 'auto' },
|
|
319
|
-
...(runConfig.thinking ? { thinking: runConfig.thinking } : {}),
|
|
320
|
-
// Thread the run abort into the model call so a Stop tears the
|
|
321
|
-
// in-flight turn down (provider passes it to fetch; the consumer
|
|
322
|
-
// also races it). Inert when never aborted.
|
|
323
|
-
signal: this.ctx.abortController.signal,
|
|
324
|
-
},
|
|
325
|
-
this.ctx.emitEvent,
|
|
326
|
-
this.ctx.drainPending,
|
|
327
|
-
runMgr.id,
|
|
328
|
-
iterationNum,
|
|
329
|
-
forceFinalize,
|
|
330
|
-
this.ctx.log,
|
|
331
|
-
iterSpan,
|
|
332
|
-
)
|
|
333
|
-
|
|
334
|
-
// Main-loop turn: also records the prompt size compaction reads.
|
|
335
|
-
runMgr.recordTurnUsage(response.usage)
|
|
428
|
+
// Main-loop turn: also records the prompt size compaction reads.
|
|
429
|
+
runMgr.recordTurnUsage(response.usage)
|
|
336
430
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
431
|
+
// The turn went through, so the run is not sitting on an
|
|
432
|
+
// irreducible prompt any more. Re-arm relief for the next one.
|
|
433
|
+
overflowRelieved = false
|
|
340
434
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
435
|
+
if (this.ctx.pluginManager) {
|
|
436
|
+
const hookResults = await this.ctx.pluginManager.executeHooks(
|
|
437
|
+
'post_llm_call',
|
|
438
|
+
{
|
|
439
|
+
runId: runMgr.id,
|
|
440
|
+
iteration: iterationNum,
|
|
441
|
+
response: Object.freeze({
|
|
442
|
+
content: response.message.content,
|
|
443
|
+
toolNames: Object.freeze(
|
|
444
|
+
(response.message.toolCalls ?? []).map((c) => c.function.name),
|
|
445
|
+
),
|
|
446
|
+
finishReason: response.finishReason,
|
|
447
|
+
usage: Object.freeze({ ...response.usage }),
|
|
448
|
+
}),
|
|
449
|
+
},
|
|
450
|
+
this.ctx.emitEvent,
|
|
451
|
+
)
|
|
452
|
+
applyLifecycleHookResults('post_llm_call', hookResults)
|
|
453
|
+
yield* this.ctx.drainPending()
|
|
454
|
+
}
|
|
361
455
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
456
|
+
this.ctx.log.debug('LLM response received', {
|
|
457
|
+
runId: runMgr.id,
|
|
458
|
+
iteration: iterationNum,
|
|
459
|
+
finishReason: response.finishReason,
|
|
460
|
+
hasContent: response.message.content !== null && response.message.content.length > 0,
|
|
461
|
+
toolCallCount: response.message.toolCalls?.length ?? 0,
|
|
462
|
+
promptTokens: response.usage.promptTokens,
|
|
463
|
+
completionTokens: response.usage.completionTokens,
|
|
464
|
+
totalTokens: runMgr.tokenUsage.totalTokens,
|
|
465
|
+
totalCost: runMgr.costInfo.totalCost,
|
|
466
|
+
})
|
|
373
467
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
468
|
+
// The context figures ride with the spend figures because a
|
|
469
|
+
// surface showing one almost always wants the other — and
|
|
470
|
+
// because the two were confusable enough that a host divided
|
|
471
|
+
// cumulative spend by a context window and shipped it. They
|
|
472
|
+
// are measured here rather than left to be derived, since the
|
|
473
|
+
// only correct derivation needs internals a host cannot see.
|
|
474
|
+
//
|
|
475
|
+
// Absent when the run has no compaction config: nothing then
|
|
476
|
+
// resolves a window, and inventing one would be the guess this
|
|
477
|
+
// replaces.
|
|
478
|
+
const contextFigures = this.ctx.compactionConfig
|
|
479
|
+
? (() => {
|
|
480
|
+
const measured = measureContext(this.ctx)
|
|
481
|
+
const window = resolveContextWindow(
|
|
482
|
+
this.ctx.compactionConfig?.contextWindowTokens,
|
|
483
|
+
runConfig.model,
|
|
484
|
+
)
|
|
485
|
+
return {
|
|
486
|
+
contextTokens: measured.tokens,
|
|
487
|
+
contextMeasuredBy: measured.source,
|
|
488
|
+
contextWindowTokens: window.tokens,
|
|
489
|
+
windowSource: window.source,
|
|
490
|
+
}
|
|
491
|
+
})()
|
|
492
|
+
: {}
|
|
380
493
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
response.message.reasoning,
|
|
389
|
-
// Rides with the turn it belongs to, like reasoning does, so
|
|
390
|
-
// trimming or compacting the turn takes its evidence with it
|
|
391
|
-
// rather than leaving citations pointing at prose that is gone.
|
|
392
|
-
response.message.citations,
|
|
393
|
-
)
|
|
394
|
-
runMgr.pushMessage(assistantMsg)
|
|
494
|
+
await this.ctx.emitEvent({
|
|
495
|
+
type: 'token_usage_updated',
|
|
496
|
+
runId: runMgr.id,
|
|
497
|
+
usage: runMgr.tokenUsage,
|
|
498
|
+
cost: runMgr.costInfo,
|
|
499
|
+
...contextFigures,
|
|
500
|
+
})
|
|
395
501
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
502
|
+
// Reasoning rides along with the turn it belongs to, so the
|
|
503
|
+
// replay contract holds automatically: trimming or compacting
|
|
504
|
+
// the assistant message takes its thinking blocks with it,
|
|
505
|
+
// and no separate atomicity rule is needed.
|
|
506
|
+
const assistantMsg = createAssistantMessage(
|
|
507
|
+
response.message.content,
|
|
508
|
+
forceFinalize ? undefined : response.message.toolCalls,
|
|
509
|
+
response.message.reasoning,
|
|
510
|
+
// Rides with the turn it belongs to, like reasoning does, so
|
|
511
|
+
// trimming or compacting the turn takes its evidence with it
|
|
512
|
+
// rather than leaving citations pointing at prose that is gone.
|
|
513
|
+
response.message.citations,
|
|
401
514
|
)
|
|
402
|
-
|
|
515
|
+
runMgr.pushMessage(assistantMsg)
|
|
403
516
|
|
|
404
|
-
|
|
517
|
+
if (this.ctx.workingStateManager && this.ctx.compactionConfig && assistantMsg.content) {
|
|
518
|
+
extractFromAssistantMessage(
|
|
519
|
+
this.ctx.workingStateManager,
|
|
520
|
+
assistantMsg.content,
|
|
521
|
+
this.ctx.compactionConfig,
|
|
522
|
+
)
|
|
523
|
+
}
|
|
405
524
|
|
|
406
|
-
|
|
407
|
-
[GENAI.USAGE_INPUT_TOKENS]: response.usage.promptTokens,
|
|
408
|
-
[GENAI.USAGE_OUTPUT_TOKENS]: response.usage.completionTokens,
|
|
409
|
-
})
|
|
410
|
-
iterSpan.setStatus({ code: SpanStatusCode.OK })
|
|
525
|
+
yield* this.ctx.drainPending()
|
|
411
526
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
hasToolCalls: forceFinalize ? false : !!response.message.toolCalls?.length,
|
|
527
|
+
iterSpan.setAttributes({
|
|
528
|
+
[GENAI.USAGE_INPUT_TOKENS]: response.usage.promptTokens,
|
|
529
|
+
[GENAI.USAGE_OUTPUT_TOKENS]: response.usage.completionTokens,
|
|
416
530
|
})
|
|
417
|
-
|
|
531
|
+
iterSpan.setStatus({ code: SpanStatusCode.OK })
|
|
418
532
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
// `tool_calls`, and three of this repo's drivers pass that
|
|
425
|
-
// value through untouched.
|
|
426
|
-
//
|
|
427
|
-
// Reading `stop` first meant the turn ended with every
|
|
428
|
-
// requested call silently skipped, an assistant message
|
|
429
|
-
// carrying tool_use blocks that were never answered, and a
|
|
430
|
-
// run that settled `end_turn` having done nothing it was
|
|
431
|
-
// asked to do. Checking the calls first costs nothing when
|
|
432
|
-
// the provider is honest and is the only thing that saves the
|
|
433
|
-
// run when it is not.
|
|
434
|
-
const hasToolCalls = (response.message.toolCalls?.length ?? 0) > 0
|
|
435
|
-
|
|
436
|
-
if (forceFinalize || !hasToolCalls) {
|
|
437
|
-
// Every task-dispatch tool (create_task, continue_task, Agent)
|
|
438
|
-
// is BLOCKING: the worker's output returns as the dispatching
|
|
439
|
-
// tool_use's canonical tool_result, so by the time the model
|
|
440
|
-
// ends its turn nothing launched by this run should still be
|
|
441
|
-
// in flight. A running task here is an orphan (interrupted
|
|
442
|
-
// tool execution, cancel race) with no delivery path back to
|
|
443
|
-
// the parent — the <task-notification> producer was removed
|
|
444
|
-
// in dc16d58, so waiting on the queue could only ever time
|
|
445
|
-
// out. Log the orphans honestly and end the turn normally.
|
|
446
|
-
if (!forceFinalize && this.hasRunningAgentTasks()) {
|
|
447
|
-
this.ctx.log.warn(
|
|
448
|
-
'LLM ended turn with agent tasks still running — ending run without waiting (orphan tasks have no delivery path)',
|
|
449
|
-
{
|
|
450
|
-
runId: runMgr.id,
|
|
451
|
-
iteration: iterationNum,
|
|
452
|
-
},
|
|
453
|
-
)
|
|
533
|
+
if (iterationActivity) {
|
|
534
|
+
this.ctx.activityStore.complete(iterationActivity.id, {
|
|
535
|
+
content: response.message.content,
|
|
536
|
+
hasToolCalls: forceFinalize ? false : !!response.message.toolCalls?.length,
|
|
537
|
+
})
|
|
454
538
|
}
|
|
455
539
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
// "continue" user message and let the loop fire
|
|
464
|
-
// another turn. The provider receives the partial
|
|
465
|
-
// assistant content + the continue prompt and
|
|
466
|
-
// resumes from where it left off, mirroring the
|
|
467
|
-
// Auto-continuation after an output-ceiling cutoff.
|
|
540
|
+
// Tool calls beat the finish reason. The reason is the
|
|
541
|
+
// provider's SUMMARY of the turn and the tool calls are the
|
|
542
|
+
// turn itself, so when they disagree the calls are the fact.
|
|
543
|
+
// Several function-calling endpoints — gateways and local servers
|
|
544
|
+
// especially — report `stop` alongside a populated
|
|
545
|
+
// `tool_calls`, and three of this repo's drivers pass that
|
|
546
|
+
// value through untouched.
|
|
468
547
|
//
|
|
469
|
-
//
|
|
470
|
-
//
|
|
471
|
-
//
|
|
472
|
-
//
|
|
473
|
-
//
|
|
474
|
-
//
|
|
475
|
-
//
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
if (
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
548
|
+
// Reading `stop` first meant the turn ended with every
|
|
549
|
+
// requested call silently skipped, an assistant message
|
|
550
|
+
// carrying tool_use blocks that were never answered, and a
|
|
551
|
+
// run that settled `end_turn` having done nothing it was
|
|
552
|
+
// asked to do. Checking the calls first costs nothing when
|
|
553
|
+
// the provider is honest and is the only thing that saves the
|
|
554
|
+
// run when it is not.
|
|
555
|
+
const hasToolCalls = (response.message.toolCalls?.length ?? 0) > 0
|
|
556
|
+
|
|
557
|
+
if (forceFinalize || !hasToolCalls) {
|
|
558
|
+
// Every task-dispatch tool (create_task, continue_task, Agent)
|
|
559
|
+
// is BLOCKING: the worker's output returns as the dispatching
|
|
560
|
+
// tool_use's canonical tool_result, so by the time the model
|
|
561
|
+
// ends its turn nothing launched by this run should still be
|
|
562
|
+
// in flight. A running task here is an orphan (interrupted
|
|
563
|
+
// tool execution, cancel race) with no delivery path back to
|
|
564
|
+
// the parent — the <task-notification> producer was removed
|
|
565
|
+
// in dc16d58, so waiting on the queue could only ever time
|
|
566
|
+
// out. Log the orphans honestly and end the turn normally.
|
|
567
|
+
if (!forceFinalize && this.hasRunningAgentTasks()) {
|
|
568
|
+
this.ctx.log.warn(
|
|
569
|
+
'LLM ended turn with agent tasks still running — ending run without waiting (orphan tasks have no delivery path)',
|
|
570
|
+
{
|
|
571
|
+
runId: runMgr.id,
|
|
572
|
+
iteration: iterationNum,
|
|
573
|
+
},
|
|
574
|
+
)
|
|
575
|
+
}
|
|
494
576
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
577
|
+
const hasContent =
|
|
578
|
+
response.message.content !== null && response.message.content.length > 0
|
|
579
|
+
|
|
580
|
+
// Auto-continuation on `stop_reason: max_tokens`. The
|
|
581
|
+
// model hit its per-call output cap mid-text (NOT
|
|
582
|
+
// mid-tool-use — that path is handled separately
|
|
583
|
+
// below via `inputTruncated`). Push a synthetic
|
|
584
|
+
// "continue" user message and let the loop fire
|
|
585
|
+
// another turn. The provider receives the partial
|
|
586
|
+
// assistant content + the continue prompt and
|
|
587
|
+
// resumes from where it left off, mirroring the
|
|
588
|
+
// Auto-continuation after an output-ceiling cutoff.
|
|
589
|
+
//
|
|
590
|
+
// Guards:
|
|
591
|
+
// - `hasContent` so we don't loop forever on an
|
|
592
|
+
// empty cutoff (a provider occasionally emits
|
|
593
|
+
// `stop_reason: max_tokens` with no content
|
|
594
|
+
// when an injected pre-fill blocks the model).
|
|
595
|
+
// - `!forceFinalize` so the forced-finalize path
|
|
596
|
+
// never auto-continues — that path is invoked
|
|
597
|
+
// specifically to extract a closing summary.
|
|
598
|
+
// - max_iterations bounds the loop in any case.
|
|
599
|
+
if (!forceFinalize && response.finishReason === 'length' && hasContent) {
|
|
600
|
+
this.ctx.log.info('LLM hit max_tokens mid-text — auto-continuing', {
|
|
505
601
|
runId: runMgr.id,
|
|
506
|
-
|
|
602
|
+
iteration: iterationNum,
|
|
603
|
+
completionTokens: response.usage.completionTokens,
|
|
507
604
|
})
|
|
508
|
-
runMgr.
|
|
509
|
-
|
|
605
|
+
runMgr.pushMessage(createUserMessage(AUTO_CONTINUATION_USER_MESSAGE))
|
|
606
|
+
await this.ctx.emitEvent({
|
|
607
|
+
type: 'iteration_completed',
|
|
608
|
+
runId: runMgr.id,
|
|
609
|
+
iteration: iterationNum,
|
|
610
|
+
hasToolCalls: false,
|
|
611
|
+
})
|
|
612
|
+
yield* this.ctx.drainPending()
|
|
613
|
+
continue
|
|
510
614
|
}
|
|
511
|
-
this.ctx.log.info('Re-prompting for structured output', {
|
|
512
|
-
runId: runMgr.id,
|
|
513
|
-
attempt,
|
|
514
|
-
limit,
|
|
515
|
-
})
|
|
516
|
-
runMgr.pushMessage(createUserMessage(STRUCTURED_OUTPUT_REPROMPT))
|
|
517
|
-
await this.ctx.emitEvent({
|
|
518
|
-
type: 'iteration_completed',
|
|
519
|
-
runId: runMgr.id,
|
|
520
|
-
iteration: iterationNum,
|
|
521
|
-
hasToolCalls: false,
|
|
522
|
-
})
|
|
523
|
-
yield* this.ctx.drainPending()
|
|
524
|
-
continue
|
|
525
|
-
}
|
|
526
615
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
//
|
|
536
|
-
// Shaped after the structured-output re-prompt directly
|
|
537
|
-
// above, which solves the same problem for one specific
|
|
538
|
-
// judge: bounded attempts, feedback as a user message, and
|
|
539
|
-
// a loud stop rather than a loop.
|
|
540
|
-
if (!forceFinalize && this.ctx.reviewAnswer) {
|
|
541
|
-
const review = await this.reviewAnswer(response.message.content ?? '')
|
|
542
|
-
if (review && !review.accept) {
|
|
543
|
-
const attempt = ++this.answerReviewAttempts
|
|
544
|
-
const limit = this.ctx.maxAnswerReviews ?? DEFAULT_ANSWER_REVIEW_LIMIT
|
|
616
|
+
// The model tried to finish in prose while a structured
|
|
617
|
+
// output was demanded. Send it back with the schema error
|
|
618
|
+
// rather than returning an unusable result — this is the
|
|
619
|
+
// re-prompt half, and it is bounded so a model that cannot
|
|
620
|
+
// satisfy the schema fails loudly instead of looping.
|
|
621
|
+
if (!forceFinalize && this.needsStructuredOutput()) {
|
|
622
|
+
const attempt = ++this.structuredOutputAttempts
|
|
623
|
+
const limit = this.structuredOutputRetryLimit()
|
|
545
624
|
if (attempt > limit) {
|
|
546
|
-
this.ctx.log.warn('
|
|
625
|
+
this.ctx.log.warn('Structured output not produced within its retries', {
|
|
547
626
|
runId: runMgr.id,
|
|
548
627
|
attempts: attempt - 1,
|
|
549
|
-
limit,
|
|
550
628
|
})
|
|
551
|
-
runMgr.setStopReason('
|
|
629
|
+
runMgr.setStopReason('structured_output_failed')
|
|
552
630
|
break
|
|
553
631
|
}
|
|
554
|
-
this.ctx.log.info('
|
|
632
|
+
this.ctx.log.info('Re-prompting for structured output', {
|
|
555
633
|
runId: runMgr.id,
|
|
556
634
|
attempt,
|
|
557
635
|
limit,
|
|
558
636
|
})
|
|
559
|
-
runMgr.pushMessage(createUserMessage(
|
|
637
|
+
runMgr.pushMessage(createUserMessage(STRUCTURED_OUTPUT_REPROMPT))
|
|
560
638
|
await this.ctx.emitEvent({
|
|
561
639
|
type: 'iteration_completed',
|
|
562
640
|
runId: runMgr.id,
|
|
@@ -566,146 +644,261 @@ export class IterationOrchestrator {
|
|
|
566
644
|
yield* this.ctx.drainPending()
|
|
567
645
|
continue
|
|
568
646
|
}
|
|
569
|
-
}
|
|
570
647
|
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
648
|
+
// Let the host judge the ANSWER and hand back work.
|
|
649
|
+
//
|
|
650
|
+
// The stop predicate is only consulted after tools ran, so
|
|
651
|
+
// there was no seam here at all: the moment the model
|
|
652
|
+
// stopped calling tools the run finalized, whatever it had
|
|
653
|
+
// produced. Verify-then-fix — run the build, feed the
|
|
654
|
+
// failure back, let it try again — meant starting a whole
|
|
655
|
+
// new run and re-supplying the context the first one had.
|
|
656
|
+
//
|
|
657
|
+
// Shaped after the structured-output re-prompt directly
|
|
658
|
+
// above, which solves the same problem for one specific
|
|
659
|
+
// judge: bounded attempts, feedback as a user message, and
|
|
660
|
+
// a loud stop rather than a loop.
|
|
661
|
+
if (!forceFinalize && this.ctx.reviewAnswer) {
|
|
662
|
+
const review = await this.reviewAnswer(response.message.content ?? '')
|
|
663
|
+
if (review && !review.accept) {
|
|
664
|
+
const attempt = ++this.answerReviewAttempts
|
|
665
|
+
const limit = this.ctx.maxAnswerReviews ?? DEFAULT_ANSWER_REVIEW_LIMIT
|
|
666
|
+
if (attempt > limit) {
|
|
667
|
+
this.ctx.log.warn('Answer rejected more times than the run allows', {
|
|
668
|
+
runId: runMgr.id,
|
|
669
|
+
attempts: attempt - 1,
|
|
670
|
+
limit,
|
|
671
|
+
})
|
|
672
|
+
runMgr.setStopReason('answer_rejected')
|
|
673
|
+
break
|
|
674
|
+
}
|
|
675
|
+
this.ctx.log.info('Answer rejected — returning it to the model', {
|
|
676
|
+
runId: runMgr.id,
|
|
677
|
+
attempt,
|
|
678
|
+
limit,
|
|
679
|
+
})
|
|
680
|
+
runMgr.pushMessage(createUserMessage(review.feedback))
|
|
681
|
+
await this.ctx.emitEvent({
|
|
682
|
+
type: 'iteration_completed',
|
|
683
|
+
runId: runMgr.id,
|
|
684
|
+
iteration: iterationNum,
|
|
685
|
+
hasToolCalls: false,
|
|
686
|
+
})
|
|
687
|
+
yield* this.ctx.drainPending()
|
|
688
|
+
continue
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// A background worker is still out there, and this turn was
|
|
693
|
+
// about to end the run.
|
|
694
|
+
//
|
|
695
|
+
// Settling here would throw away the very thing the launch
|
|
696
|
+
// existed to produce: the supervisor said "launched", the
|
|
697
|
+
// worker had not finished, and the run closed over it.
|
|
698
|
+
if (!forceFinalize && (yield* this.holdForOutstandingWork(iterationNum, false))) {
|
|
699
|
+
continue
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
if (!hasContent && !forceFinalize) {
|
|
703
|
+
this.ctx.log.warn('Empty completion detected — requesting final summary', {
|
|
593
704
|
iteration: iterationNum,
|
|
594
|
-
|
|
705
|
+
finishReason: response.finishReason,
|
|
595
706
|
})
|
|
707
|
+
await this.requestFinalResponse(model, 'end_turn')
|
|
596
708
|
yield* this.ctx.drainPending()
|
|
597
|
-
continue
|
|
598
709
|
}
|
|
599
|
-
}
|
|
600
710
|
|
|
601
|
-
|
|
602
|
-
|
|
711
|
+
await this.ctx.emitEvent({
|
|
712
|
+
type: 'iteration_completed',
|
|
713
|
+
runId: runMgr.id,
|
|
603
714
|
iteration: iterationNum,
|
|
604
|
-
|
|
715
|
+
hasToolCalls: false,
|
|
605
716
|
})
|
|
606
|
-
await this.requestFinalResponse(model, 'end_turn')
|
|
607
717
|
yield* this.ctx.drainPending()
|
|
718
|
+
// A Stop that lands AFTER the final turn streamed but before
|
|
719
|
+
// this break must settle the run as cancelled, not end_turn —
|
|
720
|
+
// otherwise the just-produced answer is recorded as a clean
|
|
721
|
+
// completion. Mirrors the between-iteration cancel at :511.
|
|
722
|
+
if (this.ctx.abortController.signal.aborted) {
|
|
723
|
+
runMgr.setStopReason('cancelled')
|
|
724
|
+
runMgr.markCancelled()
|
|
725
|
+
break
|
|
726
|
+
}
|
|
727
|
+
// The host's stop predicate, if the previous turn deferred it
|
|
728
|
+
// to let the model read a delegated result. That extra turn
|
|
729
|
+
// is prose, and `stopWhen` is consulted only after a tool
|
|
730
|
+
// batch, so the predicate is never asked again — reporting
|
|
731
|
+
// `end_turn` would name the shape of the last message rather
|
|
732
|
+
// than the reason the run is over.
|
|
733
|
+
//
|
|
734
|
+
// Only here. A terminal tool and a captured structured output
|
|
735
|
+
// also settle as `end_turn`, and there the deferred predicate
|
|
736
|
+
// is not why the run ended: those decided the answer
|
|
737
|
+
// themselves.
|
|
738
|
+
runMgr.setStopReason(stopWasDeferredForOutstandingWork ? 'stop_condition' : 'end_turn')
|
|
739
|
+
break
|
|
608
740
|
}
|
|
609
741
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
742
|
+
const reviewOutcome = yield* runToolReview(this.ctx, response, iterationNum)
|
|
743
|
+
|
|
744
|
+
// The step record is built even for a rejected batch: a run that
|
|
745
|
+
// spent a turn getting its tools refused still spent the tokens,
|
|
746
|
+
// and a caller reconstructing cost per step must see it.
|
|
747
|
+
this.recordStep({
|
|
748
|
+
stepNumber: iterationNum,
|
|
749
|
+
model,
|
|
750
|
+
messageId,
|
|
751
|
+
response,
|
|
752
|
+
toolResults: reviewOutcome.results,
|
|
753
|
+
toolExecutionMs: reviewOutcome.durationMs,
|
|
754
|
+
startedAt: stepStartedAt,
|
|
755
|
+
usageBefore,
|
|
756
|
+
costBefore,
|
|
615
757
|
})
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
if (
|
|
622
|
-
|
|
623
|
-
|
|
758
|
+
|
|
759
|
+
if (reviewOutcome.decision === 'stop') {
|
|
760
|
+
return
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (reviewOutcome.decision === 'rejected') {
|
|
764
|
+
continue
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
// A successful `structured_output` call IS the answer, so the
|
|
768
|
+
// run ends here rather than paying for another turn whose only
|
|
769
|
+
// job would be to restate it.
|
|
770
|
+
if (this.captureStructuredOutput(reviewOutcome.results)) {
|
|
771
|
+
this.ctx.log.info('Structured output produced — ending run', {
|
|
772
|
+
runId: runMgr.id,
|
|
773
|
+
iteration: iterationNum,
|
|
774
|
+
})
|
|
775
|
+
runMgr.setStopReason('end_turn')
|
|
776
|
+
await this.ctx.emitEvent({
|
|
777
|
+
type: 'iteration_completed',
|
|
778
|
+
runId: runMgr.id,
|
|
779
|
+
iteration: iterationNum,
|
|
780
|
+
hasToolCalls: true,
|
|
781
|
+
})
|
|
782
|
+
yield* this.ctx.drainPending()
|
|
624
783
|
break
|
|
625
784
|
}
|
|
626
|
-
runMgr.setStopReason('end_turn')
|
|
627
|
-
break
|
|
628
|
-
}
|
|
629
785
|
|
|
630
|
-
|
|
786
|
+
// A tool the author declared terminal settles the run with its
|
|
787
|
+
// own output, the same rule `structured_output` has always
|
|
788
|
+
// had. Without it a delegation cost the parent one more model
|
|
789
|
+
// call at full context whose only job was to restate what the
|
|
790
|
+
// worker already said — and to restate it through the parent's
|
|
791
|
+
// compacted view, so the caller did not even receive the
|
|
792
|
+
// worker's words.
|
|
793
|
+
const settled = this.terminalToolOutput(reviewOutcome.results, response)
|
|
794
|
+
if (settled !== undefined) {
|
|
795
|
+
this.ctx.log.info('Terminal tool produced the answer — ending run', {
|
|
796
|
+
runId: runMgr.id,
|
|
797
|
+
iteration: iterationNum,
|
|
798
|
+
tool: settled.toolName,
|
|
799
|
+
})
|
|
800
|
+
runMgr.setResult(settled.output)
|
|
801
|
+
runMgr.setStopReason('end_turn')
|
|
802
|
+
await this.ctx.emitEvent({
|
|
803
|
+
type: 'iteration_completed',
|
|
804
|
+
runId: runMgr.id,
|
|
805
|
+
iteration: iterationNum,
|
|
806
|
+
hasToolCalls: true,
|
|
807
|
+
})
|
|
808
|
+
yield* this.ctx.drainPending()
|
|
809
|
+
break
|
|
810
|
+
}
|
|
631
811
|
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
812
|
+
// Evaluated AFTER the tools ran, so a predicate can see what they
|
|
813
|
+
// returned — which is what makes a terminal submit_answer tool
|
|
814
|
+
// usable without discarding its output.
|
|
815
|
+
if (await this.shouldStop()) {
|
|
816
|
+
// Outstanding delegated work outranks the host's stop
|
|
817
|
+
// predicate, exactly once.
|
|
818
|
+
//
|
|
819
|
+
// This is a precedence rule chosen here, not something
|
|
820
|
+
// `stopWhen` implies — a stop predicate is a programmable
|
|
821
|
+
// halt and says nothing about whether the answer is
|
|
822
|
+
// complete, which is what separates it from a terminal
|
|
823
|
+
// tool or a captured structured output. Those decide the
|
|
824
|
+
// result, so no turn follows and a hold would buy nothing.
|
|
825
|
+
// This one only says "stop", and stopping one turn later
|
|
826
|
+
// with the worker's result in hand is a better reading of
|
|
827
|
+
// the host's intent than stopping now and discarding it.
|
|
828
|
+
//
|
|
829
|
+
// Bounded: after the notification is delivered the inbox
|
|
830
|
+
// is drained, so the predicate fires again next turn with
|
|
831
|
+
// nothing pending and the run stops. Exactly one extra
|
|
832
|
+
// turn, and `maxIterations` bounds it regardless.
|
|
833
|
+
if (yield* this.holdForOutstandingWork(iterationNum, true)) {
|
|
834
|
+
// Remember WHY the next turn exists, so the turn that
|
|
835
|
+
// ends the run can name the host's decision instead of
|
|
836
|
+
// reporting the shape of the last message.
|
|
837
|
+
this.stopDeferredForOutstandingWork = true
|
|
838
|
+
continue
|
|
839
|
+
}
|
|
646
840
|
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
841
|
+
this.ctx.log.info('Stop condition met', {
|
|
842
|
+
runId: runMgr.id,
|
|
843
|
+
iteration: iterationNum,
|
|
844
|
+
})
|
|
845
|
+
runMgr.setStopReason('stop_condition')
|
|
846
|
+
await this.ctx.emitEvent({
|
|
847
|
+
type: 'iteration_completed',
|
|
848
|
+
runId: runMgr.id,
|
|
849
|
+
iteration: iterationNum,
|
|
850
|
+
hasToolCalls: true,
|
|
851
|
+
})
|
|
852
|
+
yield* this.ctx.drainPending()
|
|
853
|
+
break
|
|
854
|
+
}
|
|
650
855
|
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
856
|
+
const checkpointSignal = yield* runIterationCheckpoint(this.ctx, iterationNum)
|
|
857
|
+
if (checkpointSignal === 'stop') {
|
|
858
|
+
return
|
|
859
|
+
}
|
|
654
860
|
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
861
|
+
// Workers that finished with nobody listening.
|
|
862
|
+
//
|
|
863
|
+
// A completion normally reaches the supervisor as the
|
|
864
|
+
// `tool_result` of the `create_task` that launched it. Two
|
|
865
|
+
// cases have no such call: a launch made in the background on
|
|
866
|
+
// purpose, and a blocking launch whose deadline passed — the
|
|
867
|
+
// model was told "timed out, it may still be running" and the
|
|
868
|
+
// worker then finished, holding a result nothing would read.
|
|
869
|
+
//
|
|
870
|
+
// This is the channel that was removed in `dc16d58` because it
|
|
871
|
+
// double-delivered: it fired for completions the blocking tool
|
|
872
|
+
// had already handed over, so the supervisor saw each result
|
|
873
|
+
// twice. The inbox restores it with the distinction that was
|
|
874
|
+
// missing — a tool that delivers a completion claims it, and
|
|
875
|
+
// only unclaimed ones arrive here.
|
|
876
|
+
//
|
|
877
|
+
// Placed beside the advisory phase deliberately: that is the
|
|
878
|
+
// established seam for putting a user message in after tool
|
|
879
|
+
// results and before the next turn.
|
|
880
|
+
const unheard = this.ctx.completionInbox?.drain() ?? []
|
|
881
|
+
if (unheard.length > 0) {
|
|
882
|
+
this.ctx.log.info('Delivering unawaited task completions', {
|
|
883
|
+
runId: runMgr.id,
|
|
884
|
+
iteration: iterationNum,
|
|
885
|
+
tasks: unheard.map((h) => h.taskId),
|
|
886
|
+
})
|
|
887
|
+
runMgr.pushMessage(createUserMessage(formatCompletionNotification(unheard)))
|
|
888
|
+
}
|
|
673
889
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
iteration: iterationNum,
|
|
686
|
-
tool: settled.toolName,
|
|
687
|
-
})
|
|
688
|
-
runMgr.setResult(settled.output)
|
|
689
|
-
runMgr.setStopReason('end_turn')
|
|
690
|
-
await this.ctx.emitEvent({
|
|
691
|
-
type: 'iteration_completed',
|
|
692
|
-
runId: runMgr.id,
|
|
693
|
-
iteration: iterationNum,
|
|
694
|
-
hasToolCalls: true,
|
|
695
|
-
})
|
|
696
|
-
yield* this.ctx.drainPending()
|
|
697
|
-
break
|
|
698
|
-
}
|
|
890
|
+
await runAdvisoryPhase(this.ctx, iterationNum, response)
|
|
891
|
+
|
|
892
|
+
if (this.ctx.pluginManager) {
|
|
893
|
+
const hookResults = await this.ctx.pluginManager.executeHooks(
|
|
894
|
+
'iteration_end',
|
|
895
|
+
{ runId: runMgr.id, iteration: iterationNum },
|
|
896
|
+
this.ctx.emitEvent,
|
|
897
|
+
)
|
|
898
|
+
applyLifecycleHookResults('iteration_end', hookResults)
|
|
899
|
+
yield* this.ctx.drainPending()
|
|
900
|
+
}
|
|
699
901
|
|
|
700
|
-
// Evaluated AFTER the tools ran, so a predicate can see what they
|
|
701
|
-
// returned — which is what makes a terminal submit_answer tool
|
|
702
|
-
// usable without discarding its output.
|
|
703
|
-
if (await this.shouldStop()) {
|
|
704
|
-
this.ctx.log.info('Stop condition met', {
|
|
705
|
-
runId: runMgr.id,
|
|
706
|
-
iteration: iterationNum,
|
|
707
|
-
})
|
|
708
|
-
runMgr.setStopReason('stop_condition')
|
|
709
902
|
await this.ctx.emitEvent({
|
|
710
903
|
type: 'iteration_completed',
|
|
711
904
|
runId: runMgr.id,
|
|
@@ -713,124 +906,184 @@ export class IterationOrchestrator {
|
|
|
713
906
|
hasToolCalls: true,
|
|
714
907
|
})
|
|
715
908
|
yield* this.ctx.drainPending()
|
|
716
|
-
|
|
717
|
-
|
|
909
|
+
} catch (err) {
|
|
910
|
+
// A Stop that aborted the in-flight turn surfaces here as a
|
|
911
|
+
// thrown abort (the provider stream was raced against the run
|
|
912
|
+
// signal). Settle it as a CANCELLATION — mirroring the
|
|
913
|
+
// between-iteration cancel at the top of the loop — rather than
|
|
914
|
+
// recording it as an SDK failure (error span + failed activity)
|
|
915
|
+
// and re-throwing. The run then returns cleanly with a
|
|
916
|
+
// 'cancelled' stop reason instead of propagating an error.
|
|
917
|
+
if (this.ctx.abortController.signal.aborted) {
|
|
918
|
+
runMgr.setStopReason('cancelled')
|
|
919
|
+
runMgr.markCancelled()
|
|
920
|
+
break
|
|
921
|
+
}
|
|
718
922
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
923
|
+
// The one provider failure the kernel can actually do something
|
|
924
|
+
// about. `context_length_exceeded` is correctly non-retryable —
|
|
925
|
+
// resending the identical prompt cannot help — but the kernel
|
|
926
|
+
// owns a compaction subsystem that can make the prompt smaller.
|
|
927
|
+
// Without this the run died holding the remedy: the threshold
|
|
928
|
+
// path had simply guessed low, which a run carrying images or a
|
|
929
|
+
// language the chars-per-token ratio does not fit will do.
|
|
930
|
+
//
|
|
931
|
+
// Relief is attempted ONCE per iteration and only when it
|
|
932
|
+
// actually shed something. A second overflow after a successful
|
|
933
|
+
// compaction means the prompt is irreducible, and looping on it
|
|
934
|
+
// would burn the budget to arrive at the same error.
|
|
935
|
+
if (
|
|
936
|
+
!overflowRelieved &&
|
|
937
|
+
classifyProviderError(err, this.ctx.provider.id).code === 'context_length_exceeded'
|
|
938
|
+
) {
|
|
939
|
+
overflowRelieved = true
|
|
940
|
+
const shed = await relieveOverflow(this.ctx)
|
|
941
|
+
if (shed) {
|
|
942
|
+
this.ctx.log.info('Retrying the turn after relieving a context overflow', {
|
|
943
|
+
runId: runMgr.id,
|
|
944
|
+
iteration: iterationNum,
|
|
945
|
+
})
|
|
946
|
+
if (iterationActivity) {
|
|
947
|
+
this.ctx.activityStore.complete(iterationActivity.id)
|
|
948
|
+
}
|
|
949
|
+
continue
|
|
950
|
+
}
|
|
951
|
+
}
|
|
723
952
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
// worker then finished, holding a result nothing would read.
|
|
732
|
-
//
|
|
733
|
-
// This is the channel that was removed in `dc16d58` because it
|
|
734
|
-
// double-delivered: it fired for completions the blocking tool
|
|
735
|
-
// had already handed over, so the supervisor saw each result
|
|
736
|
-
// twice. The inbox restores it with the distinction that was
|
|
737
|
-
// missing — a tool that delivers a completion claims it, and
|
|
738
|
-
// only unclaimed ones arrive here.
|
|
739
|
-
//
|
|
740
|
-
// Placed beside the advisory phase deliberately: that is the
|
|
741
|
-
// established seam for putting a user message in after tool
|
|
742
|
-
// results and before the next turn.
|
|
743
|
-
const unheard = this.ctx.completionInbox?.drain() ?? []
|
|
744
|
-
if (unheard.length > 0) {
|
|
745
|
-
this.ctx.log.info('Delivering unawaited task completions', {
|
|
746
|
-
runId: runMgr.id,
|
|
747
|
-
iteration: iterationNum,
|
|
748
|
-
tasks: unheard.map((h) => h.taskId),
|
|
953
|
+
if (iterationActivity) {
|
|
954
|
+
this.ctx.activityStore.fail(iterationActivity.id, toErrorMessage(err))
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
iterSpan.setStatus({
|
|
958
|
+
code: SpanStatusCode.ERROR,
|
|
959
|
+
message: toErrorMessage(err),
|
|
749
960
|
})
|
|
750
|
-
|
|
961
|
+
iterSpan.recordException(err instanceof Error ? err : new Error(String(err)))
|
|
962
|
+
throw err
|
|
963
|
+
} finally {
|
|
964
|
+
// The only place the iteration span ends. It used to be ended at each of
|
|
965
|
+
// seventeen exits, which is a rule every future edit has to
|
|
966
|
+
// remember; a generator abandoned by its consumer never reached
|
|
967
|
+
// any of them.
|
|
968
|
+
iterSpan.end()
|
|
751
969
|
}
|
|
970
|
+
}
|
|
971
|
+
} finally {
|
|
972
|
+
this.settleOutstandingWork()
|
|
973
|
+
}
|
|
974
|
+
}
|
|
752
975
|
|
|
753
|
-
|
|
976
|
+
/**
|
|
977
|
+
* Hold the run open for a worker that has not finished, and deliver it.
|
|
978
|
+
*
|
|
979
|
+
* Returns whether a completion arrived and was put in the transcript — the
|
|
980
|
+
* caller continues the loop on `true`, so the model gets a turn in which to
|
|
981
|
+
* USE the result. That turn is the entire justification for waiting, which
|
|
982
|
+
* is why only the exits that can still take one call this.
|
|
983
|
+
*
|
|
984
|
+
* Bounded by `settleGraceMs` and by `maxIterations`, so a worker that never
|
|
985
|
+
* finishes cannot keep the run open.
|
|
986
|
+
*/
|
|
987
|
+
private async *holdForOutstandingWork(
|
|
988
|
+
iterationNum: number,
|
|
989
|
+
hasToolCalls: boolean,
|
|
990
|
+
): AsyncGenerator<RunEvent, boolean> {
|
|
991
|
+
if (!this.ctx.completionInbox?.hasPendingWork) return false
|
|
992
|
+
|
|
993
|
+
// Read HERE rather than from `forceFinalize`, which was sampled at the
|
|
994
|
+
// top of the iteration: one that has since crossed the finalize point
|
|
995
|
+
// must not open a wait against a reserve it has already entered.
|
|
996
|
+
const graceMs = settleGraceMs(this.ctx.guard.remainingBeforeFinalizeMs())
|
|
997
|
+
this.ctx.log.info('Holding the run open for a background task', {
|
|
998
|
+
runId: this.ctx.runMgr.id,
|
|
999
|
+
iteration: iterationNum,
|
|
1000
|
+
graceMs,
|
|
1001
|
+
})
|
|
1002
|
+
await this.ctx.completionInbox.waitForArrival(graceMs)
|
|
754
1003
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
'iteration_end',
|
|
758
|
-
{ runId: runMgr.id, iteration: iterationNum },
|
|
759
|
-
this.ctx.emitEvent,
|
|
760
|
-
)
|
|
761
|
-
applyLifecycleHookResults('iteration_end', hookResults)
|
|
762
|
-
yield* this.ctx.drainPending()
|
|
763
|
-
}
|
|
1004
|
+
const arrived = this.ctx.completionInbox.drain()
|
|
1005
|
+
if (arrived.length === 0) return false
|
|
764
1006
|
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
// signal). Settle it as a CANCELLATION — mirroring the
|
|
776
|
-
// between-iteration cancel at the top of the loop — rather than
|
|
777
|
-
// recording it as an SDK failure (error span + failed activity)
|
|
778
|
-
// and re-throwing. The run then returns cleanly with a
|
|
779
|
-
// 'cancelled' stop reason instead of propagating an error.
|
|
780
|
-
if (this.ctx.abortController.signal.aborted) {
|
|
781
|
-
runMgr.setStopReason('cancelled')
|
|
782
|
-
runMgr.markCancelled()
|
|
783
|
-
break
|
|
784
|
-
}
|
|
1007
|
+
this.ctx.runMgr.pushMessage(createUserMessage(formatCompletionNotification(arrived)))
|
|
1008
|
+
await this.ctx.emitEvent({
|
|
1009
|
+
type: 'iteration_completed',
|
|
1010
|
+
runId: this.ctx.runMgr.id,
|
|
1011
|
+
iteration: iterationNum,
|
|
1012
|
+
hasToolCalls,
|
|
1013
|
+
})
|
|
1014
|
+
yield* this.ctx.drainPending()
|
|
1015
|
+
return true
|
|
1016
|
+
}
|
|
785
1017
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
}
|
|
1018
|
+
/**
|
|
1019
|
+
* Account for delegated work on the way out: deliver what arrived, and say
|
|
1020
|
+
* what did not.
|
|
1021
|
+
*
|
|
1022
|
+
* A run that ends with a worker outstanding must not leave the impression
|
|
1023
|
+
* that the worker's result was delivered. There are exactly two honest
|
|
1024
|
+
* outcomes and this does both:
|
|
1025
|
+
*
|
|
1026
|
+
* - **What has already arrived is delivered.** It makes no false claim,
|
|
1027
|
+
* and dropping it is pure loss — the message rides out on
|
|
1028
|
+
* `Run.messages`, so a host reads it and the next turn of a continued
|
|
1029
|
+
* thread starts with it. This does NOT wait: a hold buys the model a
|
|
1030
|
+
* turn in which to USE a result, and on an exit whose answer is already
|
|
1031
|
+
* decided there is no such turn, so waiting would delay a settled answer
|
|
1032
|
+
* to append text this run will not read. The bounded hold stays where it
|
|
1033
|
+
* was, on the exits that do have a turn left.
|
|
1034
|
+
* - **What is still running is NAMED, not cancelled.** Giving up on a wait
|
|
1035
|
+
* is a statement about the waiter, not about the work — the rule
|
|
1036
|
+
* `wait-with-idle-bound.ts` already states for the same subsystem — and
|
|
1037
|
+
* "the parent answered early" is a weaker warrant for killing a child
|
|
1038
|
+
* than "the clock ran out", not a stronger one. Killing a worker that
|
|
1039
|
+
* may be mid-write is a policy only the host can judge, and it has
|
|
1040
|
+
* `cancel_task` and the run controller to judge it with.
|
|
1041
|
+
*/
|
|
1042
|
+
private settleOutstandingWork(): void {
|
|
1043
|
+
this.deliverArrivedCompletions()
|
|
1044
|
+
this.recordAbandonedWork()
|
|
1045
|
+
}
|
|
815
1046
|
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
1047
|
+
/** Delegated work this run walked away from. See {@link settleOutstandingWork}. */
|
|
1048
|
+
private recordAbandonedWork(): void {
|
|
1049
|
+
const abandoned = this.ctx.completionInbox?.outstandingTaskIds ?? []
|
|
1050
|
+
if (abandoned.length === 0) return
|
|
819
1051
|
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
1052
|
+
this.ctx.log.warn('Run ended with delegated work still running', {
|
|
1053
|
+
runId: this.ctx.runMgr.id,
|
|
1054
|
+
tasks: abandoned,
|
|
1055
|
+
})
|
|
1056
|
+
this.ctx.runMgr.setAbandonedTaskIds(abandoned)
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
private deliverArrivedCompletions(): void {
|
|
1060
|
+
const unheard = this.ctx.completionInbox?.drain() ?? []
|
|
1061
|
+
if (unheard.length === 0) return
|
|
1062
|
+
|
|
1063
|
+
// Fix the run's answer BEFORE appending anything after it.
|
|
1064
|
+
//
|
|
1065
|
+
// `RunPersistence.resolveResult` walks the message tail backwards and
|
|
1066
|
+
// stops at the first non-assistant message, and it runs at
|
|
1067
|
+
// `markCompleted` — which is AFTER this. So a notification appended
|
|
1068
|
+
// after the final assistant turn makes the run's own answer
|
|
1069
|
+
// unreachable. Measured, on a run whose model had just said "THIS IS
|
|
1070
|
+
// THE RUN ANSWER.": `run.result` came back `undefined`. That trades a
|
|
1071
|
+
// lost worker result for a lost RUN result, which is strictly worse
|
|
1072
|
+
// than the defect this delivery exists to fix.
|
|
1073
|
+
//
|
|
1074
|
+
// Materialising resolves it while the tail is still the assistant's;
|
|
1075
|
+
// pinning it means the later re-resolution cannot undo the fix. Only
|
|
1076
|
+
// when there is something to pin: on the cancelled and thrown paths
|
|
1077
|
+
// there may be no answer, and pinning an empty string there would
|
|
1078
|
+
// suppress whatever the error path assembles.
|
|
1079
|
+
const answer = this.ctx.runMgr.materializeResult()
|
|
1080
|
+
if (answer.length > 0) this.ctx.runMgr.setResult(answer)
|
|
1081
|
+
|
|
1082
|
+
this.ctx.log.info('Delivering task completions the run would have settled over', {
|
|
1083
|
+
runId: this.ctx.runMgr.id,
|
|
1084
|
+
tasks: unheard.map((h) => h.taskId),
|
|
1085
|
+
})
|
|
1086
|
+
this.ctx.runMgr.pushMessage(createUserMessage(formatCompletionNotification(unheard)))
|
|
834
1087
|
}
|
|
835
1088
|
|
|
836
1089
|
/**
|
|
@@ -891,15 +1144,29 @@ export class IterationOrchestrator {
|
|
|
891
1144
|
} = {}
|
|
892
1145
|
|
|
893
1146
|
if (result.activeTools) {
|
|
894
|
-
// A phase list that outlives a tool rename should narrow the
|
|
895
|
-
// surface, not kill the agent mid-run.
|
|
896
1147
|
const known = result.activeTools.filter((name: string) => this.ctx.tools.has(name))
|
|
897
1148
|
const unknown = result.activeTools.filter((name: string) => !this.ctx.tools.has(name))
|
|
898
1149
|
if (unknown.length > 0) {
|
|
899
|
-
|
|
1150
|
+
// The all-unknown case gets its own sentence because it has its
|
|
1151
|
+
// own consequence. Some names dropped narrows the step; ALL of
|
|
1152
|
+
// them dropped leaves it able to call nothing — which is the
|
|
1153
|
+
// honest reading of "only these tools" when none of them exist,
|
|
1154
|
+
// and is not what a reader of "ignoring them" would expect.
|
|
1155
|
+
//
|
|
1156
|
+
// Widening back to the run's list would be worse: it grants
|
|
1157
|
+
// exactly the tools the caller asked to exclude, on the grounds
|
|
1158
|
+
// that their own list failed. A step that can call nothing is
|
|
1159
|
+
// constrained; a step that can call everything is a control
|
|
1160
|
+
// that stopped applying.
|
|
1161
|
+
const message =
|
|
1162
|
+
known.length === 0
|
|
1163
|
+
? 'prepareStep named only tools that are not registered — this step can call nothing'
|
|
1164
|
+
: 'prepareStep named tools that are not registered — ignoring them'
|
|
1165
|
+
this.ctx.log.warn(message, {
|
|
900
1166
|
runId: this.ctx.runMgr.id,
|
|
901
1167
|
stepNumber,
|
|
902
1168
|
unknown,
|
|
1169
|
+
remaining: known.length,
|
|
903
1170
|
})
|
|
904
1171
|
}
|
|
905
1172
|
prepared.allowedTools = known
|
|
@@ -1154,6 +1421,10 @@ export class IterationOrchestrator {
|
|
|
1154
1421
|
maxTokens: this.ctx.runConfig.maxResponseTokens,
|
|
1155
1422
|
cacheControl: { type: 'auto' },
|
|
1156
1423
|
...(this.ctx.runConfig.thinking ? { thinking: this.ctx.runConfig.thinking } : {}),
|
|
1424
|
+
// This turn is a hand-maintained duplicate of the one above, which
|
|
1425
|
+
// is exactly the shape a field goes missing from — so it is tested
|
|
1426
|
+
// separately rather than assumed to have been kept in step.
|
|
1427
|
+
...(this.ctx.runConfig.effort ? { effort: this.ctx.runConfig.effort } : {}),
|
|
1157
1428
|
// Cancellable too: a Stop during the closing summary must not
|
|
1158
1429
|
// stream to completion.
|
|
1159
1430
|
signal: this.ctx.abortController.signal,
|