@namzu/sdk 22.1.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +116 -0
  2. package/dist/agents/ReactiveAgent.d.ts.map +1 -1
  3. package/dist/agents/ReactiveAgent.js +3 -0
  4. package/dist/agents/ReactiveAgent.js.map +1 -1
  5. package/dist/compaction/context-window.d.ts +15 -0
  6. package/dist/compaction/context-window.d.ts.map +1 -1
  7. package/dist/compaction/context-window.js +28 -3
  8. package/dist/compaction/context-window.js.map +1 -1
  9. package/dist/provider/retry.d.ts +10 -5
  10. package/dist/provider/retry.d.ts.map +1 -1
  11. package/dist/provider/retry.js +1 -29
  12. package/dist/provider/retry.js.map +1 -1
  13. package/dist/public-runtime.d.ts +1 -0
  14. package/dist/public-runtime.d.ts.map +1 -1
  15. package/dist/public-runtime.js.map +1 -1
  16. package/dist/runtime/query/executor.d.ts +34 -0
  17. package/dist/runtime/query/executor.d.ts.map +1 -1
  18. package/dist/runtime/query/executor.js +61 -0
  19. package/dist/runtime/query/executor.js.map +1 -1
  20. package/dist/runtime/query/index.d.ts +31 -13
  21. package/dist/runtime/query/index.d.ts.map +1 -1
  22. package/dist/runtime/query/index.js +28 -6
  23. package/dist/runtime/query/index.js.map +1 -1
  24. package/dist/runtime/query/iteration/index.d.ts.map +1 -1
  25. package/dist/runtime/query/iteration/index.js +178 -20
  26. package/dist/runtime/query/iteration/index.js.map +1 -1
  27. package/dist/runtime/query/iteration/stream-turn.d.ts +14 -1
  28. package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
  29. package/dist/runtime/query/iteration/stream-turn.js +19 -3
  30. package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
  31. package/dist/runtime/query/resume-pending.d.ts.map +1 -1
  32. package/dist/runtime/query/resume-pending.js +10 -1
  33. package/dist/runtime/query/resume-pending.js.map +1 -1
  34. package/dist/runtime/query/tool-pause.d.ts +35 -0
  35. package/dist/runtime/query/tool-pause.d.ts.map +1 -1
  36. package/dist/runtime/query/tool-pause.js +35 -0
  37. package/dist/runtime/query/tool-pause.js.map +1 -1
  38. package/dist/runtime/query/tooling.d.ts +2 -0
  39. package/dist/runtime/query/tooling.d.ts.map +1 -1
  40. package/dist/runtime/query/tooling.js +3 -0
  41. package/dist/runtime/query/tooling.js.map +1 -1
  42. package/dist/types/agent/reactive.d.ts +1 -0
  43. package/dist/types/agent/reactive.d.ts.map +1 -1
  44. package/dist/types/run/step.d.ts +75 -3
  45. package/dist/types/run/step.d.ts.map +1 -1
  46. package/dist/types/run/step.js.map +1 -1
  47. package/dist/utils/backoff.d.ts +44 -0
  48. package/dist/utils/backoff.d.ts.map +1 -0
  49. package/dist/utils/backoff.js +58 -0
  50. package/dist/utils/backoff.js.map +1 -0
  51. package/package.json +1 -1
  52. package/src/agents/ReactiveAgent.ts +3 -0
  53. package/src/compaction/context-window.ts +28 -3
  54. package/src/provider/retry.ts +10 -35
  55. package/src/public-runtime.ts +5 -0
  56. package/src/runtime/query/executor.ts +74 -0
  57. package/src/runtime/query/index.ts +59 -18
  58. package/src/runtime/query/iteration/index.ts +199 -22
  59. package/src/runtime/query/iteration/stream-turn.ts +24 -3
  60. package/src/runtime/query/resume-pending.ts +10 -1
  61. package/src/runtime/query/tool-pause.ts +37 -0
  62. package/src/runtime/query/tooling.ts +5 -0
  63. package/src/types/agent/reactive.ts +1 -0
  64. package/src/types/run/step.ts +76 -3
  65. package/src/utils/backoff.ts +70 -0
@@ -34,6 +34,7 @@ import type {
34
34
  ToolCallRepairReason,
35
35
  } from '../../types/tool/repair.js'
36
36
  import { abortReasonText } from '../../utils/abort.js'
37
+ import { type BackoffPolicy, backoffWithJitter, sleep } from '../../utils/backoff.js'
37
38
  import { toErrorMessage } from '../../utils/error.js'
38
39
  import type { Logger } from '../../utils/logger.js'
39
40
  import { compressShellOutput } from '../../utils/shell-compress.js'
@@ -74,6 +75,33 @@ export const DEFAULT_TOOL_CONCURRENCY = 8
74
75
  */
75
76
  export const HOOK_RETRY_BUDGET = 1
76
77
 
78
+ /**
79
+ * Wait between in-loop tool retry attempts.
80
+ *
81
+ * There was none. A tool that declared itself retryable was re-run the
82
+ * instant it failed, as many times as its budget allowed — and the failures
83
+ * worth retrying are the ones an immediate retry makes worse: a rate limit
84
+ * answers the second call faster than it recovers, a contended lock is still
85
+ * held, a connection that has not finished opening has not finished opening.
86
+ *
87
+ * The numbers are the provider policy's, deliberately, and not because a tool
88
+ * is a model call. Nothing here has been measured against tools specifically,
89
+ * and inventing a second curve to look considered would be a guess wearing
90
+ * different digits; the shared one is at least the curve this codebase has
91
+ * already run in anger. Full jitter draws each wait from `[0, curve]`, so the
92
+ * first retry of a tool with the shipped budget waits under half a second on
93
+ * average.
94
+ *
95
+ * The ceiling is inert at the budgets anyone sets — a tool declaring
96
+ * `maxRetries: 3` never reaches 2s — and binds only a host that sets a large
97
+ * one. Override with {@link ToolExecutorConfig.toolRetryBackoff}; set
98
+ * `initialDelayMs: 0` for the previous no-wait behaviour.
99
+ */
100
+ export const DEFAULT_TOOL_RETRY_BACKOFF: BackoffPolicy = {
101
+ initialDelayMs: 500,
102
+ maxDelayMs: 16_000,
103
+ }
104
+
77
105
  /**
78
106
  * An empty arguments string means "no arguments", not "malformed" — the
79
107
  * shape a no-parameter tool arrives in.
@@ -95,6 +123,16 @@ export interface ToolExecutorConfig {
95
123
  pluginManager?: PluginLifecycleManager
96
124
  /** Run-level default deadline; per-tool `timeoutMs` overrides it. */
97
125
  toolTimeoutMs?: number
126
+ /**
127
+ * Wait between in-loop retries of a failed tool call. Defaults to
128
+ * {@link DEFAULT_TOOL_RETRY_BACKOFF}.
129
+ *
130
+ * Applies only to a tool that opted into retrying at all
131
+ * ({@link ToolDefinition.maxRetries}) or to a `post_tool_use` hook that
132
+ * asked for one, so a run whose tools all take the shipped default of
133
+ * zero retries never sleeps here.
134
+ */
135
+ toolRetryBackoff?: Partial<BackoffPolicy>
98
136
  /** Max concurrently-executing concurrency-safe tools. */
99
137
  maxToolConcurrency?: number
100
138
 
@@ -648,6 +686,10 @@ export class ToolExecutor {
648
686
  // because the SDK cannot know a tool is idempotent — silently
649
687
  // re-running a write or a payment is worse than never retrying.
650
688
  const maxRetries = Math.max(0, this.config.tools.get(toolName)?.maxRetries ?? 0)
689
+ const backoff: BackoffPolicy = {
690
+ ...DEFAULT_TOOL_RETRY_BACKOFF,
691
+ ...this.config.toolRetryBackoff,
692
+ }
651
693
  for (let attempt = 1; ; attempt++) {
652
694
  // A missing file will not appear on the second attempt; burning
653
695
  // the budget on it only delays the error the model needs to see.
@@ -664,14 +706,46 @@ export class ToolExecutor {
664
706
  const budget = post.retry ? Math.max(maxRetries, HOOK_RETRY_BUDGET) : maxRetries
665
707
  if (attempt > budget) break
666
708
 
709
+ // Wait before trying again, on the curve the provider path has
710
+ // used all along. This loop had NO delay: a tool failing on a
711
+ // transient condition — a rate-limited HTTP call, a lock, a cold
712
+ // connection — was re-run immediately, several times, which is the
713
+ // pattern most likely to prolong the very condition it is retrying
714
+ // against.
715
+ //
716
+ // Full jitter rather than a fixed wait, and the concurrency that
717
+ // makes it matter is one this loop creates itself: a model emits a
718
+ // batch of parallel calls, `executeBatch` runs up to
719
+ // DEFAULT_TOOL_CONCURRENCY of them at once, they hit the same
720
+ // rate-limited endpoint and fail together. A fixed backoff would
721
+ // resynchronise that batch on every attempt.
722
+ //
723
+ // `attempt` is 1-based here and `backoffWithJitter` is 0-based, so
724
+ // the first retry draws from `[0, initialDelayMs]`.
725
+ const delayMs = backoffWithJitter(attempt - 1, backoff)
726
+
667
727
  this.log.info('Retrying a failed tool call', {
668
728
  runId: this.config.runId,
669
729
  tool: toolName,
670
730
  attempt,
671
731
  budget,
672
732
  requestedByHook: post.retry,
733
+ delayMs,
673
734
  error: result.error,
674
735
  })
736
+
737
+ try {
738
+ await sleep(delayMs, this.config.abortSignal)
739
+ } catch {
740
+ // Stopped mid-backoff. Give up retrying and let the failure
741
+ // already in `result` be this call's answer, rather than
742
+ // throwing: every `tool_use` must be answered by a
743
+ // `tool_result` with the same id, and an abort escaping from
744
+ // here would leave this one open in the transcript for a
745
+ // resume to trip over.
746
+ break
747
+ }
748
+
675
749
  result = await this.runOnce(toolName, input, toolContext)
676
750
  post = await this.runPostToolHook(toolName, input, result)
677
751
  }
@@ -83,6 +83,7 @@ import type { TaskStore } from '../../types/task/index.js'
83
83
  import type { ToolRegistryContract } from '../../types/tool/index.js'
84
84
  import type { RepairToolCall } from '../../types/tool/repair.js'
85
85
  import type { VerificationGateConfig } from '../../types/verification/index.js'
86
+ import type { BackoffPolicy } from '../../utils/backoff.js'
86
87
  import type { ModelPricing } from '../../utils/cost.js'
87
88
  import { getRootLogger } from '../../utils/logger.js'
88
89
  import { VerificationGate } from '../../verification/gate.js'
@@ -98,7 +99,7 @@ import { isWorkingMemoryMessage } from './iteration/phases/working-memory.js'
98
99
  import { applyLifecycleHookResults } from './plugin-hooks.js'
99
100
  import { PromptBuilder } from './prompt.js'
100
101
  import type { PromptSegments } from './prompt.js'
101
- import type { PendingAnswers, QuestionParkBinding } from './question-park.js'
102
+ import { PendingAnswers, QuestionParkBinding } from './question-park.js'
102
103
  import { ResultAssembler } from './result.js'
103
104
  import {
104
105
  type PendingResumePlan,
@@ -168,14 +169,20 @@ export interface QueryParams {
168
169
  emergencySave?: boolean
169
170
 
170
171
  /**
171
- * Durability for questions raised from inside a tool.
172
+ * Durability for questions raised by a tool that closed over its
173
+ * binding before the run existed.
172
174
  *
173
- * The tool that asks is built before the run exists, so the binding is
174
- * created by whoever builds the tools and attached here — that is what
175
- * lets one tool instance be durable inside a run and inert outside one.
176
- * Without it, a question park exists only as a suspended `await`: kill
177
- * the process while somebody is looking at the card and the answer can
178
- * never be applied.
175
+ * The built-in `ask_user_question` is built with the agent's tool
176
+ * registry, so only whoever builds the tools can hand it one — that is
177
+ * what lets a single tool instance be durable inside a run and inert
178
+ * outside one. Without it, THAT tool's park is only a suspended
179
+ * `await`: kill the process while somebody is looking at the card and
180
+ * the answer can never be applied.
181
+ *
182
+ * Not required for `ToolContext.requestPause`. The run builds that
183
+ * seam per call and binds its own recorder when none is passed, so a
184
+ * pause raised from a host-authored tool is durable on every surface
185
+ * rather than only on the one agent class that supplies this.
179
186
  */
180
187
  questionParks?: QuestionParkBinding
181
188
 
@@ -192,16 +199,28 @@ export interface QueryParams {
192
199
  /**
193
200
  * The registry a re-entered `ask_user_question` reads its answer from.
194
201
  *
195
- * Same shape as {@link questionParks}: the tool is built before the run
196
- * exists, so the instance is created by whoever builds the tools and
197
- * filled here on the resume path. Without it a resumed run re-asks a
198
- * question the user already answered.
202
+ * Same shape, same reason and same limit as {@link questionParks}: it
203
+ * exists for a tool that closed over the instance before the run did,
204
+ * and without it a resumed run re-asks that tool's question. A pause
205
+ * from `ToolContext.requestPause` needs none, because the run fills
206
+ * its own on the resume path.
199
207
  */
200
208
  pendingAnswers?: PendingAnswers
201
209
 
202
210
  /** Default per-tool execution deadline. See {@link ToolDefinition.timeoutMs}. */
203
211
  toolTimeoutMs?: number
204
212
 
213
+ /**
214
+ * Wait between in-loop retries of a failed tool call, with full jitter.
215
+ * Defaults to {@link DEFAULT_TOOL_RETRY_BACKOFF}.
216
+ *
217
+ * Only reached by a tool that opted into retrying
218
+ * ({@link ToolDefinition.maxRetries}) or a `post_tool_use` hook that asked
219
+ * for one. Set `initialDelayMs: 0` for the retry-immediately behaviour
220
+ * this loop had before it had any backoff at all.
221
+ */
222
+ toolRetryBackoff?: Partial<BackoffPolicy>
223
+
205
224
  /** Max concurrently-executing concurrency-safe tools in one batch. */
206
225
  maxToolConcurrency?: number
207
226
 
@@ -874,6 +893,25 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
874
893
  ? []
875
894
  : withDeferredDiscoveryTool(params.tools, params.allowedTools)
876
895
 
896
+ // The two halves of a durable pause, owned by the RUN when the host
897
+ // does not own them.
898
+ //
899
+ // `SupervisorAgent` builds both before the run exists, because the
900
+ // tools it builds close over them, and it passes them in. Nothing else
901
+ // could: neither type is exported from `public-runtime.ts`, so a host
902
+ // on `ReactiveAgent`, `drainQuery` or `resumeRun` had no way to supply
903
+ // either — and `ToolContext.requestPause`, which every tool author is
904
+ // handed, silently wrote no checkpoint and could receive no answer on
905
+ // those surfaces. Which agent class the host happened to pick is not
906
+ // visible at the call site, so the degradation was invisible too.
907
+ //
908
+ // A run-local pair is enough for the general seam because `query()`
909
+ // builds its `createToolPause` itself, below, and can hand it the
910
+ // run's own. Pinned by the "a pause is durable on any surface" cases
911
+ // in `__tests__/tool-pause-resume.test.ts`.
912
+ const questionParks = params.questionParks ?? new QuestionParkBinding()
913
+ const pendingAnswers = params.pendingAnswers ?? new PendingAnswers()
914
+
877
915
  // is null only when the run has no disk layout (tests,
878
916
  // in-memory hosts); the budget then degrades to middle-elision.
879
917
  const runDirForTools = ctx.runMgr.getRunDir()
@@ -891,6 +929,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
891
929
  invocationState: params.invocationState,
892
930
  pluginManager: params.pluginManager,
893
931
  ...(params.toolTimeoutMs !== undefined ? { toolTimeoutMs: params.toolTimeoutMs } : {}),
932
+ ...(params.toolRetryBackoff !== undefined
933
+ ? { toolRetryBackoff: params.toolRetryBackoff }
934
+ : {}),
894
935
  ...(params.maxToolConcurrency !== undefined
895
936
  ? { maxToolConcurrency: params.maxToolConcurrency }
896
937
  : {}),
@@ -915,8 +956,8 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
915
956
  runId: ctx.runId,
916
957
  toolUseId,
917
958
  parkHandler: params.resumeHandler,
918
- ...(params.questionParks ? { recorder: params.questionParks } : {}),
919
- ...(params.pendingAnswers ? { pendingAnswers: params.pendingAnswers } : {}),
959
+ recorder: questionParks,
960
+ pendingAnswers,
920
961
  }),
921
962
  },
922
963
  ctx.activityStore,
@@ -1130,7 +1171,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1130
1171
  // the checkpoint did not exist: nothing on disk said a human owed
1131
1172
  // this run an answer, and a remote host could not observe the
1132
1173
  // question at all.
1133
- params.questionParks?.bind({
1174
+ questionParks.bind({
1134
1175
  record: async (question) => {
1135
1176
  try {
1136
1177
  const checkpoint = await checkpointMgr.create(ctx.runMgr, ctx.runMgr.currentIteration)
@@ -1537,9 +1578,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1537
1578
  // closed over its registry when the agent was constructed,
1538
1579
  // long before this run existed, so the answers are copied in
1539
1580
  // rather than passed down.
1540
- if (pendingResume.answers && params.pendingAnswers) {
1581
+ if (pendingResume.answers) {
1541
1582
  for (const [questionId, answer] of pendingResume.answers.entries()) {
1542
- params.pendingAnswers.set(questionId, answer)
1583
+ pendingAnswers.set(questionId, answer)
1543
1584
  }
1544
1585
  }
1545
1586
 
@@ -1642,7 +1683,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1642
1683
  // Same reasoning for the question channel: the tools outlive the
1643
1684
  // run that bound them, so leaving it attached would have a later
1644
1685
  // run's question written into this run's checkpoint store.
1645
- params.questionParks?.unbind()
1686
+ questionParks.unbind()
1646
1687
 
1647
1688
  // Offer what the run learned to whoever decides what is worth
1648
1689
  // keeping. In `finally` and awaited: a run that failed still
@@ -32,6 +32,7 @@ import type { AnswerReview } from '../../../types/run/answer-review.js'
32
32
  import type {
33
33
  PrepareStepResult,
34
34
  RunEvent,
35
+ StepFailure,
35
36
  StepProvenance,
36
37
  StepResult,
37
38
  StopReason,
@@ -258,6 +259,30 @@ export class IterationOrchestrator {
258
259
  {},
259
260
  parentContext(this.ctx.rootSpan),
260
261
  )
262
+
263
+ // Everything the step record needs, hoisted so the `catch` can
264
+ // read whatever the iteration got as far as computing.
265
+ //
266
+ // The failure path is the one the ledger's own argument was
267
+ // written for and the one it never reached: an iteration that
268
+ // threw recorded a span exception and re-threw, so the turn with
269
+ // no record was exactly the turn that went wrong. A reader could
270
+ // not tell that from a turn that never happened.
271
+ //
272
+ // Declared as `let` with real initial values rather than left
273
+ // undefined, because a failure BEFORE the snapshot below is
274
+ // taken has spent nothing, and these are then exact. The success
275
+ // path is untouched: the assignments inside the try still happen
276
+ // where they always did, so compaction and the working-memory
277
+ // refresh stay outside a successful step's window.
278
+ let stepStartedAt = Date.now()
279
+ let usageBefore: TokenUsage = { ...runMgr.tokenUsage }
280
+ let costBefore: CostInfo = { ...runMgr.costInfo }
281
+ let stepModel = model
282
+ let stepMessageId: MessageId | undefined
283
+ let stepResponse: ChatCompletionResponse | undefined
284
+ let stepServedBy: StepProvenance | undefined
285
+
261
286
  try {
262
287
  // Tool spans for this turn belong under this iteration. Inside
263
288
  // the try rather than before it: a throw from any of these left
@@ -303,9 +328,9 @@ export class IterationOrchestrator {
303
328
  // tool_use/tool_result blocks.
304
329
  // Snapshot the cumulative counters so the step can report ITS
305
330
  // own usage rather than the run total.
306
- const stepStartedAt = Date.now()
307
- const usageBefore = { ...runMgr.tokenUsage }
308
- const costBefore = { ...runMgr.costInfo }
331
+ stepStartedAt = Date.now()
332
+ usageBefore = { ...runMgr.tokenUsage }
333
+ costBefore = { ...runMgr.costInfo }
309
334
 
310
335
  // Shape this step before calling the model. `stopWhen` decides
311
336
  // whether to keep going; this decides HOW. No-op when the host
@@ -320,7 +345,7 @@ export class IterationOrchestrator {
320
345
  // still call any of them by name.
321
346
  this.ctx.toolExecutor.setStepAllowedTools(stepAllowedTools)
322
347
  const enforceToolInputSchema = enforcedModelInputToolNames(this.ctx.tools, llmTools)
323
- const stepModel = step.model ?? model
348
+ stepModel = step.model ?? model
324
349
 
325
350
  const baseMessages = forceFinalize
326
351
  ? [
@@ -389,6 +414,16 @@ export class IterationOrchestrator {
389
414
  // aggregated `ChatCompletionResponse` for the legacy
390
415
  // downstream paths (assistantMsg construction, working
391
416
  // state extraction, telemetry attribute stamping).
417
+ //
418
+ // The message id is minted HERE, immediately before the call
419
+ // that announces it,
420
+ // rather than inside that call. The return value never arrives
421
+ // when the stream throws, so a step recorded from the catch
422
+ // could otherwise never name the message — and a stream that
423
+ // died part-way has already emitted both `message_started` and
424
+ // `message_completed` under this id, which is the trail a
425
+ // reader wants most on exactly that turn.
426
+ stepMessageId = generateMessageId()
392
427
  const { response, messageId } = yield* streamProviderTurn(
393
428
  this.ctx.provider,
394
429
  {
@@ -424,7 +459,9 @@ export class IterationOrchestrator {
424
459
  forceFinalize,
425
460
  this.ctx.log,
426
461
  iterSpan,
462
+ stepMessageId,
427
463
  )
464
+ stepResponse = response
428
465
 
429
466
  // Who answered THIS turn.
430
467
  //
@@ -459,6 +496,7 @@ export class IterationOrchestrator {
459
496
  chainIndex: member.index,
460
497
  }
461
498
  })()
499
+ stepServedBy = servedBy
462
500
 
463
501
  // Main-loop turn: also records the prompt size compaction reads.
464
502
  //
@@ -1014,6 +1052,76 @@ export class IterationOrchestrator {
1014
1052
  })
1015
1053
  yield* this.ctx.drainPending()
1016
1054
  } catch (err) {
1055
+ const cancelled = this.ctx.abortController.signal.aborted
1056
+
1057
+ // This iteration gets a step too, and it is the one the
1058
+ // argument three hundred lines above was actually about.
1059
+ //
1060
+ // That docblock makes the case for a rejected tool batch — "a
1061
+ // run that spent a turn getting its tools refused still spent
1062
+ // the tokens" — and every call site it produced sat on a
1063
+ // success path. So the ledger was complete except on the turns
1064
+ // that failed, which is the worst shape it could have: an
1065
+ // evidence record that goes quiet exactly where something went
1066
+ // wrong reads as "nothing went wrong". A reader could not
1067
+ // distinguish iteration N failing from iteration N never
1068
+ // happening, while the events said plainly that it started.
1069
+ //
1070
+ // Recorded HERE, at the top of the catch, rather than at each
1071
+ // of its exits — the same reasoning the success path already
1072
+ // wrote down for itself. All three exits spend a turn: the
1073
+ // cancellation breaks, the overflow-relief retry continues
1074
+ // under a NEW iteration number (so its tokens belong to no
1075
+ // later step), and the re-throw ends the run.
1076
+ //
1077
+ // What it carries is what the iteration got as far as knowing.
1078
+ // `usage` is the same subtraction a successful step makes, so
1079
+ // a turn that failed after the provider answered carries that
1080
+ // answer's tokens, and one that failed before it carries the
1081
+ // zero it actually spent. Nothing is estimated to fill a gap.
1082
+ //
1083
+ // At most ONE step per iteration. Both success paths record
1084
+ // before the work that follows them — the advisory phase, the
1085
+ // structured-output capture, the `iteration_end` hooks, the
1086
+ // terminal `iteration_completed` — and any of those can throw
1087
+ // into here. A second entry numbered N would double-count that
1088
+ // turn's tokens against `run.tokenUsage`, which is the same
1089
+ // class of wrong as dropping them and harder to notice, since
1090
+ // the ledger would look fuller rather than emptier. That turn's
1091
+ // own verdict is already written down; the failure that
1092
+ // followed it reaches the caller as the run's error.
1093
+ if (this.steps.at(-1)?.stepNumber === iterationNum) {
1094
+ this.ctx.log.warn('Iteration failed after its step was already recorded', {
1095
+ runId: runMgr.id,
1096
+ iteration: iterationNum,
1097
+ error: toErrorMessage(err),
1098
+ })
1099
+ } else {
1100
+ this.recordStep({
1101
+ stepNumber: iterationNum,
1102
+ model: stepModel,
1103
+ ...(stepServedBy ? { servedBy: stepServedBy } : {}),
1104
+ ...(stepMessageId ? { messageId: stepMessageId } : {}),
1105
+ ...(stepResponse ? { response: stepResponse } : {}),
1106
+ // Tool outcomes are produced and returned together by
1107
+ // `runToolReview`, so a throw from inside it leaves none
1108
+ // to salvage: an empty list here means "none came back",
1109
+ // which is what the shorter-than-`toolCalls` contract
1110
+ // says.
1111
+ toolResults: [],
1112
+ toolExecutionMs: 0,
1113
+ startedAt: stepStartedAt,
1114
+ usageBefore,
1115
+ costBefore,
1116
+ unfinished: cancelled
1117
+ ? { finishReason: 'cancelled' }
1118
+ : {
1119
+ finishReason: 'error',
1120
+ failure: describeStepFailure(err, this.ctx.provider.id),
1121
+ },
1122
+ })
1123
+ }
1124
+
1017
1125
  // A Stop that aborted the in-flight turn surfaces here as a
1018
1126
  // thrown abort (the provider stream was raced against the run
1019
1127
  // signal). Settle it as a CANCELLATION — mirroring the
@@ -1021,7 +1129,7 @@ export class IterationOrchestrator {
1021
1129
  // recording it as an SDK failure (error span + failed activity)
1022
1130
  // and re-throwing. The run then returns cleanly with a
1023
1131
  // 'cancelled' stop reason instead of propagating an error.
1024
- if (this.ctx.abortController.signal.aborted) {
1132
+ if (cancelled) {
1025
1133
  runMgr.setStopReason('cancelled')
1026
1134
  runMgr.markCancelled()
1027
1135
  break
@@ -1308,39 +1416,69 @@ export class IterationOrchestrator {
1308
1416
  private recordStep(input: {
1309
1417
  stepNumber: number
1310
1418
  model: string
1311
- servedBy: StepProvenance
1312
- messageId: MessageId
1313
- response: ChatCompletionResponse
1419
+ servedBy?: StepProvenance
1420
+ messageId?: MessageId
1421
+ /**
1422
+ * The turn's response. Absent only when the iteration failed before
1423
+ * the provider produced one — see `unfinished`.
1424
+ */
1425
+ response?: ChatCompletionResponse
1314
1426
  toolResults: readonly ToolCallOutcome[]
1315
1427
  toolExecutionMs: number
1316
1428
  startedAt: number
1317
1429
  usageBefore: TokenUsage
1318
1430
  costBefore: CostInfo
1431
+ /**
1432
+ * Set only by the `catch`, for an iteration that did not finish.
1433
+ *
1434
+ * The same writer builds both records on purpose: a failed turn's
1435
+ * step is a `StepResult` like any other, so a caller reconstructing
1436
+ * cost or history sorts them together instead of discovering that
1437
+ * failures live somewhere else.
1438
+ */
1439
+ unfinished?: { finishReason: 'error' | 'cancelled'; failure?: StepFailure }
1319
1440
  }): void {
1320
1441
  const { runMgr } = this.ctx
1321
- const toolCalls = input.response.message.toolCalls ?? []
1442
+ const toolCalls = input.response?.message.toolCalls ?? []
1322
1443
  const byId = new Map(input.toolResults.map((r) => [r.toolCallId, r]))
1323
1444
 
1324
1445
  const step: StepResult = {
1325
1446
  stepNumber: input.stepNumber,
1326
1447
  model: input.model,
1327
- servedBy: input.servedBy,
1328
- messageId: input.messageId,
1329
- content: input.response.message.content,
1448
+ ...(input.servedBy ? { servedBy: input.servedBy } : {}),
1449
+ ...(input.messageId ? { messageId: input.messageId } : {}),
1450
+ content: input.response?.message.content ?? null,
1330
1451
  toolCalls,
1331
1452
  // Ordered by the tool CALLS, not by completion, so the record
1332
1453
  // matches what the model asked for.
1333
- toolResults: toolCalls.map((tc) => {
1454
+ //
1455
+ // On an unfinished step the calls with no outcome are DROPPED
1456
+ // rather than filled with `{output: '', isError: false}`. That
1457
+ // filler is a reading of "the batch was refused" on the success
1458
+ // path, where every call in a batch shares one verdict; under a
1459
+ // step that says `error` it would say a tool ran and returned
1460
+ // nothing successfully, which is the same lie one level down as
1461
+ // the missing step itself.
1462
+ toolResults: toolCalls.flatMap((tc) => {
1334
1463
  const outcome = byId.get(tc.id)
1335
- return {
1336
- toolCallId: tc.id,
1337
- toolName: tc.function.name,
1338
- output: outcome?.output ?? '',
1339
- isError: outcome?.isError ?? false,
1340
- durationMs: 0,
1341
- }
1464
+ if (input.unfinished && !outcome) return []
1465
+ return [
1466
+ {
1467
+ toolCallId: tc.id,
1468
+ toolName: tc.function.name,
1469
+ output: outcome?.output ?? '',
1470
+ isError: outcome?.isError ?? false,
1471
+ durationMs: 0,
1472
+ },
1473
+ ]
1342
1474
  }),
1343
- finishReason: input.response.finishReason,
1475
+ // The turn's own verdict where there is one. A step that ended in
1476
+ // the catch has none — no provider reported `error` or
1477
+ // `cancelled` — so `unfinished` wins even when a response had
1478
+ // already arrived: a turn that answered and then threw during
1479
+ // tool execution did not end in `tool_calls`.
1480
+ finishReason: input.unfinished?.finishReason ?? input.response?.finishReason ?? 'error',
1481
+ ...(input.unfinished?.failure ? { failure: input.unfinished.failure } : {}),
1344
1482
  usage: subtractUsage(runMgr.tokenUsage, input.usageBefore),
1345
1483
  costDelta: {
1346
1484
  ...runMgr.costInfo,
@@ -1352,7 +1490,26 @@ export class IterationOrchestrator {
1352
1490
  }
1353
1491
 
1354
1492
  this.steps.push(step)
1355
- this.ctx.onStepFinish?.(step)
1493
+
1494
+ if (!input.unfinished) {
1495
+ this.ctx.onStepFinish?.(step)
1496
+ return
1497
+ }
1498
+
1499
+ // Nothing here is allowed to throw over the failure that is already
1500
+ // unwinding — the same rule `settleCancelledTurn` states for the
1501
+ // cancellation path. A host callback that throws while being told a
1502
+ // turn failed would REPLACE the reason the turn failed, so the run
1503
+ // would report the observer's bug and lose the original.
1504
+ try {
1505
+ this.ctx.onStepFinish?.(step)
1506
+ } catch (err) {
1507
+ this.ctx.log.warn('onStepFinish threw while recording a failed step', {
1508
+ runId: runMgr.id,
1509
+ step: input.stepNumber,
1510
+ error: toErrorMessage(err),
1511
+ })
1512
+ }
1356
1513
  }
1357
1514
 
1358
1515
  /** Turns spent asking the model again for a valid structured output. */
@@ -1621,6 +1778,26 @@ export class IterationOrchestrator {
1621
1778
  }
1622
1779
  }
1623
1780
 
1781
+ /**
1782
+ * Fold whatever ended an iteration into the record a reader gets.
1783
+ *
1784
+ * Classified through `classifyProviderError` — the same call the catch
1785
+ * already makes to decide whether compaction relief applies — so the step's
1786
+ * verdict and the loop's own decision cannot drift apart. It also handles a
1787
+ * failure that is not a provider failure at all: the code set's `unknown`
1788
+ * means "unclassifiable", which is the true answer for a plugin hook that
1789
+ * threw and is left saying so rather than dressed up as something specific.
1790
+ */
1791
+ function describeStepFailure(err: unknown, providerId: string): StepFailure {
1792
+ const classified = classifyProviderError(err, providerId)
1793
+ return {
1794
+ message: toErrorMessage(err),
1795
+ code: classified.code,
1796
+ ...(classified.status !== undefined ? { status: classified.status } : {}),
1797
+ retryable: classified.retryable,
1798
+ }
1799
+ }
1800
+
1624
1801
  /** Per-step usage: the delta between two cumulative snapshots. */
1625
1802
  function subtractUsage(after: TokenUsage, before: TokenUsage): TokenUsage {
1626
1803
  return {
@@ -134,6 +134,19 @@ export async function* streamProviderTurn(
134
134
  forceFinalize: boolean,
135
135
  log: Logger,
136
136
  parentSpan?: Span,
137
+ /**
138
+ * The id to announce this message under.
139
+ *
140
+ * Supplied by the loop so a turn that THROWS still leaves the caller
141
+ * holding the id it announced. The return value never arrives on a
142
+ * failure, so without this the one case where a failed step most wants
143
+ * to point at the event stream — a stream that died after
144
+ * `message_started`, having already emitted `message_completed` on the
145
+ * way out — is precisely the case that could not.
146
+ *
147
+ * Optional, so a caller with no use for the id is unchanged.
148
+ */
149
+ announceAs?: import('../../../types/ids/index.js').MessageId,
137
150
  ): AsyncGenerator<RunEvent, StreamingTurnResult> {
138
151
  // The `chat {model}` span the GenAI conventions require. There was none:
139
152
  // `chatSpanName` existed with zero call sites, so a trace carried no LLM
@@ -151,7 +164,7 @@ export async function* streamProviderTurn(
151
164
  ...(params.maxTokens !== undefined ? { [GENAI.REQUEST_MAX_TOKENS]: params.maxTokens } : {}),
152
165
  })
153
166
 
154
- const messageId = generateMessageId()
167
+ const messageId = announceAs ?? generateMessageId()
155
168
  await emitEvent({ type: 'message_started', runId, iteration, messageId })
156
169
  yield* drainPending()
157
170
 
@@ -194,7 +207,12 @@ export async function* streamProviderTurn(
194
207
  // verbatim — so the map is drained in index order at the end.
195
208
  const reasoningBuckets = new Map<
196
209
  number,
197
- { type: 'thinking' | 'redacted_thinking'; text: string; signature?: string; encrypted?: string }
210
+ {
211
+ type: 'thinking' | 'redacted_thinking'
212
+ text: string
213
+ signature?: string
214
+ encrypted?: string
215
+ }
198
216
  >()
199
217
 
200
218
  // Citations arrive as their own deltas, in the order the model made
@@ -205,7 +223,10 @@ export async function* streamProviderTurn(
205
223
  let streamError: string | undefined
206
224
  let streamCause: unknown
207
225
 
208
- const stream = provider.chatStream({ ...params, stream: true }) as AsyncIterable<StreamChunk>
226
+ const stream = provider.chatStream({
227
+ ...params,
228
+ stream: true,
229
+ }) as AsyncIterable<StreamChunk>
209
230
 
210
231
  // Drive the stream manually so each `.next()` can be RACED against the run
211
232
  // abort: a Stop tears the in-flight model request down (the provider got
@@ -9,6 +9,7 @@ import type { ChatCompletionResponse } from '../../types/provider/index.js'
9
9
  import type { Logger } from '../../utils/logger.js'
10
10
  import type { PriorToolResults, ToolCallDenials, ToolExecutor } from './executor.js'
11
11
  import { PendingAnswers } from './question-park.js'
12
+ import { isPauseForCall } from './tool-pause.js'
12
13
 
13
14
  /**
14
15
  * Apply a decision collected out-of-band to the tool calls a run parked on.
@@ -154,7 +155,15 @@ function planQuestionResume(
154
155
  // otherwise have its answer delivered to whatever tool now holds that
155
156
  // slot — the misdirection the asking tool's own id guard exists to
156
157
  // prevent, checked here too because by then the tool has been entered.
157
- if (!assistant.toolCalls.some((tc) => tc.id === questionId)) {
158
+ //
159
+ // Through `isPauseForCall` rather than by equality, because a parked
160
+ // question id is not a call id. The general seam appends the tool
161
+ // author's pause name to it, so equality compared
162
+ // `call_1:target_environment` against `call_1`, could never hold, and
163
+ // refused every cross-process resume of a host-authored pause. Only
164
+ // the built-in question tool got through, and only because it parks
165
+ // under the bare tool-use id.
166
+ if (!assistant.toolCalls.some((tc) => isPauseForCall(questionId, tc.id))) {
158
167
  log.error('The parked question does not belong to any unanswered call in this turn', {
159
168
  checkpointId: checkpoint.id,
160
169
  questionId,