@namzu/sdk 23.0.0 → 26.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +208 -0
  2. package/README.md +1 -1
  3. package/dist/agents/ReactiveAgent.d.ts.map +1 -1
  4. package/dist/agents/ReactiveAgent.js +3 -0
  5. package/dist/agents/ReactiveAgent.js.map +1 -1
  6. package/dist/connector/mcp/stdio.d.ts +11 -0
  7. package/dist/connector/mcp/stdio.d.ts.map +1 -1
  8. package/dist/connector/mcp/stdio.js +106 -1
  9. package/dist/connector/mcp/stdio.js.map +1 -1
  10. package/dist/constants/telemetry/index.d.ts +15 -1
  11. package/dist/constants/telemetry/index.d.ts.map +1 -1
  12. package/dist/constants/telemetry/index.js +15 -1
  13. package/dist/constants/telemetry/index.js.map +1 -1
  14. package/dist/provider/fallback.d.ts.map +1 -1
  15. package/dist/provider/fallback.js +8 -2
  16. package/dist/provider/fallback.js.map +1 -1
  17. package/dist/provider/retry.d.ts +10 -5
  18. package/dist/provider/retry.d.ts.map +1 -1
  19. package/dist/provider/retry.js +11 -31
  20. package/dist/provider/retry.js.map +1 -1
  21. package/dist/public-runtime.d.ts +1 -0
  22. package/dist/public-runtime.d.ts.map +1 -1
  23. package/dist/public-runtime.js.map +1 -1
  24. package/dist/registry/tool/execute.d.ts.map +1 -1
  25. package/dist/registry/tool/execute.js +12 -0
  26. package/dist/registry/tool/execute.js.map +1 -1
  27. package/dist/runtime/query/executor.d.ts +34 -0
  28. package/dist/runtime/query/executor.d.ts.map +1 -1
  29. package/dist/runtime/query/executor.js +61 -0
  30. package/dist/runtime/query/executor.js.map +1 -1
  31. package/dist/runtime/query/index.d.ts +31 -13
  32. package/dist/runtime/query/index.d.ts.map +1 -1
  33. package/dist/runtime/query/index.js +28 -6
  34. package/dist/runtime/query/index.js.map +1 -1
  35. package/dist/runtime/query/iteration/index.d.ts.map +1 -1
  36. package/dist/runtime/query/iteration/index.js +178 -20
  37. package/dist/runtime/query/iteration/index.js.map +1 -1
  38. package/dist/runtime/query/iteration/stream-turn.d.ts +14 -1
  39. package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
  40. package/dist/runtime/query/iteration/stream-turn.js +19 -3
  41. package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
  42. package/dist/runtime/query/resume-pending.d.ts.map +1 -1
  43. package/dist/runtime/query/resume-pending.js +10 -1
  44. package/dist/runtime/query/resume-pending.js.map +1 -1
  45. package/dist/runtime/query/tool-pause.d.ts +35 -0
  46. package/dist/runtime/query/tool-pause.d.ts.map +1 -1
  47. package/dist/runtime/query/tool-pause.js +35 -0
  48. package/dist/runtime/query/tool-pause.js.map +1 -1
  49. package/dist/runtime/query/tooling.d.ts +2 -0
  50. package/dist/runtime/query/tooling.d.ts.map +1 -1
  51. package/dist/runtime/query/tooling.js +3 -0
  52. package/dist/runtime/query/tooling.js.map +1 -1
  53. package/dist/types/agent/reactive.d.ts +1 -0
  54. package/dist/types/agent/reactive.d.ts.map +1 -1
  55. package/dist/types/connector/mcp.d.ts +17 -0
  56. package/dist/types/connector/mcp.d.ts.map +1 -1
  57. package/dist/types/provider/interface.d.ts +23 -2
  58. package/dist/types/provider/interface.d.ts.map +1 -1
  59. package/dist/types/run/step.d.ts +75 -3
  60. package/dist/types/run/step.d.ts.map +1 -1
  61. package/dist/types/run/step.js.map +1 -1
  62. package/dist/utils/backoff.d.ts +44 -0
  63. package/dist/utils/backoff.d.ts.map +1 -0
  64. package/dist/utils/backoff.js +58 -0
  65. package/dist/utils/backoff.js.map +1 -0
  66. package/package.json +1 -1
  67. package/src/agents/ReactiveAgent.ts +3 -0
  68. package/src/connector/mcp/stdio.ts +106 -1
  69. package/src/constants/telemetry/index.ts +16 -1
  70. package/src/provider/fallback.ts +8 -2
  71. package/src/provider/retry.ts +20 -37
  72. package/src/public-runtime.ts +5 -0
  73. package/src/registry/tool/execute.ts +12 -0
  74. package/src/runtime/query/executor.ts +74 -0
  75. package/src/runtime/query/index.ts +59 -18
  76. package/src/runtime/query/iteration/index.ts +199 -22
  77. package/src/runtime/query/iteration/stream-turn.ts +24 -3
  78. package/src/runtime/query/resume-pending.ts +10 -1
  79. package/src/runtime/query/tool-pause.ts +37 -0
  80. package/src/runtime/query/tooling.ts +5 -0
  81. package/src/types/agent/reactive.ts +1 -0
  82. package/src/types/connector/mcp.ts +17 -0
  83. package/src/types/provider/interface.ts +23 -2
  84. package/src/types/run/step.ts +76 -3
  85. package/src/utils/backoff.ts +70 -0
@@ -13,6 +13,111 @@ import { type Logger, getRootLogger } from '../../utils/logger.js'
13
13
  */
14
14
  const TERMINATE_GRACE_MS = 2_000
15
15
 
16
+ /**
17
+ * The variables a child needs in order to be a working process at all.
18
+ *
19
+ * This spawn used to pass `{ ...process.env, ...config.env }`, so a connected
20
+ * server received every credential the host happened to hold — measured at 119
21
+ * variables on a developer machine, including a planted secret the server had
22
+ * no reason to see. A server that needs one token was handed all of them, and
23
+ * nothing in the config said so.
24
+ *
25
+ * The list below is process plumbing, not secrets: where to find executables,
26
+ * where the home and temp directories are, what the locale is. Dropping any of
27
+ * it does not harden anything and does break servers — a child with no `PATH`
28
+ * cannot resolve its own interpreter.
29
+ *
30
+ * Anything else a server needs is now named: `env` gives it a literal value,
31
+ * and `inheritEnv` names a parent variable to pass through. Naming is the
32
+ * point — a grant that has to be written down is a grant somebody can review.
33
+ *
34
+ * Windows spellings are matched case-insensitively, because its environment is
35
+ * case-insensitive and a lookup for `Path` against a key stored as `PATH` would
36
+ * silently drop it — which would present as "the server does not start on
37
+ * Windows" rather than as anything to do with this list.
38
+ */
39
+ const BASE_ENV_KEYS: readonly string[] = [
40
+ // Everywhere.
41
+ 'PATH',
42
+ 'LANG',
43
+ 'LC_ALL',
44
+ 'LC_CTYPE',
45
+ 'TZ',
46
+ // POSIX.
47
+ 'HOME',
48
+ 'SHELL',
49
+ 'TMPDIR',
50
+ 'USER',
51
+ 'LOGNAME',
52
+ // Windows. `SystemRoot` and `ComSpec` are load-bearing: without them a
53
+ // child cannot resolve system DLLs or the command interpreter.
54
+ 'PATHEXT',
55
+ 'SystemRoot',
56
+ 'SystemDrive',
57
+ 'ComSpec',
58
+ 'WINDIR',
59
+ 'TEMP',
60
+ 'TMP',
61
+ 'USERPROFILE',
62
+ 'HOMEDRIVE',
63
+ 'HOMEPATH',
64
+ 'APPDATA',
65
+ 'LOCALAPPDATA',
66
+ 'PROGRAMDATA',
67
+ 'PROGRAMFILES',
68
+ 'NUMBER_OF_PROCESSORS',
69
+ 'PROCESSOR_ARCHITECTURE',
70
+ ]
71
+
72
+ /**
73
+ * Read one variable from the parent, honouring the platform's own casing rules.
74
+ *
75
+ * Node exposes `process.env` on Windows through a case-insensitive proxy, so a
76
+ * direct lookup already works there — but the KEY this returns has to be the
77
+ * one the parent actually uses, or a child comparing key names sees a spelling
78
+ * the host never set.
79
+ */
80
+ function readParentVar(source: NodeJS.ProcessEnv, name: string): [string, string] | undefined {
81
+ const direct = source[name]
82
+ if (direct !== undefined) return [name, direct]
83
+ if (process.platform !== 'win32') return undefined
84
+ const lowered = name.toLowerCase()
85
+ for (const [key, value] of Object.entries(source)) {
86
+ if (key.toLowerCase() === lowered && value !== undefined) return [key, value]
87
+ }
88
+ return undefined
89
+ }
90
+
91
+ /**
92
+ * What the child is handed: plumbing, then the named inheritances, then the
93
+ * literal values.
94
+ *
95
+ * Later wins, and the order is the precedence an operator would guess: a
96
+ * literal `env` entry overrides an inherited one, and both override the base.
97
+ * Exported for the tests, which assert on the ENV rather than on the spawn —
98
+ * a test that only checked the config was accepted would have passed against
99
+ * the version this replaces.
100
+ */
101
+ export function buildChildEnv(
102
+ config: Pick<MCPStdioTransportConfig, 'env' | 'inheritEnv'>,
103
+ source: NodeJS.ProcessEnv = process.env,
104
+ ): Record<string, string> {
105
+ const env: Record<string, string> = {}
106
+ for (const name of BASE_ENV_KEYS) {
107
+ const found = readParentVar(source, name)
108
+ if (found) env[found[0]] = found[1]
109
+ }
110
+ for (const name of config.inheritEnv ?? []) {
111
+ const found = readParentVar(source, name)
112
+ // A named variable the parent does not hold is simply absent. Refusing
113
+ // the spawn would turn an optional credential into a startup failure,
114
+ // and inventing an empty string would tell the server it has one.
115
+ if (found) env[found[0]] = found[1]
116
+ }
117
+ for (const [name, value] of Object.entries(config.env ?? {})) env[name] = value
118
+ return env
119
+ }
120
+
16
121
  export class StdioTransport implements MCPTransport {
17
122
  private process: ChildProcess | null = null
18
123
  private messageHandlers: Array<(message: MCPJsonRpcMessage) => void> = []
@@ -32,7 +137,7 @@ export class StdioTransport implements MCPTransport {
32
137
  if (this.connected) return
33
138
 
34
139
  this.process = spawn(this.config.command, this.config.args ?? [], {
35
- env: { ...process.env, ...this.config.env },
140
+ env: buildChildEnv(this.config),
36
141
  cwd: this.config.cwd,
37
142
  stdio: ['pipe', 'pipe', 'pipe'],
38
143
  })
@@ -23,7 +23,22 @@ export const GENAI = {
23
23
 
24
24
  TOOL_NAME: 'gen_ai.tool.name',
25
25
  TOOL_TYPE: 'gen_ai.tool.type',
26
- TOOL_CALL_ID: 'gen_ai.tool.call_id',
26
+
27
+ /**
28
+ * The id of the tool call this span is about.
29
+ *
30
+ * Spelled `call.id`, matching the registry and matching the two
31
+ * neighbours above — this read `gen_ai.tool.call_id`, one underscore
32
+ * where the convention has a dot, so a consumer grouping by the
33
+ * conventional name found nothing under it. Nothing errored, because a
34
+ * span attribute is a free-form key and a wrong one is simply a key
35
+ * nobody asked for.
36
+ *
37
+ * Pinned by `telemetry/__tests__/tool-call-id-attribute.test.ts`, which
38
+ * also drives the emitter: the spelling was only half the defect, and
39
+ * the constant had no writer at all.
40
+ */
41
+ TOOL_CALL_ID: 'gen_ai.tool.call.id',
27
42
 
28
43
  AGENT_NAME: 'gen_ai.agent.name',
29
44
  AGENT_ID: 'gen_ai.agent.id',
@@ -382,7 +382,13 @@ export function withProviderFallback(
382
382
  },
383
383
  chatStream,
384
384
  ...(first.provider.listModels ? { listModels: () => first.provider.listModels?.() } : {}),
385
- ...(first.provider.healthCheck ? { healthCheck: () => first.provider.healthCheck?.() } : {}),
386
- ...(first.provider.doctorCheck ? { doctorCheck: () => first.provider.doctorCheck?.() } : {}),
385
+ // Forwarded for the reason `retry.ts` forwards it: a wrapper that drops
386
+ // the model turns a probe the caller could answer into one it cannot.
387
+ ...(first.provider.healthCheck
388
+ ? { healthCheck: (model?: string) => first.provider.healthCheck?.(model) }
389
+ : {}),
390
+ ...(first.provider.doctorCheck
391
+ ? { doctorCheck: (model?: string) => first.provider.doctorCheck?.(model) }
392
+ : {}),
387
393
  } as LLMProvider
388
394
  }
@@ -1,15 +1,20 @@
1
1
  import { classifyProviderError, isAbortError } from '../types/provider/errors.js'
2
2
  import type { ChatCompletionParams, LLMProvider, StreamChunk } from '../types/provider/index.js'
3
+ import { type BackoffPolicy, backoffWithJitter, sleep } from '../utils/backoff.js'
3
4
  import type { Logger } from '../utils/logger.js'
4
5
  import { isProviderRequestError } from './errors.js'
5
6
 
6
- export interface ProviderRetryConfig {
7
+ /**
8
+ * `BackoffPolicy` plus the provider-specific parts: how many attempts, and
9
+ * how long a server-directed wait may be honoured for.
10
+ *
11
+ * The curve itself lives in `utils/backoff.ts` and is shared with the tool
12
+ * executor's in-loop retry — which had no backoff at all while this one was
13
+ * being careful about jitter two directories away.
14
+ */
15
+ export interface ProviderRetryConfig extends BackoffPolicy {
7
16
  /** Retry attempts AFTER the initial try. `0` disables retrying. */
8
17
  readonly maxRetries: number
9
- /** First backoff, doubled each attempt. */
10
- readonly initialDelayMs: number
11
- /** Ceiling for a single backoff, before jitter. */
12
- readonly maxDelayMs: number
13
18
  /**
14
19
  * Cap on a server-directed `Retry-After`. A provider asking for 15
15
20
  * minutes should not silently park an interactive run for 15 minutes;
@@ -34,36 +39,6 @@ export const DEFAULT_PROVIDER_RETRY: ProviderRetryConfig = {
34
39
  maxRetryAfterMs: 60_000,
35
40
  }
36
41
 
37
- /**
38
- * Full jitter (AWS's formulation): sleep a uniform random amount in
39
- * `[0, backoff]` rather than `backoff` exactly. Equal-jitter and no-jitter
40
- * both keep a fleet of clients that failed together retrying together;
41
- * full jitter is what actually spreads a thundering herd.
42
- */
43
- function backoffWithJitter(attempt: number, config: ProviderRetryConfig, random: () => number) {
44
- const exponential = Math.min(config.initialDelayMs * 2 ** attempt, config.maxDelayMs)
45
- return Math.round(random() * exponential)
46
- }
47
-
48
- function sleep(ms: number, signal?: AbortSignal): Promise<void> {
49
- if (ms <= 0) return Promise.resolve()
50
- return new Promise<void>((resolve, reject) => {
51
- if (signal?.aborted) {
52
- reject(signal.reason)
53
- return
54
- }
55
- const timer = setTimeout(() => {
56
- signal?.removeEventListener('abort', onAbort)
57
- resolve()
58
- }, ms)
59
- const onAbort = () => {
60
- clearTimeout(timer)
61
- reject(signal?.reason)
62
- }
63
- signal?.addEventListener('abort', onAbort, { once: true })
64
- })
65
- }
66
-
67
42
  export interface WithProviderRetryOptions {
68
43
  readonly config?: Partial<ProviderRetryConfig>
69
44
  readonly log?: Logger
@@ -256,7 +231,15 @@ export function withProviderRetry(
256
231
  },
257
232
  chatStream,
258
233
  ...(provider.listModels ? { listModels: () => provider.listModels?.() } : {}),
259
- ...(provider.healthCheck ? { healthCheck: () => provider.healthCheck?.() } : {}),
260
- ...(provider.doctorCheck ? { doctorCheck: () => provider.doctorCheck?.() } : {}),
234
+ // The model is forwarded, not dropped. A wrapper that swallowed it would
235
+ // leave the wrapped driver probing whatever it probes with no argument
236
+ // — which for at least one driver is "nothing", so the check would come
237
+ // back unanswerable purely because it was wrapped.
238
+ ...(provider.healthCheck
239
+ ? { healthCheck: (model?: string) => provider.healthCheck?.(model) }
240
+ : {}),
241
+ ...(provider.doctorCheck
242
+ ? { doctorCheck: (model?: string) => provider.doctorCheck?.(model) }
243
+ : {}),
261
244
  } as LLMProvider
262
245
  }
@@ -357,6 +357,11 @@ export type {
357
357
  WithProviderFallbackOptions,
358
358
  WithProviderRetryOptions,
359
359
  } from './provider/index.js'
360
+ // The curve `ProviderRetryConfig` extends and `query({ toolRetryBackoff })`
361
+ // takes a partial of. Both public surfaces name it, so a consumer has to be
362
+ // able to name it too — a type reachable only through an inline `import(...)`
363
+ // in a `.d.ts` is not a type anyone writes down.
364
+ export type { BackoffPolicy } from './utils/backoff.js'
360
365
 
361
366
  export {
362
367
  assertIsolation,
@@ -415,9 +415,21 @@ Executable tool names, descriptions, and JSON input schemas are attached through
415
415
 
416
416
  return tracer.startActiveSpan(toolSpanName(toolName), {}, parentCtx, async (span) => {
417
417
  try {
418
+ // The call id joins this span to the assistant block that asked
419
+ // for it. Without it a trace shows that `Bash` ran four times
420
+ // this turn and cannot say which span answers which
421
+ // `tool_use` — and the id was already in hand here, threaded
422
+ // through `ToolContext` for the tools that reply
423
+ // asynchronously.
424
+ //
425
+ // Conditional because `toolUseId` is optional: a host calling
426
+ // a tool directly, outside a run, has no call to correlate to,
427
+ // and an attribute set to `undefined` is worse than an absent
428
+ // one — it reaches the exporter as a key with no value.
418
429
  span.setAttributes({
419
430
  [GENAI.TOOL_NAME]: toolName,
420
431
  [GENAI.TOOL_TYPE]: 'function',
432
+ ...(context.toolUseId !== undefined ? { [GENAI.TOOL_CALL_ID]: context.toolUseId } : {}),
421
433
  })
422
434
 
423
435
  const tool = this.getOrThrow(toolName)
@@ -34,6 +34,7 @@ import type {
34
34
  ToolCallRepairReason,
35
35
  } from '../../types/tool/repair.js'
36
36
  import { abortReasonText } from '../../utils/abort.js'
37
+ import { type BackoffPolicy, backoffWithJitter, sleep } from '../../utils/backoff.js'
37
38
  import { toErrorMessage } from '../../utils/error.js'
38
39
  import type { Logger } from '../../utils/logger.js'
39
40
  import { compressShellOutput } from '../../utils/shell-compress.js'
@@ -74,6 +75,33 @@ export const DEFAULT_TOOL_CONCURRENCY = 8
74
75
  */
75
76
  export const HOOK_RETRY_BUDGET = 1
76
77
 
78
+ /**
79
+ * Wait between in-loop tool retry attempts.
80
+ *
81
+ * There was none. A tool that declared itself retryable was re-run the
82
+ * instant it failed, as many times as its budget allowed — and the failures
83
+ * worth retrying are the ones an immediate retry makes worse: a rate limit
84
+ * answers the second call faster than it recovers, a contended lock is still
85
+ * held, a connection that has not finished opening has not finished opening.
86
+ *
87
+ * The numbers are the provider policy's, deliberately, and not because a tool
88
+ * is a model call. Nothing here has been measured against tools specifically,
89
+ * and inventing a second curve to look considered would be a guess wearing
90
+ * different digits; the shared one is at least the curve this codebase has
91
+ * already run in anger. Full jitter draws each wait from `[0, curve]`, so the
92
+ * first retry of a tool with the shipped budget waits under half a second on
93
+ * average.
94
+ *
95
+ * The ceiling is inert at the budgets anyone sets — a tool declaring
96
+ * `maxRetries: 3` never reaches 2s — and binds only a host that sets a large
97
+ * one. Override with {@link ToolExecutorConfig.toolRetryBackoff}; set
98
+ * `initialDelayMs: 0` for the previous no-wait behaviour.
99
+ */
100
+ export const DEFAULT_TOOL_RETRY_BACKOFF: BackoffPolicy = {
101
+ initialDelayMs: 500,
102
+ maxDelayMs: 16_000,
103
+ }
104
+
77
105
  /**
78
106
  * An empty arguments string means "no arguments", not "malformed" — the
79
107
  * shape a no-parameter tool arrives in.
@@ -95,6 +123,16 @@ export interface ToolExecutorConfig {
95
123
  pluginManager?: PluginLifecycleManager
96
124
  /** Run-level default deadline; per-tool `timeoutMs` overrides it. */
97
125
  toolTimeoutMs?: number
126
+ /**
127
+ * Wait between in-loop retries of a failed tool call. Defaults to
128
+ * {@link DEFAULT_TOOL_RETRY_BACKOFF}.
129
+ *
130
+ * Applies only to a tool that opted into retrying at all
131
+ * ({@link ToolDefinition.maxRetries}) or to a `post_tool_use` hook that
132
+ * asked for one, so a run whose tools all take the shipped default of
133
+ * zero retries never sleeps here.
134
+ */
135
+ toolRetryBackoff?: Partial<BackoffPolicy>
98
136
  /** Max concurrently-executing concurrency-safe tools. */
99
137
  maxToolConcurrency?: number
100
138
 
@@ -648,6 +686,10 @@ export class ToolExecutor {
648
686
  // because the SDK cannot know a tool is idempotent — silently
649
687
  // re-running a write or a payment is worse than never retrying.
650
688
  const maxRetries = Math.max(0, this.config.tools.get(toolName)?.maxRetries ?? 0)
689
+ const backoff: BackoffPolicy = {
690
+ ...DEFAULT_TOOL_RETRY_BACKOFF,
691
+ ...this.config.toolRetryBackoff,
692
+ }
651
693
  for (let attempt = 1; ; attempt++) {
652
694
  // A missing file will not appear on the second attempt; burning
653
695
  // the budget on it only delays the error the model needs to see.
@@ -664,14 +706,46 @@ export class ToolExecutor {
664
706
  const budget = post.retry ? Math.max(maxRetries, HOOK_RETRY_BUDGET) : maxRetries
665
707
  if (attempt > budget) break
666
708
 
709
+ // Wait before trying again, on the curve the provider path has
710
+ // used all along. This loop had NO delay: a tool failing on a
711
+ // transient condition — a rate-limited HTTP call, a lock, a cold
712
+ // connection — was re-run immediately, several times, which is the
713
+ // pattern most likely to prolong the very condition it is retrying
714
+ // against.
715
+ //
716
+ // Full jitter rather than a fixed wait, and the concurrency that
717
+ // makes it matter is one this loop creates itself: a model emits a
718
+ // batch of parallel calls, `executeBatch` runs up to
719
+ // DEFAULT_TOOL_CONCURRENCY of them at once, they hit the same
720
+ // rate-limited endpoint and fail together. A fixed backoff would
721
+ // resynchronise that batch on every attempt.
722
+ //
723
+ // `attempt` is 1-based here and `backoffWithJitter` is 0-based, so
724
+ // the first retry draws from `[0, initialDelayMs]`.
725
+ const delayMs = backoffWithJitter(attempt - 1, backoff)
726
+
667
727
  this.log.info('Retrying a failed tool call', {
668
728
  runId: this.config.runId,
669
729
  tool: toolName,
670
730
  attempt,
671
731
  budget,
672
732
  requestedByHook: post.retry,
733
+ delayMs,
673
734
  error: result.error,
674
735
  })
736
+
737
+ try {
738
+ await sleep(delayMs, this.config.abortSignal)
739
+ } catch {
740
+ // Stopped mid-backoff. Give up retrying and let the failure
741
+ // already in `result` be this call's answer, rather than
742
+ // throwing: every `tool_use` must be answered by a
743
+ // `tool_result` with the same id, and an abort escaping from
744
+ // here would leave this one open in the transcript for a
745
+ // resume to trip over.
746
+ break
747
+ }
748
+
675
749
  result = await this.runOnce(toolName, input, toolContext)
676
750
  post = await this.runPostToolHook(toolName, input, result)
677
751
  }
@@ -83,6 +83,7 @@ import type { TaskStore } from '../../types/task/index.js'
83
83
  import type { ToolRegistryContract } from '../../types/tool/index.js'
84
84
  import type { RepairToolCall } from '../../types/tool/repair.js'
85
85
  import type { VerificationGateConfig } from '../../types/verification/index.js'
86
+ import type { BackoffPolicy } from '../../utils/backoff.js'
86
87
  import type { ModelPricing } from '../../utils/cost.js'
87
88
  import { getRootLogger } from '../../utils/logger.js'
88
89
  import { VerificationGate } from '../../verification/gate.js'
@@ -98,7 +99,7 @@ import { isWorkingMemoryMessage } from './iteration/phases/working-memory.js'
98
99
  import { applyLifecycleHookResults } from './plugin-hooks.js'
99
100
  import { PromptBuilder } from './prompt.js'
100
101
  import type { PromptSegments } from './prompt.js'
101
- import type { PendingAnswers, QuestionParkBinding } from './question-park.js'
102
+ import { PendingAnswers, QuestionParkBinding } from './question-park.js'
102
103
  import { ResultAssembler } from './result.js'
103
104
  import {
104
105
  type PendingResumePlan,
@@ -168,14 +169,20 @@ export interface QueryParams {
168
169
  emergencySave?: boolean
169
170
 
170
171
  /**
171
- * Durability for questions raised from inside a tool.
172
+ * Durability for questions raised by a tool that closed over its
173
+ * binding before the run existed.
172
174
  *
173
- * The tool that asks is built before the run exists, so the binding is
174
- * created by whoever builds the tools and attached here — that is what
175
- * lets one tool instance be durable inside a run and inert outside one.
176
- * Without it, a question park exists only as a suspended `await`: kill
177
- * the process while somebody is looking at the card and the answer can
178
- * never be applied.
175
+ * The built-in `ask_user_question` is built with the agent's tool
176
+ * registry, so only whoever builds the tools can hand it one — that is
177
+ * what lets a single tool instance be durable inside a run and inert
178
+ * outside one. Without it, THAT tool's park is only a suspended
179
+ * `await`: kill the process while somebody is looking at the card and
180
+ * the answer can never be applied.
181
+ *
182
+ * Not required for `ToolContext.requestPause`. The run builds that
183
+ * seam per call and binds its own recorder when none is passed, so a
184
+ * pause raised from a host-authored tool is durable on every surface
185
+ * rather than only on the one agent class that supplies this.
179
186
  */
180
187
  questionParks?: QuestionParkBinding
181
188
 
@@ -192,16 +199,28 @@ export interface QueryParams {
192
199
  /**
193
200
  * The registry a re-entered `ask_user_question` reads its answer from.
194
201
  *
195
- * Same shape as {@link questionParks}: the tool is built before the run
196
- * exists, so the instance is created by whoever builds the tools and
197
- * filled here on the resume path. Without it a resumed run re-asks a
198
- * question the user already answered.
202
+ * Same shape, same reason and same limit as {@link questionParks}: it
203
+ * exists for a tool that closed over the instance before the run did,
204
+ * and without it a resumed run re-asks that tool's question. A pause
205
+ * from `ToolContext.requestPause` needs none, because the run fills
206
+ * its own on the resume path.
199
207
  */
200
208
  pendingAnswers?: PendingAnswers
201
209
 
202
210
  /** Default per-tool execution deadline. See {@link ToolDefinition.timeoutMs}. */
203
211
  toolTimeoutMs?: number
204
212
 
213
+ /**
214
+ * Wait between in-loop retries of a failed tool call, with full jitter.
215
+ * Defaults to {@link DEFAULT_TOOL_RETRY_BACKOFF}.
216
+ *
217
+ * Only reached by a tool that opted into retrying
218
+ * ({@link ToolDefinition.maxRetries}) or a `post_tool_use` hook that asked
219
+ * for one. Set `initialDelayMs: 0` for the retry-immediately behaviour
220
+ * this loop had before it had any backoff at all.
221
+ */
222
+ toolRetryBackoff?: Partial<BackoffPolicy>
223
+
205
224
  /** Max concurrently-executing concurrency-safe tools in one batch. */
206
225
  maxToolConcurrency?: number
207
226
 
@@ -874,6 +893,25 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
874
893
  ? []
875
894
  : withDeferredDiscoveryTool(params.tools, params.allowedTools)
876
895
 
896
+ // The two halves of a durable pause, owned by the RUN when the host
897
+ // does not own them.
898
+ //
899
+ // `SupervisorAgent` builds both before the run exists, because the
900
+ // tools it builds close over them, and it passes them in. Nothing else
901
+ // could: neither type is exported from `public-runtime.ts`, so a host
902
+ // on `ReactiveAgent`, `drainQuery` or `resumeRun` had no way to supply
903
+ // either — and `ToolContext.requestPause`, which every tool author is
904
+ // handed, silently wrote no checkpoint and could receive no answer on
905
+ // those surfaces. Which agent class the host happened to pick is not
906
+ // visible at the call site, so the degradation was invisible too.
907
+ //
908
+ // A run-local pair is enough for the general seam because `query()`
909
+ // builds its `createToolPause` itself, below, and can hand it the
910
+ // run's own. Pinned by the "a pause is durable on any surface" cases
911
+ // in `__tests__/tool-pause-resume.test.ts`.
912
+ const questionParks = params.questionParks ?? new QuestionParkBinding()
913
+ const pendingAnswers = params.pendingAnswers ?? new PendingAnswers()
914
+
877
915
  // is null only when the run has no disk layout (tests,
878
916
  // in-memory hosts); the budget then degrades to middle-elision.
879
917
  const runDirForTools = ctx.runMgr.getRunDir()
@@ -891,6 +929,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
891
929
  invocationState: params.invocationState,
892
930
  pluginManager: params.pluginManager,
893
931
  ...(params.toolTimeoutMs !== undefined ? { toolTimeoutMs: params.toolTimeoutMs } : {}),
932
+ ...(params.toolRetryBackoff !== undefined
933
+ ? { toolRetryBackoff: params.toolRetryBackoff }
934
+ : {}),
894
935
  ...(params.maxToolConcurrency !== undefined
895
936
  ? { maxToolConcurrency: params.maxToolConcurrency }
896
937
  : {}),
@@ -915,8 +956,8 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
915
956
  runId: ctx.runId,
916
957
  toolUseId,
917
958
  parkHandler: params.resumeHandler,
918
- ...(params.questionParks ? { recorder: params.questionParks } : {}),
919
- ...(params.pendingAnswers ? { pendingAnswers: params.pendingAnswers } : {}),
959
+ recorder: questionParks,
960
+ pendingAnswers,
920
961
  }),
921
962
  },
922
963
  ctx.activityStore,
@@ -1130,7 +1171,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1130
1171
  // the checkpoint did not exist: nothing on disk said a human owed
1131
1172
  // this run an answer, and a remote host could not observe the
1132
1173
  // question at all.
1133
- params.questionParks?.bind({
1174
+ questionParks.bind({
1134
1175
  record: async (question) => {
1135
1176
  try {
1136
1177
  const checkpoint = await checkpointMgr.create(ctx.runMgr, ctx.runMgr.currentIteration)
@@ -1537,9 +1578,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1537
1578
  // closed over its registry when the agent was constructed,
1538
1579
  // long before this run existed, so the answers are copied in
1539
1580
  // rather than passed down.
1540
- if (pendingResume.answers && params.pendingAnswers) {
1581
+ if (pendingResume.answers) {
1541
1582
  for (const [questionId, answer] of pendingResume.answers.entries()) {
1542
- params.pendingAnswers.set(questionId, answer)
1583
+ pendingAnswers.set(questionId, answer)
1543
1584
  }
1544
1585
  }
1545
1586
 
@@ -1642,7 +1683,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1642
1683
  // Same reasoning for the question channel: the tools outlive the
1643
1684
  // run that bound them, so leaving it attached would have a later
1644
1685
  // run's question written into this run's checkpoint store.
1645
- params.questionParks?.unbind()
1686
+ questionParks.unbind()
1646
1687
 
1647
1688
  // Offer what the run learned to whoever decides what is worth
1648
1689
  // keeping. In `finally` and awaited: a run that failed still