@namzu/sdk 23.0.0 → 26.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +208 -0
- package/README.md +1 -1
- package/dist/agents/ReactiveAgent.d.ts.map +1 -1
- package/dist/agents/ReactiveAgent.js +3 -0
- package/dist/agents/ReactiveAgent.js.map +1 -1
- package/dist/connector/mcp/stdio.d.ts +11 -0
- package/dist/connector/mcp/stdio.d.ts.map +1 -1
- package/dist/connector/mcp/stdio.js +106 -1
- package/dist/connector/mcp/stdio.js.map +1 -1
- package/dist/constants/telemetry/index.d.ts +15 -1
- package/dist/constants/telemetry/index.d.ts.map +1 -1
- package/dist/constants/telemetry/index.js +15 -1
- package/dist/constants/telemetry/index.js.map +1 -1
- package/dist/provider/fallback.d.ts.map +1 -1
- package/dist/provider/fallback.js +8 -2
- package/dist/provider/fallback.js.map +1 -1
- package/dist/provider/retry.d.ts +10 -5
- package/dist/provider/retry.d.ts.map +1 -1
- package/dist/provider/retry.js +11 -31
- package/dist/provider/retry.js.map +1 -1
- package/dist/public-runtime.d.ts +1 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +12 -0
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/runtime/query/executor.d.ts +34 -0
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +61 -0
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/index.d.ts +31 -13
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +28 -6
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +178 -20
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.d.ts +14 -1
- package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.js +19 -3
- package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
- package/dist/runtime/query/resume-pending.d.ts.map +1 -1
- package/dist/runtime/query/resume-pending.js +10 -1
- package/dist/runtime/query/resume-pending.js.map +1 -1
- package/dist/runtime/query/tool-pause.d.ts +35 -0
- package/dist/runtime/query/tool-pause.d.ts.map +1 -1
- package/dist/runtime/query/tool-pause.js +35 -0
- package/dist/runtime/query/tool-pause.js.map +1 -1
- package/dist/runtime/query/tooling.d.ts +2 -0
- package/dist/runtime/query/tooling.d.ts.map +1 -1
- package/dist/runtime/query/tooling.js +3 -0
- package/dist/runtime/query/tooling.js.map +1 -1
- package/dist/types/agent/reactive.d.ts +1 -0
- package/dist/types/agent/reactive.d.ts.map +1 -1
- package/dist/types/connector/mcp.d.ts +17 -0
- package/dist/types/connector/mcp.d.ts.map +1 -1
- package/dist/types/provider/interface.d.ts +23 -2
- package/dist/types/provider/interface.d.ts.map +1 -1
- package/dist/types/run/step.d.ts +75 -3
- package/dist/types/run/step.d.ts.map +1 -1
- package/dist/types/run/step.js.map +1 -1
- package/dist/utils/backoff.d.ts +44 -0
- package/dist/utils/backoff.d.ts.map +1 -0
- package/dist/utils/backoff.js +58 -0
- package/dist/utils/backoff.js.map +1 -0
- package/package.json +1 -1
- package/src/agents/ReactiveAgent.ts +3 -0
- package/src/connector/mcp/stdio.ts +106 -1
- package/src/constants/telemetry/index.ts +16 -1
- package/src/provider/fallback.ts +8 -2
- package/src/provider/retry.ts +20 -37
- package/src/public-runtime.ts +5 -0
- package/src/registry/tool/execute.ts +12 -0
- package/src/runtime/query/executor.ts +74 -0
- package/src/runtime/query/index.ts +59 -18
- package/src/runtime/query/iteration/index.ts +199 -22
- package/src/runtime/query/iteration/stream-turn.ts +24 -3
- package/src/runtime/query/resume-pending.ts +10 -1
- package/src/runtime/query/tool-pause.ts +37 -0
- package/src/runtime/query/tooling.ts +5 -0
- package/src/types/agent/reactive.ts +1 -0
- package/src/types/connector/mcp.ts +17 -0
- package/src/types/provider/interface.ts +23 -2
- package/src/types/run/step.ts +76 -3
- package/src/utils/backoff.ts +70 -0
|
@@ -13,6 +13,111 @@ import { type Logger, getRootLogger } from '../../utils/logger.js'
|
|
|
13
13
|
*/
|
|
14
14
|
const TERMINATE_GRACE_MS = 2_000
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* The variables a child needs in order to be a working process at all.
|
|
18
|
+
*
|
|
19
|
+
* This spawn used to pass `{ ...process.env, ...config.env }`, so a connected
|
|
20
|
+
* server received every credential the host happened to hold — measured at 119
|
|
21
|
+
* variables on a developer machine, including a planted secret the server had
|
|
22
|
+
* no reason to see. A server that needs one token was handed all of them, and
|
|
23
|
+
* nothing in the config said so.
|
|
24
|
+
*
|
|
25
|
+
* The list below is process plumbing, not secrets: where to find executables,
|
|
26
|
+
* where the home and temp directories are, what the locale is. Dropping any of
|
|
27
|
+
* it does not harden anything and does break servers — a child with no `PATH`
|
|
28
|
+
* cannot resolve its own interpreter.
|
|
29
|
+
*
|
|
30
|
+
* Anything else a server needs is now named: `env` gives it a literal value,
|
|
31
|
+
* and `inheritEnv` names a parent variable to pass through. Naming is the
|
|
32
|
+
* point — a grant that has to be written down is a grant somebody can review.
|
|
33
|
+
*
|
|
34
|
+
* Windows spellings are matched case-insensitively, because its environment is
|
|
35
|
+
* case-insensitive and a lookup for `Path` against a key stored as `PATH` would
|
|
36
|
+
* silently drop it — which would present as "the server does not start on
|
|
37
|
+
* Windows" rather than as anything to do with this list.
|
|
38
|
+
*/
|
|
39
|
+
const BASE_ENV_KEYS: readonly string[] = [
|
|
40
|
+
// Everywhere.
|
|
41
|
+
'PATH',
|
|
42
|
+
'LANG',
|
|
43
|
+
'LC_ALL',
|
|
44
|
+
'LC_CTYPE',
|
|
45
|
+
'TZ',
|
|
46
|
+
// POSIX.
|
|
47
|
+
'HOME',
|
|
48
|
+
'SHELL',
|
|
49
|
+
'TMPDIR',
|
|
50
|
+
'USER',
|
|
51
|
+
'LOGNAME',
|
|
52
|
+
// Windows. `SystemRoot` and `ComSpec` are load-bearing: without them a
|
|
53
|
+
// child cannot resolve system DLLs or the command interpreter.
|
|
54
|
+
'PATHEXT',
|
|
55
|
+
'SystemRoot',
|
|
56
|
+
'SystemDrive',
|
|
57
|
+
'ComSpec',
|
|
58
|
+
'WINDIR',
|
|
59
|
+
'TEMP',
|
|
60
|
+
'TMP',
|
|
61
|
+
'USERPROFILE',
|
|
62
|
+
'HOMEDRIVE',
|
|
63
|
+
'HOMEPATH',
|
|
64
|
+
'APPDATA',
|
|
65
|
+
'LOCALAPPDATA',
|
|
66
|
+
'PROGRAMDATA',
|
|
67
|
+
'PROGRAMFILES',
|
|
68
|
+
'NUMBER_OF_PROCESSORS',
|
|
69
|
+
'PROCESSOR_ARCHITECTURE',
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Read one variable from the parent, honouring the platform's own casing rules.
|
|
74
|
+
*
|
|
75
|
+
* Node exposes `process.env` on Windows through a case-insensitive proxy, so a
|
|
76
|
+
* direct lookup already works there — but the KEY this returns has to be the
|
|
77
|
+
* one the parent actually uses, or a child comparing key names sees a spelling
|
|
78
|
+
* the host never set.
|
|
79
|
+
*/
|
|
80
|
+
function readParentVar(source: NodeJS.ProcessEnv, name: string): [string, string] | undefined {
|
|
81
|
+
const direct = source[name]
|
|
82
|
+
if (direct !== undefined) return [name, direct]
|
|
83
|
+
if (process.platform !== 'win32') return undefined
|
|
84
|
+
const lowered = name.toLowerCase()
|
|
85
|
+
for (const [key, value] of Object.entries(source)) {
|
|
86
|
+
if (key.toLowerCase() === lowered && value !== undefined) return [key, value]
|
|
87
|
+
}
|
|
88
|
+
return undefined
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* What the child is handed: plumbing, then the named inheritances, then the
|
|
93
|
+
* literal values.
|
|
94
|
+
*
|
|
95
|
+
* Later wins, and the order is the precedence an operator would guess: a
|
|
96
|
+
* literal `env` entry overrides an inherited one, and both override the base.
|
|
97
|
+
* Exported for the tests, which assert on the ENV rather than on the spawn —
|
|
98
|
+
* a test that only checked the config was accepted would have passed against
|
|
99
|
+
* the version this replaces.
|
|
100
|
+
*/
|
|
101
|
+
export function buildChildEnv(
|
|
102
|
+
config: Pick<MCPStdioTransportConfig, 'env' | 'inheritEnv'>,
|
|
103
|
+
source: NodeJS.ProcessEnv = process.env,
|
|
104
|
+
): Record<string, string> {
|
|
105
|
+
const env: Record<string, string> = {}
|
|
106
|
+
for (const name of BASE_ENV_KEYS) {
|
|
107
|
+
const found = readParentVar(source, name)
|
|
108
|
+
if (found) env[found[0]] = found[1]
|
|
109
|
+
}
|
|
110
|
+
for (const name of config.inheritEnv ?? []) {
|
|
111
|
+
const found = readParentVar(source, name)
|
|
112
|
+
// A named variable the parent does not hold is simply absent. Refusing
|
|
113
|
+
// the spawn would turn an optional credential into a startup failure,
|
|
114
|
+
// and inventing an empty string would tell the server it has one.
|
|
115
|
+
if (found) env[found[0]] = found[1]
|
|
116
|
+
}
|
|
117
|
+
for (const [name, value] of Object.entries(config.env ?? {})) env[name] = value
|
|
118
|
+
return env
|
|
119
|
+
}
|
|
120
|
+
|
|
16
121
|
export class StdioTransport implements MCPTransport {
|
|
17
122
|
private process: ChildProcess | null = null
|
|
18
123
|
private messageHandlers: Array<(message: MCPJsonRpcMessage) => void> = []
|
|
@@ -32,7 +137,7 @@ export class StdioTransport implements MCPTransport {
|
|
|
32
137
|
if (this.connected) return
|
|
33
138
|
|
|
34
139
|
this.process = spawn(this.config.command, this.config.args ?? [], {
|
|
35
|
-
env:
|
|
140
|
+
env: buildChildEnv(this.config),
|
|
36
141
|
cwd: this.config.cwd,
|
|
37
142
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
38
143
|
})
|
|
@@ -23,7 +23,22 @@ export const GENAI = {
|
|
|
23
23
|
|
|
24
24
|
TOOL_NAME: 'gen_ai.tool.name',
|
|
25
25
|
TOOL_TYPE: 'gen_ai.tool.type',
|
|
26
|
-
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The id of the tool call this span is about.
|
|
29
|
+
*
|
|
30
|
+
* Spelled `call.id`, matching the registry and matching the two
|
|
31
|
+
* neighbours above — this read `gen_ai.tool.call_id`, one underscore
|
|
32
|
+
* where the convention has a dot, so a consumer grouping by the
|
|
33
|
+
* conventional name found nothing under it. Nothing errored, because a
|
|
34
|
+
* span attribute is a free-form key and a wrong one is simply a key
|
|
35
|
+
* nobody asked for.
|
|
36
|
+
*
|
|
37
|
+
* Pinned by `telemetry/__tests__/tool-call-id-attribute.test.ts`, which
|
|
38
|
+
* also drives the emitter: the spelling was only half the defect, and
|
|
39
|
+
* the constant had no writer at all.
|
|
40
|
+
*/
|
|
41
|
+
TOOL_CALL_ID: 'gen_ai.tool.call.id',
|
|
27
42
|
|
|
28
43
|
AGENT_NAME: 'gen_ai.agent.name',
|
|
29
44
|
AGENT_ID: 'gen_ai.agent.id',
|
package/src/provider/fallback.ts
CHANGED
|
@@ -382,7 +382,13 @@ export function withProviderFallback(
|
|
|
382
382
|
},
|
|
383
383
|
chatStream,
|
|
384
384
|
...(first.provider.listModels ? { listModels: () => first.provider.listModels?.() } : {}),
|
|
385
|
-
|
|
386
|
-
|
|
385
|
+
// Forwarded for the reason `retry.ts` forwards it: a wrapper that drops
|
|
386
|
+
// the model turns a probe the caller could answer into one it cannot.
|
|
387
|
+
...(first.provider.healthCheck
|
|
388
|
+
? { healthCheck: (model?: string) => first.provider.healthCheck?.(model) }
|
|
389
|
+
: {}),
|
|
390
|
+
...(first.provider.doctorCheck
|
|
391
|
+
? { doctorCheck: (model?: string) => first.provider.doctorCheck?.(model) }
|
|
392
|
+
: {}),
|
|
387
393
|
} as LLMProvider
|
|
388
394
|
}
|
package/src/provider/retry.ts
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
import { classifyProviderError, isAbortError } from '../types/provider/errors.js'
|
|
2
2
|
import type { ChatCompletionParams, LLMProvider, StreamChunk } from '../types/provider/index.js'
|
|
3
|
+
import { type BackoffPolicy, backoffWithJitter, sleep } from '../utils/backoff.js'
|
|
3
4
|
import type { Logger } from '../utils/logger.js'
|
|
4
5
|
import { isProviderRequestError } from './errors.js'
|
|
5
6
|
|
|
6
|
-
|
|
7
|
+
/**
|
|
8
|
+
* `BackoffPolicy` plus the provider-specific parts: how many attempts, and
|
|
9
|
+
* how long a server-directed wait may be honoured for.
|
|
10
|
+
*
|
|
11
|
+
* The curve itself lives in `utils/backoff.ts` and is shared with the tool
|
|
12
|
+
* executor's in-loop retry — which had no backoff at all while this one was
|
|
13
|
+
* being careful about jitter two directories away.
|
|
14
|
+
*/
|
|
15
|
+
export interface ProviderRetryConfig extends BackoffPolicy {
|
|
7
16
|
/** Retry attempts AFTER the initial try. `0` disables retrying. */
|
|
8
17
|
readonly maxRetries: number
|
|
9
|
-
/** First backoff, doubled each attempt. */
|
|
10
|
-
readonly initialDelayMs: number
|
|
11
|
-
/** Ceiling for a single backoff, before jitter. */
|
|
12
|
-
readonly maxDelayMs: number
|
|
13
18
|
/**
|
|
14
19
|
* Cap on a server-directed `Retry-After`. A provider asking for 15
|
|
15
20
|
* minutes should not silently park an interactive run for 15 minutes;
|
|
@@ -34,36 +39,6 @@ export const DEFAULT_PROVIDER_RETRY: ProviderRetryConfig = {
|
|
|
34
39
|
maxRetryAfterMs: 60_000,
|
|
35
40
|
}
|
|
36
41
|
|
|
37
|
-
/**
|
|
38
|
-
* Full jitter (AWS's formulation): sleep a uniform random amount in
|
|
39
|
-
* `[0, backoff]` rather than `backoff` exactly. Equal-jitter and no-jitter
|
|
40
|
-
* both keep a fleet of clients that failed together retrying together;
|
|
41
|
-
* full jitter is what actually spreads a thundering herd.
|
|
42
|
-
*/
|
|
43
|
-
function backoffWithJitter(attempt: number, config: ProviderRetryConfig, random: () => number) {
|
|
44
|
-
const exponential = Math.min(config.initialDelayMs * 2 ** attempt, config.maxDelayMs)
|
|
45
|
-
return Math.round(random() * exponential)
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
49
|
-
if (ms <= 0) return Promise.resolve()
|
|
50
|
-
return new Promise<void>((resolve, reject) => {
|
|
51
|
-
if (signal?.aborted) {
|
|
52
|
-
reject(signal.reason)
|
|
53
|
-
return
|
|
54
|
-
}
|
|
55
|
-
const timer = setTimeout(() => {
|
|
56
|
-
signal?.removeEventListener('abort', onAbort)
|
|
57
|
-
resolve()
|
|
58
|
-
}, ms)
|
|
59
|
-
const onAbort = () => {
|
|
60
|
-
clearTimeout(timer)
|
|
61
|
-
reject(signal?.reason)
|
|
62
|
-
}
|
|
63
|
-
signal?.addEventListener('abort', onAbort, { once: true })
|
|
64
|
-
})
|
|
65
|
-
}
|
|
66
|
-
|
|
67
42
|
export interface WithProviderRetryOptions {
|
|
68
43
|
readonly config?: Partial<ProviderRetryConfig>
|
|
69
44
|
readonly log?: Logger
|
|
@@ -256,7 +231,15 @@ export function withProviderRetry(
|
|
|
256
231
|
},
|
|
257
232
|
chatStream,
|
|
258
233
|
...(provider.listModels ? { listModels: () => provider.listModels?.() } : {}),
|
|
259
|
-
|
|
260
|
-
|
|
234
|
+
// The model is forwarded, not dropped. A wrapper that swallowed it would
|
|
235
|
+
// leave the wrapped driver probing whatever it probes with no argument
|
|
236
|
+
// — which for at least one driver is "nothing", so the check would come
|
|
237
|
+
// back unanswerable purely because it was wrapped.
|
|
238
|
+
...(provider.healthCheck
|
|
239
|
+
? { healthCheck: (model?: string) => provider.healthCheck?.(model) }
|
|
240
|
+
: {}),
|
|
241
|
+
...(provider.doctorCheck
|
|
242
|
+
? { doctorCheck: (model?: string) => provider.doctorCheck?.(model) }
|
|
243
|
+
: {}),
|
|
261
244
|
} as LLMProvider
|
|
262
245
|
}
|
package/src/public-runtime.ts
CHANGED
|
@@ -357,6 +357,11 @@ export type {
|
|
|
357
357
|
WithProviderFallbackOptions,
|
|
358
358
|
WithProviderRetryOptions,
|
|
359
359
|
} from './provider/index.js'
|
|
360
|
+
// The curve `ProviderRetryConfig` extends and `query({ toolRetryBackoff })`
|
|
361
|
+
// takes a partial of. Both public surfaces name it, so a consumer has to be
|
|
362
|
+
// able to name it too — a type reachable only through an inline `import(...)`
|
|
363
|
+
// in a `.d.ts` is not a type anyone writes down.
|
|
364
|
+
export type { BackoffPolicy } from './utils/backoff.js'
|
|
360
365
|
|
|
361
366
|
export {
|
|
362
367
|
assertIsolation,
|
|
@@ -415,9 +415,21 @@ Executable tool names, descriptions, and JSON input schemas are attached through
|
|
|
415
415
|
|
|
416
416
|
return tracer.startActiveSpan(toolSpanName(toolName), {}, parentCtx, async (span) => {
|
|
417
417
|
try {
|
|
418
|
+
// The call id joins this span to the assistant block that asked
|
|
419
|
+
// for it. Without it a trace shows that `Bash` ran four times
|
|
420
|
+
// this turn and cannot say which span answers which
|
|
421
|
+
// `tool_use` — and the id was already in hand here, threaded
|
|
422
|
+
// through `ToolContext` for the tools that reply
|
|
423
|
+
// asynchronously.
|
|
424
|
+
//
|
|
425
|
+
// Conditional because `toolUseId` is optional: a host calling
|
|
426
|
+
// a tool directly, outside a run, has no call to correlate to,
|
|
427
|
+
// and an attribute set to `undefined` is worse than an absent
|
|
428
|
+
// one — it reaches the exporter as a key with no value.
|
|
418
429
|
span.setAttributes({
|
|
419
430
|
[GENAI.TOOL_NAME]: toolName,
|
|
420
431
|
[GENAI.TOOL_TYPE]: 'function',
|
|
432
|
+
...(context.toolUseId !== undefined ? { [GENAI.TOOL_CALL_ID]: context.toolUseId } : {}),
|
|
421
433
|
})
|
|
422
434
|
|
|
423
435
|
const tool = this.getOrThrow(toolName)
|
|
@@ -34,6 +34,7 @@ import type {
|
|
|
34
34
|
ToolCallRepairReason,
|
|
35
35
|
} from '../../types/tool/repair.js'
|
|
36
36
|
import { abortReasonText } from '../../utils/abort.js'
|
|
37
|
+
import { type BackoffPolicy, backoffWithJitter, sleep } from '../../utils/backoff.js'
|
|
37
38
|
import { toErrorMessage } from '../../utils/error.js'
|
|
38
39
|
import type { Logger } from '../../utils/logger.js'
|
|
39
40
|
import { compressShellOutput } from '../../utils/shell-compress.js'
|
|
@@ -74,6 +75,33 @@ export const DEFAULT_TOOL_CONCURRENCY = 8
|
|
|
74
75
|
*/
|
|
75
76
|
export const HOOK_RETRY_BUDGET = 1
|
|
76
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Wait between in-loop tool retry attempts.
|
|
80
|
+
*
|
|
81
|
+
* There was none. A tool that declared itself retryable was re-run the
|
|
82
|
+
* instant it failed, as many times as its budget allowed — and the failures
|
|
83
|
+
* worth retrying are the ones an immediate retry makes worse: a rate limit
|
|
84
|
+
* answers the second call faster than it recovers, a contended lock is still
|
|
85
|
+
* held, a connection that has not finished opening has not finished opening.
|
|
86
|
+
*
|
|
87
|
+
* The numbers are the provider policy's, deliberately, and not because a tool
|
|
88
|
+
* is a model call. Nothing here has been measured against tools specifically,
|
|
89
|
+
* and inventing a second curve to look considered would be a guess wearing
|
|
90
|
+
* different digits; the shared one is at least the curve this codebase has
|
|
91
|
+
* already run in anger. Full jitter draws each wait from `[0, curve]`, so the
|
|
92
|
+
* first retry of a tool with the shipped budget waits under half a second on
|
|
93
|
+
* average.
|
|
94
|
+
*
|
|
95
|
+
* The ceiling is inert at the budgets anyone sets — a tool declaring
|
|
96
|
+
* `maxRetries: 3` never reaches 2s — and binds only a host that sets a large
|
|
97
|
+
* one. Override with {@link ToolExecutorConfig.toolRetryBackoff}; set
|
|
98
|
+
* `initialDelayMs: 0` for the previous no-wait behaviour.
|
|
99
|
+
*/
|
|
100
|
+
export const DEFAULT_TOOL_RETRY_BACKOFF: BackoffPolicy = {
|
|
101
|
+
initialDelayMs: 500,
|
|
102
|
+
maxDelayMs: 16_000,
|
|
103
|
+
}
|
|
104
|
+
|
|
77
105
|
/**
|
|
78
106
|
* An empty arguments string means "no arguments", not "malformed" — the
|
|
79
107
|
* shape a no-parameter tool arrives in.
|
|
@@ -95,6 +123,16 @@ export interface ToolExecutorConfig {
|
|
|
95
123
|
pluginManager?: PluginLifecycleManager
|
|
96
124
|
/** Run-level default deadline; per-tool `timeoutMs` overrides it. */
|
|
97
125
|
toolTimeoutMs?: number
|
|
126
|
+
/**
|
|
127
|
+
* Wait between in-loop retries of a failed tool call. Defaults to
|
|
128
|
+
* {@link DEFAULT_TOOL_RETRY_BACKOFF}.
|
|
129
|
+
*
|
|
130
|
+
* Applies only to a tool that opted into retrying at all
|
|
131
|
+
* ({@link ToolDefinition.maxRetries}) or to a `post_tool_use` hook that
|
|
132
|
+
* asked for one, so a run whose tools all take the shipped default of
|
|
133
|
+
* zero retries never sleeps here.
|
|
134
|
+
*/
|
|
135
|
+
toolRetryBackoff?: Partial<BackoffPolicy>
|
|
98
136
|
/** Max concurrently-executing concurrency-safe tools. */
|
|
99
137
|
maxToolConcurrency?: number
|
|
100
138
|
|
|
@@ -648,6 +686,10 @@ export class ToolExecutor {
|
|
|
648
686
|
// because the SDK cannot know a tool is idempotent — silently
|
|
649
687
|
// re-running a write or a payment is worse than never retrying.
|
|
650
688
|
const maxRetries = Math.max(0, this.config.tools.get(toolName)?.maxRetries ?? 0)
|
|
689
|
+
const backoff: BackoffPolicy = {
|
|
690
|
+
...DEFAULT_TOOL_RETRY_BACKOFF,
|
|
691
|
+
...this.config.toolRetryBackoff,
|
|
692
|
+
}
|
|
651
693
|
for (let attempt = 1; ; attempt++) {
|
|
652
694
|
// A missing file will not appear on the second attempt; burning
|
|
653
695
|
// the budget on it only delays the error the model needs to see.
|
|
@@ -664,14 +706,46 @@ export class ToolExecutor {
|
|
|
664
706
|
const budget = post.retry ? Math.max(maxRetries, HOOK_RETRY_BUDGET) : maxRetries
|
|
665
707
|
if (attempt > budget) break
|
|
666
708
|
|
|
709
|
+
// Wait before trying again, on the curve the provider path has
|
|
710
|
+
// used all along. This loop had NO delay: a tool failing on a
|
|
711
|
+
// transient condition — a rate-limited HTTP call, a lock, a cold
|
|
712
|
+
// connection — was re-run immediately, several times, which is the
|
|
713
|
+
// pattern most likely to prolong the very condition it is retrying
|
|
714
|
+
// against.
|
|
715
|
+
//
|
|
716
|
+
// Full jitter rather than a fixed wait, and the concurrency that
|
|
717
|
+
// makes it matter is one this loop creates itself: a model emits a
|
|
718
|
+
// batch of parallel calls, `executeBatch` runs up to
|
|
719
|
+
// DEFAULT_TOOL_CONCURRENCY of them at once, they hit the same
|
|
720
|
+
// rate-limited endpoint and fail together. A fixed backoff would
|
|
721
|
+
// resynchronise that batch on every attempt.
|
|
722
|
+
//
|
|
723
|
+
// `attempt` is 1-based here and `backoffWithJitter` is 0-based, so
|
|
724
|
+
// the first retry draws from `[0, initialDelayMs]`.
|
|
725
|
+
const delayMs = backoffWithJitter(attempt - 1, backoff)
|
|
726
|
+
|
|
667
727
|
this.log.info('Retrying a failed tool call', {
|
|
668
728
|
runId: this.config.runId,
|
|
669
729
|
tool: toolName,
|
|
670
730
|
attempt,
|
|
671
731
|
budget,
|
|
672
732
|
requestedByHook: post.retry,
|
|
733
|
+
delayMs,
|
|
673
734
|
error: result.error,
|
|
674
735
|
})
|
|
736
|
+
|
|
737
|
+
try {
|
|
738
|
+
await sleep(delayMs, this.config.abortSignal)
|
|
739
|
+
} catch {
|
|
740
|
+
// Stopped mid-backoff. Give up retrying and let the failure
|
|
741
|
+
// already in `result` be this call's answer, rather than
|
|
742
|
+
// throwing: every `tool_use` must be answered by a
|
|
743
|
+
// `tool_result` with the same id, and an abort escaping from
|
|
744
|
+
// here would leave this one open in the transcript for a
|
|
745
|
+
// resume to trip over.
|
|
746
|
+
break
|
|
747
|
+
}
|
|
748
|
+
|
|
675
749
|
result = await this.runOnce(toolName, input, toolContext)
|
|
676
750
|
post = await this.runPostToolHook(toolName, input, result)
|
|
677
751
|
}
|
|
@@ -83,6 +83,7 @@ import type { TaskStore } from '../../types/task/index.js'
|
|
|
83
83
|
import type { ToolRegistryContract } from '../../types/tool/index.js'
|
|
84
84
|
import type { RepairToolCall } from '../../types/tool/repair.js'
|
|
85
85
|
import type { VerificationGateConfig } from '../../types/verification/index.js'
|
|
86
|
+
import type { BackoffPolicy } from '../../utils/backoff.js'
|
|
86
87
|
import type { ModelPricing } from '../../utils/cost.js'
|
|
87
88
|
import { getRootLogger } from '../../utils/logger.js'
|
|
88
89
|
import { VerificationGate } from '../../verification/gate.js'
|
|
@@ -98,7 +99,7 @@ import { isWorkingMemoryMessage } from './iteration/phases/working-memory.js'
|
|
|
98
99
|
import { applyLifecycleHookResults } from './plugin-hooks.js'
|
|
99
100
|
import { PromptBuilder } from './prompt.js'
|
|
100
101
|
import type { PromptSegments } from './prompt.js'
|
|
101
|
-
import
|
|
102
|
+
import { PendingAnswers, QuestionParkBinding } from './question-park.js'
|
|
102
103
|
import { ResultAssembler } from './result.js'
|
|
103
104
|
import {
|
|
104
105
|
type PendingResumePlan,
|
|
@@ -168,14 +169,20 @@ export interface QueryParams {
|
|
|
168
169
|
emergencySave?: boolean
|
|
169
170
|
|
|
170
171
|
/**
|
|
171
|
-
* Durability for questions raised
|
|
172
|
+
* Durability for questions raised by a tool that closed over its
|
|
173
|
+
* binding before the run existed.
|
|
172
174
|
*
|
|
173
|
-
* The
|
|
174
|
-
*
|
|
175
|
-
* lets
|
|
176
|
-
* Without it,
|
|
177
|
-
* the process while somebody is looking at the card and
|
|
178
|
-
* never be applied.
|
|
175
|
+
* The built-in `ask_user_question` is built with the agent's tool
|
|
176
|
+
* registry, so only whoever builds the tools can hand it one — that is
|
|
177
|
+
* what lets a single tool instance be durable inside a run and inert
|
|
178
|
+
* outside one. Without it, THAT tool's park is only a suspended
|
|
179
|
+
* `await`: kill the process while somebody is looking at the card and
|
|
180
|
+
* the answer can never be applied.
|
|
181
|
+
*
|
|
182
|
+
* Not required for `ToolContext.requestPause`. The run builds that
|
|
183
|
+
* seam per call and binds its own recorder when none is passed, so a
|
|
184
|
+
* pause raised from a host-authored tool is durable on every surface
|
|
185
|
+
* rather than only on the one agent class that supplies this.
|
|
179
186
|
*/
|
|
180
187
|
questionParks?: QuestionParkBinding
|
|
181
188
|
|
|
@@ -192,16 +199,28 @@ export interface QueryParams {
|
|
|
192
199
|
/**
|
|
193
200
|
* The registry a re-entered `ask_user_question` reads its answer from.
|
|
194
201
|
*
|
|
195
|
-
* Same shape as {@link questionParks}:
|
|
196
|
-
* exists
|
|
197
|
-
*
|
|
198
|
-
*
|
|
202
|
+
* Same shape, same reason and same limit as {@link questionParks}: it
|
|
203
|
+
* exists for a tool that closed over the instance before the run did,
|
|
204
|
+
* and without it a resumed run re-asks that tool's question. A pause
|
|
205
|
+
* from `ToolContext.requestPause` needs none, because the run fills
|
|
206
|
+
* its own on the resume path.
|
|
199
207
|
*/
|
|
200
208
|
pendingAnswers?: PendingAnswers
|
|
201
209
|
|
|
202
210
|
/** Default per-tool execution deadline. See {@link ToolDefinition.timeoutMs}. */
|
|
203
211
|
toolTimeoutMs?: number
|
|
204
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Wait between in-loop retries of a failed tool call, with full jitter.
|
|
215
|
+
* Defaults to {@link DEFAULT_TOOL_RETRY_BACKOFF}.
|
|
216
|
+
*
|
|
217
|
+
* Only reached by a tool that opted into retrying
|
|
218
|
+
* ({@link ToolDefinition.maxRetries}) or a `post_tool_use` hook that asked
|
|
219
|
+
* for one. Set `initialDelayMs: 0` for the retry-immediately behaviour
|
|
220
|
+
* this loop had before it had any backoff at all.
|
|
221
|
+
*/
|
|
222
|
+
toolRetryBackoff?: Partial<BackoffPolicy>
|
|
223
|
+
|
|
205
224
|
/** Max concurrently-executing concurrency-safe tools in one batch. */
|
|
206
225
|
maxToolConcurrency?: number
|
|
207
226
|
|
|
@@ -874,6 +893,25 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
874
893
|
? []
|
|
875
894
|
: withDeferredDiscoveryTool(params.tools, params.allowedTools)
|
|
876
895
|
|
|
896
|
+
// The two halves of a durable pause, owned by the RUN when the host
|
|
897
|
+
// does not own them.
|
|
898
|
+
//
|
|
899
|
+
// `SupervisorAgent` builds both before the run exists, because the
|
|
900
|
+
// tools it builds close over them, and it passes them in. Nothing else
|
|
901
|
+
// could: neither type is exported from `public-runtime.ts`, so a host
|
|
902
|
+
// on `ReactiveAgent`, `drainQuery` or `resumeRun` had no way to supply
|
|
903
|
+
// either — and `ToolContext.requestPause`, which every tool author is
|
|
904
|
+
// handed, silently wrote no checkpoint and could receive no answer on
|
|
905
|
+
// those surfaces. Which agent class the host happened to pick is not
|
|
906
|
+
// visible at the call site, so the degradation was invisible too.
|
|
907
|
+
//
|
|
908
|
+
// A run-local pair is enough for the general seam because `query()`
|
|
909
|
+
// builds its `createToolPause` itself, below, and can hand it the
|
|
910
|
+
// run's own. Pinned by the "a pause is durable on any surface" cases
|
|
911
|
+
// in `__tests__/tool-pause-resume.test.ts`.
|
|
912
|
+
const questionParks = params.questionParks ?? new QuestionParkBinding()
|
|
913
|
+
const pendingAnswers = params.pendingAnswers ?? new PendingAnswers()
|
|
914
|
+
|
|
877
915
|
// is null only when the run has no disk layout (tests,
|
|
878
916
|
// in-memory hosts); the budget then degrades to middle-elision.
|
|
879
917
|
const runDirForTools = ctx.runMgr.getRunDir()
|
|
@@ -891,6 +929,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
891
929
|
invocationState: params.invocationState,
|
|
892
930
|
pluginManager: params.pluginManager,
|
|
893
931
|
...(params.toolTimeoutMs !== undefined ? { toolTimeoutMs: params.toolTimeoutMs } : {}),
|
|
932
|
+
...(params.toolRetryBackoff !== undefined
|
|
933
|
+
? { toolRetryBackoff: params.toolRetryBackoff }
|
|
934
|
+
: {}),
|
|
894
935
|
...(params.maxToolConcurrency !== undefined
|
|
895
936
|
? { maxToolConcurrency: params.maxToolConcurrency }
|
|
896
937
|
: {}),
|
|
@@ -915,8 +956,8 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
915
956
|
runId: ctx.runId,
|
|
916
957
|
toolUseId,
|
|
917
958
|
parkHandler: params.resumeHandler,
|
|
918
|
-
|
|
919
|
-
|
|
959
|
+
recorder: questionParks,
|
|
960
|
+
pendingAnswers,
|
|
920
961
|
}),
|
|
921
962
|
},
|
|
922
963
|
ctx.activityStore,
|
|
@@ -1130,7 +1171,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1130
1171
|
// the checkpoint did not exist: nothing on disk said a human owed
|
|
1131
1172
|
// this run an answer, and a remote host could not observe the
|
|
1132
1173
|
// question at all.
|
|
1133
|
-
|
|
1174
|
+
questionParks.bind({
|
|
1134
1175
|
record: async (question) => {
|
|
1135
1176
|
try {
|
|
1136
1177
|
const checkpoint = await checkpointMgr.create(ctx.runMgr, ctx.runMgr.currentIteration)
|
|
@@ -1537,9 +1578,9 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1537
1578
|
// closed over its registry when the agent was constructed,
|
|
1538
1579
|
// long before this run existed, so the answers are copied in
|
|
1539
1580
|
// rather than passed down.
|
|
1540
|
-
if (pendingResume.answers
|
|
1581
|
+
if (pendingResume.answers) {
|
|
1541
1582
|
for (const [questionId, answer] of pendingResume.answers.entries()) {
|
|
1542
|
-
|
|
1583
|
+
pendingAnswers.set(questionId, answer)
|
|
1543
1584
|
}
|
|
1544
1585
|
}
|
|
1545
1586
|
|
|
@@ -1642,7 +1683,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1642
1683
|
// Same reasoning for the question channel: the tools outlive the
|
|
1643
1684
|
// run that bound them, so leaving it attached would have a later
|
|
1644
1685
|
// run's question written into this run's checkpoint store.
|
|
1645
|
-
|
|
1686
|
+
questionParks.unbind()
|
|
1646
1687
|
|
|
1647
1688
|
// Offer what the run learned to whoever decides what is worth
|
|
1648
1689
|
// keeping. In `finally` and awaited: a run that failed still
|