@namzu/sdk 15.1.0 → 17.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/dist/bridge/a2a/mapper.d.ts.map +1 -1
- package/dist/bridge/a2a/mapper.js +13 -0
- package/dist/bridge/a2a/mapper.js.map +1 -1
- package/dist/bridge/sse/mapper.d.ts.map +1 -1
- package/dist/bridge/sse/mapper.js +16 -0
- package/dist/bridge/sse/mapper.js.map +1 -1
- package/dist/contracts/api.d.ts +8 -1
- package/dist/contracts/api.d.ts.map +1 -1
- package/dist/provider/fallback.d.ts +93 -0
- package/dist/provider/fallback.d.ts.map +1 -0
- package/dist/provider/fallback.js +265 -0
- package/dist/provider/fallback.js.map +1 -0
- package/dist/provider/index.d.ts +2 -0
- package/dist/provider/index.d.ts.map +1 -1
- package/dist/provider/index.js +1 -0
- package/dist/provider/index.js.map +1 -1
- package/dist/public-runtime.d.ts +2 -2
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +1 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/run/reporter.d.ts.map +1 -1
- package/dist/run/reporter.js +16 -0
- package/dist/run/reporter.js.map +1 -1
- package/dist/runtime/query/index.d.ts +17 -0
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +56 -3
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.js +25 -0
- package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
- package/dist/types/provider/stream.d.ts +35 -0
- package/dist/types/provider/stream.d.ts.map +1 -1
- package/dist/types/run/events.d.ts +29 -0
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/utils/frontmatter.d.ts +12 -7
- package/dist/utils/frontmatter.d.ts.map +1 -1
- package/dist/utils/frontmatter.js +28 -7
- package/dist/utils/frontmatter.js.map +1 -1
- package/package.json +1 -1
- package/src/bridge/a2a/mapper.ts +16 -0
- package/src/bridge/sse/mapper.ts +17 -0
- package/src/contracts/api.ts +7 -0
- package/src/provider/fallback.ts +300 -0
- package/src/provider/index.ts +2 -0
- package/src/public-runtime.ts +7 -1
- package/src/run/reporter.ts +20 -0
- package/src/runtime/query/index.ts +80 -3
- package/src/runtime/query/iteration/stream-turn.ts +26 -0
- package/src/types/provider/stream.ts +37 -0
- package/src/types/run/events.ts +29 -0
- package/src/utils/frontmatter.ts +32 -7
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Advance through a declared provider chain when a member cannot serve.
|
|
3
|
+
*
|
|
4
|
+
* The sibling of `withProviderRetry`, one level out: retry asks "will the SAME
|
|
5
|
+
* member succeed if I ask again?", this asks "will ANOTHER member succeed if I
|
|
6
|
+
* ask it instead?". They compose in exactly one order —
|
|
7
|
+
* `fallback(retry(m0), retry(m1), …)` — and that order is the policy, not an
|
|
8
|
+
* implementation detail:
|
|
9
|
+
*
|
|
10
|
+
* - the primary is the operator's choice and gets its retries first, so a
|
|
11
|
+
* throttle or a 5xx only reaches this decorator once the inner budget is
|
|
12
|
+
* spent;
|
|
13
|
+
* - a failure the inner loop refuses to retry (`auth`, `not_found`) arrives
|
|
14
|
+
* here immediately, which is what "fall over at once on a bad credential"
|
|
15
|
+
* means — retrying a wrong key just spends the turn;
|
|
16
|
+
* - a server-directed `Retry-After` is honoured by the inner loop before any
|
|
17
|
+
* error escapes, so a transient wait is a wait and not a swap.
|
|
18
|
+
*
|
|
19
|
+
* None of those three behaviours is implemented here. They fall out of the
|
|
20
|
+
* nesting, which is why `query()` builds the composition rather than letting a
|
|
21
|
+
* host assemble it in whichever order it happens to pick.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { classifyProviderError, isAbortError } from '../types/provider/errors.js'
|
|
25
|
+
import type { ChatCompletionParams, LLMProvider, StreamChunk } from '../types/provider/index.js'
|
|
26
|
+
import type { Logger } from '../utils/logger.js'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* One member of the chain: a constructed provider, and the model to ask it for.
|
|
30
|
+
*
|
|
31
|
+
* `model` overrides {@link ChatCompletionParams.model} for this member's
|
|
32
|
+
* requests and nothing else. Every shipped driver reads that field
|
|
33
|
+
* (`params.model`, or `params.model || <its configured default>`), so a member
|
|
34
|
+
* needs no reconstruction to be asked for a different model — which is what
|
|
35
|
+
* keeps this decorator ignorant of credentials, base URLs and registries. A
|
|
36
|
+
* chain is an ordered list of (provider, model); building one is the host's
|
|
37
|
+
* job, walking it is this file's.
|
|
38
|
+
*
|
|
39
|
+
* Absent means "whatever the request already asked for", which is the right
|
|
40
|
+
* default for a member declared without a model: the registry default was
|
|
41
|
+
* resolved into the request before it got here.
|
|
42
|
+
*/
|
|
43
|
+
export interface ProviderChainMember {
|
|
44
|
+
readonly provider: LLMProvider
|
|
45
|
+
readonly model?: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface WithProviderFallbackOptions {
|
|
49
|
+
readonly log?: Logger
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Failures that are a property of the REQUEST rather than of the member serving
|
|
54
|
+
* it.
|
|
55
|
+
*
|
|
56
|
+
* This is the whole decision, stated once. A request fault reproduces
|
|
57
|
+
* identically on the next member, so falling over would spend a second
|
|
58
|
+
* provider's money to buy the same error. Everything else — a throttle, an
|
|
59
|
+
* outage, a rejected credential, a model this provider does not have, a stream
|
|
60
|
+
* that came back malformed — is a fact about the member, and the next member is
|
|
61
|
+
* worth asking.
|
|
62
|
+
*
|
|
63
|
+
* The three entries, and what each of them being here means:
|
|
64
|
+
*
|
|
65
|
+
* - `context_length_exceeded` — the run's own remedy is compaction, which sheds
|
|
66
|
+
* history and asks again. Falling over instead would carry the same oversized
|
|
67
|
+
* prompt to a provider that has not been given the chance to fit it. Note the
|
|
68
|
+
* limit of that reasoning: a chain MAY pair a small window with a larger one,
|
|
69
|
+
* and then a fallback genuinely could have succeeded. Compaction is still
|
|
70
|
+
* preferred because it is cheaper and it keeps the operator on the provider
|
|
71
|
+
* they chose.
|
|
72
|
+
* - `invalid_request` — namzu built a request the provider rejected. The
|
|
73
|
+
* counter-case is real and is recorded here rather than hidden: drivers
|
|
74
|
+
* translate the same tool schema differently onto the wire, so a 400 about a
|
|
75
|
+
* JSON Schema dialect (see `provider/errors.ts`) is provider-specific and
|
|
76
|
+
* another member might accept it. Abort still wins, because the common case
|
|
77
|
+
* is a request that is wrong everywhere, and spending the entire chain to
|
|
78
|
+
* rediscover one defect is the worse failure.
|
|
79
|
+
* - `content_filter` — a refusal. No shipped driver reaches this through a
|
|
80
|
+
* THROW today: the drivers that surface a filtered completion do so as
|
|
81
|
+
* `finishReason: 'content_filter'`, which is a finished stream, not an error.
|
|
82
|
+
* The input that makes this entry fire is a driver that throws instead — the
|
|
83
|
+
* classifier maps a `content_policy_violation` structural code onto it, and
|
|
84
|
+
* `ProviderError` is a public type a third-party driver constructs directly.
|
|
85
|
+
* Named rather than omitted for that reason; see
|
|
86
|
+
* `docs/conventions/a-check-that-cannot-fail.md` for why an unnameable one
|
|
87
|
+
* would have been left out.
|
|
88
|
+
*/
|
|
89
|
+
const REQUEST_FAULT_CODES: ReadonlySet<string> = new Set([
|
|
90
|
+
'context_length_exceeded',
|
|
91
|
+
'invalid_request',
|
|
92
|
+
'content_filter',
|
|
93
|
+
])
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Does this failure justify asking the next member?
|
|
97
|
+
*
|
|
98
|
+
* The classification is taken from `classifyProviderError` and not re-derived.
|
|
99
|
+
* A second classifier that disagreed with the first would be two answers to one
|
|
100
|
+
* question, and the retry decorator already stands on that same answer.
|
|
101
|
+
*
|
|
102
|
+
* `status === 404` is read ALONGSIDE the code, and it is not a second opinion —
|
|
103
|
+
* it is a second field of the one classification. A 404 reaches `not_found`
|
|
104
|
+
* only on the unclassified path (`codeFromStatus`). A driver that classified
|
|
105
|
+
* its own 404 into a `ProviderRequestError` gets `kind: 'bad_request'` from
|
|
106
|
+
* `classifyProviderHttpStatus`, which `KIND_TO_CODE` maps to
|
|
107
|
+
* `invalid_request` — a request fault, so the run would ABORT on a model the
|
|
108
|
+
* next member may well have. The status survives on both paths; the code does
|
|
109
|
+
* not, so the status is what this reads.
|
|
110
|
+
*/
|
|
111
|
+
function shouldFallOver(err: unknown, providerId: string): boolean {
|
|
112
|
+
const classified = classifyProviderError(err, providerId)
|
|
113
|
+
if (classified.status === 404) return true
|
|
114
|
+
return !REQUEST_FAULT_CODES.has(classified.code)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Has the consumer already been handed something a restart would duplicate?
|
|
119
|
+
*
|
|
120
|
+
* Deliberately NOT the retry decorator's rule, and the difference is the bug
|
|
121
|
+
* this function exists to prevent. Retry treats every chunk that is not
|
|
122
|
+
* error-only as output, which is correct INSIDE retry because each attempt
|
|
123
|
+
* starts with the flag cleared. Reused one level out it is wrong in the worst
|
|
124
|
+
* possible place: the inner decorator emits a backoff notice through this
|
|
125
|
+
* stream on its way to sleeping, that notice is not error-only, and a fallback
|
|
126
|
+
* reading it as output would refuse to advance for the rest of the turn — for
|
|
127
|
+
* the 429/5xx path, which is the main reason a chain is declared at all.
|
|
128
|
+
*
|
|
129
|
+
* So this asks the narrow question instead: did a chunk carry something the
|
|
130
|
+
* orchestrator turned into a visible event? `text_delta`, tool-input fragments
|
|
131
|
+
* and their per-tool boundary, reasoning blocks and citations all did.
|
|
132
|
+
* `usage`, `finishReason` and the two control notices did not.
|
|
133
|
+
*/
|
|
134
|
+
function isOutputChunk(chunk: StreamChunk): boolean {
|
|
135
|
+
if (chunk.retry !== undefined || chunk.fallback !== undefined) return false
|
|
136
|
+
const { content, toolCalls, toolCallEnd, reasoning, citation } = chunk.delta
|
|
137
|
+
return Boolean(
|
|
138
|
+
content ||
|
|
139
|
+
toolCalls?.length ||
|
|
140
|
+
toolCallEnd ||
|
|
141
|
+
reasoning !== undefined ||
|
|
142
|
+
citation !== undefined,
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Wrap an ordered chain so a member that cannot serve is replaced in place.
|
|
148
|
+
*
|
|
149
|
+
* ## The cursor's lifetime IS the scope
|
|
150
|
+
*
|
|
151
|
+
* Once this decorator advances, every later request in its life goes to the new
|
|
152
|
+
* member; the chain never rewinds. That is deliberate and it is how "the
|
|
153
|
+
* primary is restored at each new user message" is implemented — by NOT
|
|
154
|
+
* implementing it. `query()` builds one of these per call and a host's call is
|
|
155
|
+
* its turn, so the cursor cannot outlive the turn because the object cannot.
|
|
156
|
+
* There is no reset to forget to call, and no way for a rate limit at 14:00 to
|
|
157
|
+
* leave an operator on a cheaper model at 17:00.
|
|
158
|
+
*
|
|
159
|
+
* Rewinding within the turn would be the alternative, and it is worse: the
|
|
160
|
+
* member that just failed would be re-asked on the next iteration of the same
|
|
161
|
+
* turn, which is a retry wearing a chain's clothes and which the inner
|
|
162
|
+
* decorator already declined to do.
|
|
163
|
+
*
|
|
164
|
+
* ## Each member is tried at most once per turn, and the whole chain is walked
|
|
165
|
+
*
|
|
166
|
+
* A chain of N members yields up to N attempts, not one. An operator who
|
|
167
|
+
* declares four members and is served only by the second on a bad day has been
|
|
168
|
+
* given three decorative entries — that is the declared-but-undriven defect
|
|
169
|
+
* this file exists to remove, and stopping after one step would reintroduce it
|
|
170
|
+
* at position 2 instead of position 1. When the last member fails, its error is
|
|
171
|
+
* thrown untouched and ordinary error handling takes over.
|
|
172
|
+
*
|
|
173
|
+
* ## Once output is out, there is no fallback
|
|
174
|
+
*
|
|
175
|
+
* Inherited from retry, for the same reason and not a weaker one: a stream that
|
|
176
|
+
* has emitted bytes cannot be restarted without duplicating them, and the
|
|
177
|
+
* consumer has already appended them to a message it is rendering. A mid-stream
|
|
178
|
+
* failure after output is surfaced, never swapped. See {@link isOutputChunk} —
|
|
179
|
+
* the definition of "output" is where this property is actually won or lost.
|
|
180
|
+
*
|
|
181
|
+
* ## Capabilities are the head's
|
|
182
|
+
*
|
|
183
|
+
* The getters below are transparent, so a run negotiates tools, vision and
|
|
184
|
+
* documents ONCE against `members[0]` and keeps that answer after a swap. This
|
|
185
|
+
* is a real limitation and it is why the host is expected to refuse a chain
|
|
186
|
+
* whose members disagree before ever building one (`@namzu/cli` does, and only
|
|
187
|
+
* runs a mismatched chain when the operator has said so explicitly). Taking the
|
|
188
|
+
* intersection here instead would cost the primary a capability on every run to
|
|
189
|
+
* guard against a failure that happens rarely.
|
|
190
|
+
*/
|
|
191
|
+
export function withProviderFallback(
|
|
192
|
+
members: readonly ProviderChainMember[],
|
|
193
|
+
options: WithProviderFallbackOptions = {},
|
|
194
|
+
): LLMProvider {
|
|
195
|
+
const first = members[0]
|
|
196
|
+
if (!first) {
|
|
197
|
+
throw new Error('withProviderFallback needs at least one member')
|
|
198
|
+
}
|
|
199
|
+
// A one-member chain is the identity. Returning the provider itself rather
|
|
200
|
+
// than a wrapper that can never advance keeps the no-chain path byte-identical
|
|
201
|
+
// to what it was before this file existed.
|
|
202
|
+
if (members.length === 1) return first.provider
|
|
203
|
+
|
|
204
|
+
const log = options.log
|
|
205
|
+
let cursor = 0
|
|
206
|
+
|
|
207
|
+
async function* chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
|
|
208
|
+
for (;;) {
|
|
209
|
+
const member = members[cursor]
|
|
210
|
+
// Unreachable while `cursor` only ever moves to an index this loop has
|
|
211
|
+
// bounds-checked below; kept because the alternative on a future edit is
|
|
212
|
+
// calling `chatStream` on undefined, which names neither the field nor
|
|
213
|
+
// the fix.
|
|
214
|
+
if (!member) throw new Error(`provider chain has no member at position ${cursor}`)
|
|
215
|
+
|
|
216
|
+
const request = member.model !== undefined ? { ...params, model: member.model } : params
|
|
217
|
+
let produced = false
|
|
218
|
+
try {
|
|
219
|
+
for await (const chunk of member.provider.chatStream(request)) {
|
|
220
|
+
if (isOutputChunk(chunk)) produced = true
|
|
221
|
+
yield chunk
|
|
222
|
+
}
|
|
223
|
+
return
|
|
224
|
+
} catch (err) {
|
|
225
|
+
// A Stop is control flow, not a provider failure. Without this the
|
|
226
|
+
// classifier — which has no concept of cancellation — would file an
|
|
227
|
+
// abort as some ordinary failure and the run would walk the entire
|
|
228
|
+
// chain re-issuing an already-cancelled request, one member at a
|
|
229
|
+
// time, instead of settling as `cancelled`. The retry decorator
|
|
230
|
+
// guards the same way and for the same reason.
|
|
231
|
+
if (isAbortError(err) || params.signal?.aborted) throw err
|
|
232
|
+
|
|
233
|
+
const next = cursor + 1
|
|
234
|
+
const to = members[next]
|
|
235
|
+
if (produced || !to || !shouldFallOver(err, member.provider.id)) {
|
|
236
|
+
log?.warn('Provider chain: not falling over', {
|
|
237
|
+
provider: member.provider.id,
|
|
238
|
+
position: cursor,
|
|
239
|
+
reason: produced
|
|
240
|
+
? 'stream already produced output — cannot restart without duplicating it'
|
|
241
|
+
: !to
|
|
242
|
+
? 'chain exhausted'
|
|
243
|
+
: 'the failure is a property of the request, not of the provider',
|
|
244
|
+
})
|
|
245
|
+
throw err
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const classified = classifyProviderError(err, member.provider.id)
|
|
249
|
+
cursor = next
|
|
250
|
+
log?.warn('Provider chain: falling over', {
|
|
251
|
+
from: member.provider.id,
|
|
252
|
+
fromPosition: cursor - 1,
|
|
253
|
+
to: to.provider.id,
|
|
254
|
+
toPosition: cursor,
|
|
255
|
+
code: classified.code,
|
|
256
|
+
status: classified.status,
|
|
257
|
+
})
|
|
258
|
+
|
|
259
|
+
// In-band, exactly like the retry notice and for the same reason: the
|
|
260
|
+
// consumer is blocked inside this iterator, so the stream is the only
|
|
261
|
+
// channel that reaches it at the moment the swap happens rather than
|
|
262
|
+
// after the replacement request has already run.
|
|
263
|
+
yield {
|
|
264
|
+
id: '',
|
|
265
|
+
delta: {},
|
|
266
|
+
fallback: {
|
|
267
|
+
fromIndex: cursor - 1,
|
|
268
|
+
fromProviderId: member.provider.id,
|
|
269
|
+
...(member.model !== undefined ? { fromModel: member.model } : {}),
|
|
270
|
+
toIndex: cursor,
|
|
271
|
+
toProviderId: to.provider.id,
|
|
272
|
+
...(to.model !== undefined ? { toModel: to.model } : {}),
|
|
273
|
+
code: classified.code,
|
|
274
|
+
...(classified.status !== undefined ? { status: classified.status } : {}),
|
|
275
|
+
reason: classified.message,
|
|
276
|
+
},
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Transparent to capability negotiation and identity, like the retry
|
|
283
|
+
// decorator. The head's declarations are what a run is configured from; see
|
|
284
|
+
// the note on capabilities above for the limitation that carries.
|
|
285
|
+
return {
|
|
286
|
+
get id() {
|
|
287
|
+
return first.provider.id
|
|
288
|
+
},
|
|
289
|
+
get name() {
|
|
290
|
+
return first.provider.name
|
|
291
|
+
},
|
|
292
|
+
get capabilities() {
|
|
293
|
+
return first.provider.capabilities
|
|
294
|
+
},
|
|
295
|
+
chatStream,
|
|
296
|
+
...(first.provider.listModels ? { listModels: () => first.provider.listModels?.() } : {}),
|
|
297
|
+
...(first.provider.healthCheck ? { healthCheck: () => first.provider.healthCheck?.() } : {}),
|
|
298
|
+
...(first.provider.doctorCheck ? { doctorCheck: () => first.provider.doctorCheck?.() } : {}),
|
|
299
|
+
} as LLMProvider
|
|
300
|
+
}
|
package/src/provider/index.ts
CHANGED
|
@@ -35,3 +35,5 @@ export {
|
|
|
35
35
|
} from '../types/provider/errors.js'
|
|
36
36
|
export { DEFAULT_PROVIDER_RETRY, withProviderRetry } from './retry.js'
|
|
37
37
|
export type { ProviderRetryConfig, WithProviderRetryOptions } from './retry.js'
|
|
38
|
+
export { withProviderFallback } from './fallback.js'
|
|
39
|
+
export type { ProviderChainMember, WithProviderFallbackOptions } from './fallback.js'
|
package/src/public-runtime.ts
CHANGED
|
@@ -268,9 +268,15 @@ export {
|
|
|
268
268
|
registerMock,
|
|
269
269
|
resolveProviderCapabilities,
|
|
270
270
|
UnknownProviderError,
|
|
271
|
+
withProviderFallback,
|
|
271
272
|
withProviderRetry,
|
|
272
273
|
} from './provider/index.js'
|
|
273
|
-
export type {
|
|
274
|
+
export type {
|
|
275
|
+
ProviderChainMember,
|
|
276
|
+
ProviderRetryConfig,
|
|
277
|
+
WithProviderFallbackOptions,
|
|
278
|
+
WithProviderRetryOptions,
|
|
279
|
+
} from './provider/index.js'
|
|
274
280
|
|
|
275
281
|
export {
|
|
276
282
|
assertIsolation,
|
package/src/run/reporter.ts
CHANGED
|
@@ -332,6 +332,26 @@ export function createRunReporter(parentLogger?: Logger): RunReporter {
|
|
|
332
332
|
})
|
|
333
333
|
break
|
|
334
334
|
|
|
335
|
+
case 'provider_fallback':
|
|
336
|
+
// `warn` for the same reason as a retry, and one stronger: the rest
|
|
337
|
+
// of this run is being served by a provider the caller did not pick.
|
|
338
|
+
log.warn(
|
|
339
|
+
`Provider ${event.fromProviderId} could not serve — continuing on ${event.toProviderId}`,
|
|
340
|
+
{
|
|
341
|
+
runId: event.runId,
|
|
342
|
+
iteration: event.iteration,
|
|
343
|
+
fromIndex: event.fromIndex,
|
|
344
|
+
fromProviderId: event.fromProviderId,
|
|
345
|
+
fromModel: event.fromModel,
|
|
346
|
+
toIndex: event.toIndex,
|
|
347
|
+
toProviderId: event.toProviderId,
|
|
348
|
+
toModel: event.toModel,
|
|
349
|
+
code: event.code,
|
|
350
|
+
status: event.status,
|
|
351
|
+
},
|
|
352
|
+
)
|
|
353
|
+
break
|
|
354
|
+
|
|
335
355
|
default: {
|
|
336
356
|
const _exhaustive: never = event
|
|
337
357
|
throw new Error(`Unhandled run event type: ${(_exhaustive as RunEvent).type}`)
|
|
@@ -16,6 +16,7 @@ import type { CompactionConfig } from '../../config/runtime.js'
|
|
|
16
16
|
import { TOOL_OUTPUT_DIR_NAME } from '../../constants/tools/index.js'
|
|
17
17
|
import { EmergencySaveManager } from '../../manager/run/emergency.js'
|
|
18
18
|
import { resolveProviderCapabilities } from '../../provider/capabilities.js'
|
|
19
|
+
import { type ProviderChainMember, withProviderFallback } from '../../provider/fallback.js'
|
|
19
20
|
import { type ProviderRetryConfig, withProviderRetry } from '../../provider/retry.js'
|
|
20
21
|
import type { PathBuilder } from '../../session/workspace/path-builder.js'
|
|
21
22
|
import {
|
|
@@ -120,6 +121,23 @@ export interface QueryParams {
|
|
|
120
121
|
*/
|
|
121
122
|
retry?: Partial<ProviderRetryConfig> | false
|
|
122
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Members to fall over to, in order, when {@link provider} cannot serve.
|
|
126
|
+
*
|
|
127
|
+
* Absent means what it always meant: one provider, no failover. Each member
|
|
128
|
+
* is tried at most once per call and the chain never rewinds, so the scope
|
|
129
|
+
* of a swap is this `query()` — for a host whose call is one user turn, that
|
|
130
|
+
* is turn scope with no reset to forget. See `withProviderFallback`.
|
|
131
|
+
*
|
|
132
|
+
* Two things this does NOT do, both deliberate. Capabilities are negotiated
|
|
133
|
+
* once against {@link provider}, so a member that declares less will be sent
|
|
134
|
+
* a request shaped for the head — refuse a disagreeing chain before you
|
|
135
|
+
* build one. And a fallback loses the prompt cache: the replacement provider
|
|
136
|
+
* has never seen this conversation, so the turn re-reads its whole context
|
|
137
|
+
* at full price.
|
|
138
|
+
*/
|
|
139
|
+
fallbackProviders?: readonly ProviderChainMember[]
|
|
140
|
+
|
|
123
141
|
/**
|
|
124
142
|
* Install process-level crash handlers that dump this run's state to
|
|
125
143
|
* `<runDir>/../emergency/<runId>.json` on SIGINT, SIGTERM or an
|
|
@@ -472,6 +490,46 @@ export interface QueryParams {
|
|
|
472
490
|
strictCapabilities?: boolean
|
|
473
491
|
}
|
|
474
492
|
|
|
493
|
+
/**
|
|
494
|
+
* Refuse to price a run whose tokens two differently-priced members may produce.
|
|
495
|
+
*
|
|
496
|
+
* `RunPersistence` holds ONE {@link ModelPricing} table and applies it to every
|
|
497
|
+
* accumulation regardless of which model produced the tokens. Across a swap that
|
|
498
|
+
* makes `costInfo.totalCost` wrong by an unbounded margin, and silently — the
|
|
499
|
+
* number keeps the shape of an answer. `CostInfo` cannot express the truth
|
|
500
|
+
* either: it carries `inputCostPer1M` / `outputCostPer1M`, and there is no
|
|
501
|
+
* honest value for those once a total spans two rate cards.
|
|
502
|
+
*
|
|
503
|
+
* So the total is refused rather than blended. Naming what that costs is part
|
|
504
|
+
* of the refusal, because the caller loses `costLimitUsd` with it: the guard
|
|
505
|
+
* enforces that limit from this same accumulated total, and a limit enforced
|
|
506
|
+
* with the wrong rate card stops a run early or late by the same unbounded
|
|
507
|
+
* margin. A budget that is quietly wrong is worse than a budget that is
|
|
508
|
+
* declined.
|
|
509
|
+
*
|
|
510
|
+
* Reachable, not decorative: a host that passes `pricing` and declares a chain
|
|
511
|
+
* hits it on the first call. It costs `@namzu/cli` nothing, which passes no
|
|
512
|
+
* pricing at all — its `/cost` already reports that the provider gave no price.
|
|
513
|
+
*
|
|
514
|
+
* The way out is per-member pricing, which needs a `CostInfo` that can sum over
|
|
515
|
+
* heterogeneous rates. That is a public-type change and it is not this one.
|
|
516
|
+
*/
|
|
517
|
+
function assertCostIsAttributable(
|
|
518
|
+
chain: readonly ProviderChainMember[],
|
|
519
|
+
pricing: ModelPricing | undefined,
|
|
520
|
+
): void {
|
|
521
|
+
if (pricing === undefined || chain.length < 2) return
|
|
522
|
+
throw new NamzuError({
|
|
523
|
+
code: 'invalid_config',
|
|
524
|
+
message:
|
|
525
|
+
`A provider chain of ${chain.length} members was declared together with a single pricing table. ` +
|
|
526
|
+
'One table cannot price two members, so the run would report a total that is wrong by an unbounded ' +
|
|
527
|
+
'margin — and `runConfig.costLimitUsd` would be enforced against that same wrong total. ' +
|
|
528
|
+
'Either drop `pricing` (usage is still reported per model in the run) or declare one member.',
|
|
529
|
+
details: { chainLength: chain.length },
|
|
530
|
+
})
|
|
531
|
+
}
|
|
532
|
+
|
|
475
533
|
export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run> {
|
|
476
534
|
// Boot-time filesystem migration (session-hierarchy.md §13.4.1). First
|
|
477
535
|
// call per process per root actually runs; subsequent calls short-circuit
|
|
@@ -489,10 +547,29 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
489
547
|
// of its warns behind `options.log`, and this is its only production
|
|
490
548
|
// call site — so without it the "failed, retrying" and "failed, giving
|
|
491
549
|
// up" lines were dead code and a backoff left no trace anywhere.
|
|
492
|
-
|
|
550
|
+
//
|
|
551
|
+
// With a chain declared, the same sentence holds one level out: retry is
|
|
552
|
+
// applied per MEMBER and the fallback decorator wraps the result, so the
|
|
553
|
+
// composition is `fallback(retry(m0), retry(m1), …)`. That order is not a
|
|
554
|
+
// preference. Assembled the other way round — which is what a host gets if
|
|
555
|
+
// it wraps its own chain and hands the result in, because this function
|
|
556
|
+
// would then wrap THAT in retry — an exhausted chain gets restarted from
|
|
557
|
+
// the head by the outer loop and a throttle on the last member is counted
|
|
558
|
+
// by two budgets. Building it here is what makes the order unspellable
|
|
559
|
+
// wrong.
|
|
560
|
+
const chain: readonly ProviderChainMember[] = [
|
|
561
|
+
{ provider: params.provider },
|
|
562
|
+
...(params.fallbackProviders ?? []),
|
|
563
|
+
]
|
|
564
|
+
assertCostIsAttributable(chain, params.pricing)
|
|
565
|
+
const withRetry = (provider: LLMProvider): LLMProvider =>
|
|
493
566
|
params.retry === false
|
|
494
|
-
?
|
|
495
|
-
: withProviderRetry(
|
|
567
|
+
? provider
|
|
568
|
+
: withProviderRetry(provider, { config: params.retry, log: getRootLogger() })
|
|
569
|
+
const resilientProvider = withProviderFallback(
|
|
570
|
+
chain.map((member) => ({ ...member, provider: withRetry(member.provider) })),
|
|
571
|
+
{ log: getRootLogger() },
|
|
572
|
+
)
|
|
496
573
|
|
|
497
574
|
const ctx = RunContextFactory.build({
|
|
498
575
|
agentId: params.agentId,
|
|
@@ -258,6 +258,32 @@ export async function* streamProviderTurn(
|
|
|
258
258
|
continue
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
+
// A chain swap, not output, and handled beside the retry notice
|
|
262
|
+
// because it is the same kind of thing: a fact about HOW the answer
|
|
263
|
+
// is being produced, arriving on the only channel open while the
|
|
264
|
+
// consumer is blocked inside the provider's iterator. It carries no
|
|
265
|
+
// delta, so nothing below applies to it either.
|
|
266
|
+
if (chunk.fallback) {
|
|
267
|
+
await emitEvent({
|
|
268
|
+
type: 'provider_fallback',
|
|
269
|
+
runId,
|
|
270
|
+
iteration,
|
|
271
|
+
fromIndex: chunk.fallback.fromIndex,
|
|
272
|
+
fromProviderId: chunk.fallback.fromProviderId,
|
|
273
|
+
...(chunk.fallback.fromModel !== undefined
|
|
274
|
+
? { fromModel: chunk.fallback.fromModel }
|
|
275
|
+
: {}),
|
|
276
|
+
toIndex: chunk.fallback.toIndex,
|
|
277
|
+
toProviderId: chunk.fallback.toProviderId,
|
|
278
|
+
...(chunk.fallback.toModel !== undefined ? { toModel: chunk.fallback.toModel } : {}),
|
|
279
|
+
code: chunk.fallback.code,
|
|
280
|
+
...(chunk.fallback.status !== undefined ? { status: chunk.fallback.status } : {}),
|
|
281
|
+
reason: chunk.fallback.reason,
|
|
282
|
+
})
|
|
283
|
+
yield* drainPending()
|
|
284
|
+
continue
|
|
285
|
+
}
|
|
286
|
+
|
|
261
287
|
if (chunk.error) {
|
|
262
288
|
streamError = chunk.error
|
|
263
289
|
break
|
|
@@ -81,6 +81,18 @@ export interface StreamChunk {
|
|
|
81
81
|
* output.
|
|
82
82
|
*/
|
|
83
83
|
retry?: ProviderRetryNotice
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The call failed and a different member of the provider chain is taking
|
|
87
|
+
* over from here.
|
|
88
|
+
*
|
|
89
|
+
* Emitted by the fallback decorator, never by a driver, and it rides the
|
|
90
|
+
* stream for the reason {@link retry} does. Like a retry notice it carries
|
|
91
|
+
* no delta and must not be treated as output — and that distinction is
|
|
92
|
+
* load-bearing twice over, because the fallback decorator reads these
|
|
93
|
+
* chunks too when deciding whether output has already gone out.
|
|
94
|
+
*/
|
|
95
|
+
fallback?: ProviderFallbackNotice
|
|
84
96
|
}
|
|
85
97
|
|
|
86
98
|
/** See {@link StreamChunk.retry}. */
|
|
@@ -96,3 +108,28 @@ export interface ProviderRetryNotice {
|
|
|
96
108
|
/** The delay came from the server's own `Retry-After`, not backoff. */
|
|
97
109
|
readonly serverDirected: boolean
|
|
98
110
|
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* See {@link StreamChunk.fallback}.
|
|
114
|
+
*
|
|
115
|
+
* Both members are named, not just the new one. Naming only the replacement
|
|
116
|
+
* does not tell an operator which of their declared members went down, and on a
|
|
117
|
+
* chain of four that is the only fact they can act on.
|
|
118
|
+
*
|
|
119
|
+
* The positions are 0-based indices into the chain as the host declared it, so
|
|
120
|
+
* a surface can name a member the way its own configuration does rather than
|
|
121
|
+
* inventing a second numbering.
|
|
122
|
+
*/
|
|
123
|
+
export interface ProviderFallbackNotice {
|
|
124
|
+
readonly fromIndex: number
|
|
125
|
+
readonly fromProviderId: string
|
|
126
|
+
readonly fromModel?: string
|
|
127
|
+
readonly toIndex: number
|
|
128
|
+
readonly toProviderId: string
|
|
129
|
+
readonly toModel?: string
|
|
130
|
+
/** Classified failure code, as `classifyProviderError` reports it. */
|
|
131
|
+
readonly code: string
|
|
132
|
+
readonly status?: number
|
|
133
|
+
/** The classified failure's own sentence, already redacted at its source. */
|
|
134
|
+
readonly reason: string
|
|
135
|
+
}
|
package/src/types/run/events.ts
CHANGED
|
@@ -186,6 +186,35 @@ type CoreRunEvent =
|
|
|
186
186
|
/** The delay came from the server's own `Retry-After`. */
|
|
187
187
|
serverDirected: boolean
|
|
188
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* A member of the provider chain could not serve, and a later member has
|
|
191
|
+
* taken over. The run continues from where it stopped.
|
|
192
|
+
*
|
|
193
|
+
* This event is the feature's honesty. A chain that swapped silently would
|
|
194
|
+
* produce a run that succeeded while quietly not doing what the operator
|
|
195
|
+
* asked — served by a provider they did not choose, at a price and a
|
|
196
|
+
* quality they did not agree to, with nothing in the transcript saying so.
|
|
197
|
+
* A host is expected to SHOW this, not log it.
|
|
198
|
+
*
|
|
199
|
+
* Emitted at the moment of the swap, before the replacement request runs.
|
|
200
|
+
*/
|
|
201
|
+
| {
|
|
202
|
+
type: 'provider_fallback'
|
|
203
|
+
runId: RunId
|
|
204
|
+
iteration: number
|
|
205
|
+
/** 0-based position in the chain, as the host declared it. */
|
|
206
|
+
fromIndex: number
|
|
207
|
+
fromProviderId: string
|
|
208
|
+
fromModel?: string
|
|
209
|
+
toIndex: number
|
|
210
|
+
toProviderId: string
|
|
211
|
+
toModel?: string
|
|
212
|
+
/** Classified failure code, as the boundary classifier reports it. */
|
|
213
|
+
code: string
|
|
214
|
+
status?: number
|
|
215
|
+
/** The classified failure's own sentence. */
|
|
216
|
+
reason: string
|
|
217
|
+
}
|
|
189
218
|
| {
|
|
190
219
|
type: 'tool_completed'
|
|
191
220
|
runId: RunId
|
package/src/utils/frontmatter.ts
CHANGED
|
@@ -16,19 +16,24 @@
|
|
|
16
16
|
* half-understands YAML produces a value that passes validation and means
|
|
17
17
|
* nothing.
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
* *block* sequence
|
|
19
|
+
* That refusal is now total for lists. A *block* sequence
|
|
21
20
|
*
|
|
22
21
|
* ```yaml
|
|
23
22
|
* allowed-tools:
|
|
24
23
|
* - Read
|
|
25
24
|
* ```
|
|
26
25
|
*
|
|
27
|
-
*
|
|
28
|
-
* flow form `[Read, Grep]`
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
26
|
+
* used to be silently dropped — its lines carry no `:` and were skipped, so the
|
|
27
|
+
* key came back **absent** — while the flow form `[Read, Grep]` threw. One
|
|
28
|
+
* spelling of a list was a hard error and the other was silence, and the silent
|
|
29
|
+
* one is the shape an author actually writes, because the block form is the
|
|
30
|
+
* natural YAML for a list.
|
|
31
|
+
*
|
|
32
|
+
* It matters most for a key like `allowed-tools`: a skill that asked for `Bash`
|
|
33
|
+
* and silently did not get it is indistinguishable from one that never asked,
|
|
34
|
+
* which is a capability quietly not granted rather than a formatting nicety.
|
|
35
|
+
* Both readers this replaced behaved that way, so it was inherited rather than
|
|
36
|
+
* introduced; it is now refused, naming the key.
|
|
32
37
|
*
|
|
33
38
|
* **Vocabulary belongs to the caller.** This returns the parsed map; it does
|
|
34
39
|
* not know what a skill needs or what a command needs, and it validates no
|
|
@@ -176,6 +181,26 @@ export function parseFrontmatter(raw: string, source: string): ParsedFrontmatter
|
|
|
176
181
|
|
|
177
182
|
if (/^\s/.test(line)) {
|
|
178
183
|
if (!currentKey) continue
|
|
184
|
+
|
|
185
|
+
// A block sequence item. Refused, not skipped.
|
|
186
|
+
//
|
|
187
|
+
// These lines carry no `:`, so the `continue` below used to drop them
|
|
188
|
+
// and the key — having no scalar value and no mapping entries — came
|
|
189
|
+
// back ABSENT. The flow form `[Read, Grep]` already threw, so one
|
|
190
|
+
// spelling of a list was a hard error and the other was silence.
|
|
191
|
+
//
|
|
192
|
+
// The block form is the more natural YAML for a list, which is what
|
|
193
|
+
// made this worth closing: `allowed-tools` is a list, so this is the
|
|
194
|
+
// shape an author actually writes, and a skill that asked for `Bash`
|
|
195
|
+
// and silently did not get it is indistinguishable from one that never
|
|
196
|
+
// asked. A capability quietly not granted is the worst thing this
|
|
197
|
+
// reader can produce.
|
|
198
|
+
if (/^\s*-\s/.test(line)) {
|
|
199
|
+
throw new Error(
|
|
200
|
+
`${source}: "${currentKey}" uses a block sequence (a "- " list), which this reader does not support. Write it as a single-line value instead. Refusing rather than reading "${currentKey}" as absent, which is what silently dropping the list would mean.`,
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
|
|
179
204
|
const colonIdx = line.indexOf(':')
|
|
180
205
|
if (colonIdx === -1) continue
|
|
181
206
|
const key = line.slice(0, colonIdx).trim()
|