@namzu/sdk 17.0.0 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/dist/manager/run/persistence.d.ts +11 -0
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +13 -0
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/provider/fallback.d.ts +66 -0
- package/dist/provider/fallback.d.ts.map +1 -1
- package/dist/provider/fallback.js +20 -0
- package/dist/provider/fallback.js.map +1 -1
- package/dist/provider/index.d.ts +1 -1
- package/dist/provider/index.d.ts.map +1 -1
- package/dist/provider/retry.d.ts +9 -0
- package/dist/provider/retry.d.ts.map +1 -1
- package/dist/provider/retry.js +40 -3
- package/dist/provider/retry.js.map +1 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +22 -2
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +40 -1
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/context.d.ts +17 -0
- package/dist/runtime/query/iteration/phases/context.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/context.js.map +1 -1
- package/dist/types/run/entity.d.ts +38 -0
- package/dist/types/run/entity.d.ts.map +1 -1
- package/dist/types/run/step.d.ts +53 -0
- package/dist/types/run/step.d.ts.map +1 -1
- package/dist/types/run/step.js.map +1 -1
- package/package.json +1 -1
- package/src/manager/run/persistence.ts +14 -0
- package/src/provider/fallback.ts +88 -0
- package/src/provider/index.ts +5 -1
- package/src/provider/retry.ts +51 -4
- package/src/runtime/query/index.ts +26 -2
- package/src/runtime/query/iteration/index.ts +43 -1
- package/src/runtime/query/iteration/phases/context.ts +17 -0
- package/src/types/run/entity.ts +38 -0
- package/src/types/run/step.ts +54 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../../../src/types/run/step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAEnD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IAC1B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,SAAS,CAAA;IACpB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAA;IAC9B,uDAAuD;IACvD,WAAW,EAAE,SAAS,cAAc,EAAE,CAAA;IACtC,YAAY,EAAE,MAAM,GAAG,YAAY,GAAG,QAAQ,GAAG,gBAAgB,CAAA;IACjE,2DAA2D;IAC3D,KAAK,EAAE,UAAU,CAAA;IACjB,0EAA0E;IAC1E,SAAS,EAAE,QAAQ,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,eAAe,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,oEAAoE;IACpE,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,kBAAkB,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAErF,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IACrC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAA;CAC5B;AAED,0CAA0C;AAC1C,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,aAAa,CAEpD;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,aAAa,CAG7D;AAED,kDAAkD;AAClD,wBAAgB,KAAK,CAAC,GAAG,UAAU,EAAE,aAAa,EAAE,GAAG,aAAa,CAOnE"}
|
|
1
|
+
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../../../src/types/run/step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAEnD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IAC1B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAA;IAClB;;;;;;;;;;;;OAYG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,SAAS,EAAE,SAAS,CAAA;IACpB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAA;IAC9B,uDAAuD;IACvD,WAAW,EAAE,SAAS,cAAc,EAAE,CAAA;IACtC,YAAY,EAAE,MAAM,GAAG,YAAY,GAAG,QAAQ,GAAG,gBAAgB,CAAA;IACjE,2DAA2D;IAC3D,KAAK,EAAE,UAAU,CAAA;IACjB,0EAA0E;IAC1E,SAAS,EAAE,QAAQ,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,eAAe,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,oEAAoE;IACpE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,oEAAoE;IACpE,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,kBAAkB,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAErF,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IACrC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAA;CAC5B;AAED,0CAA0C;AAC1C,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,aAAa,CAEpD;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,aAAa,CAG7D;AAED,kDAAkD;AAClD,wBAAgB,KAAK,CAAC,GAAG,UAAU,EAAE,aAAa,EAAE,GAAG,aAAa,CAOnE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"step.js","sourceRoot":"","sources":["../../../src/types/run/step.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"step.js","sourceRoot":"","sources":["../../../src/types/run/step.ts"],"names":[],"mappings":"AA2HA,0CAA0C;AAC1C,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,GAAG,KAAe;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,OAAO,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AAC3F,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,KAAK,CAAC,GAAG,UAA2B;IACnD,OAAO,KAAK,EAAE,KAAK,EAAE,EAAE;QACtB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAA;QACxC,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC,CAAA;AACF,CAAC"}
|
package/package.json
CHANGED
|
@@ -286,6 +286,20 @@ export class RunPersistence {
|
|
|
286
286
|
this.run.steps = steps
|
|
287
287
|
}
|
|
288
288
|
|
|
289
|
+
/**
|
|
290
|
+
* Record that a provider chain advanced, so the run record stops naming a
|
|
291
|
+
* member that did not serve.
|
|
292
|
+
*
|
|
293
|
+
* Last writer wins on purpose: a chain of four can advance three times in
|
|
294
|
+
* one run, and `metadata.servingProvider` answers "who was serving when
|
|
295
|
+
* this ended", not "who was ever asked". The full sequence is in the
|
|
296
|
+
* transcript's `provider_fallback` events and, per turn, in
|
|
297
|
+
* `steps[].servedBy`.
|
|
298
|
+
*/
|
|
299
|
+
setServingProvider(providerId: string): void {
|
|
300
|
+
this.run.metadata.servingProvider = providerId
|
|
301
|
+
}
|
|
302
|
+
|
|
289
303
|
clearLastPromptTokens(): void {
|
|
290
304
|
this._lastPromptTokens = undefined
|
|
291
305
|
this._lastPromptMessageCount = undefined
|
package/src/provider/fallback.ts
CHANGED
|
@@ -45,8 +45,75 @@ export interface ProviderChainMember {
|
|
|
45
45
|
readonly model?: string
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* The member serving from now on.
|
|
50
|
+
*
|
|
51
|
+
* `index` is a position in the chain the host declared, so a reader can name
|
|
52
|
+
* the member without holding the chain: "member 2 of 4" is the sentence an
|
|
53
|
+
* operator writes in an incident note.
|
|
54
|
+
*/
|
|
55
|
+
export interface ServingMember {
|
|
56
|
+
readonly index: number
|
|
57
|
+
readonly providerId: string
|
|
58
|
+
/** Absent for a member declared without one — see {@link ProviderChainMember.model}. */
|
|
59
|
+
readonly model?: string
|
|
60
|
+
}
|
|
61
|
+
|
|
48
62
|
export interface WithProviderFallbackOptions {
|
|
49
63
|
readonly log?: Logger
|
|
64
|
+
/**
|
|
65
|
+
* Called once per swap, with the member that serves from here on.
|
|
66
|
+
*
|
|
67
|
+
* A callback is enough to describe the WHOLE truth, not a sample of it,
|
|
68
|
+
* and that is a property of the cursor rather than of this option: the
|
|
69
|
+
* chain never rewinds, so "who is serving" is exactly "the head, plus
|
|
70
|
+
* every swap so far". A listener that starts at member 0 and applies each
|
|
71
|
+
* call is never behind.
|
|
72
|
+
*
|
|
73
|
+
* It exists beside the in-band `fallback` chunk rather than instead of it
|
|
74
|
+
* because the two have different observers and neither covers the other's
|
|
75
|
+
* case. The chunk reaches whoever is iterating the stream, at the moment
|
|
76
|
+
* of the swap — that is the operator. This reaches a party that has to
|
|
77
|
+
* know AFTER the request is over and may never have iterated the stream at
|
|
78
|
+
* all — that is the run record. Two things follow that the chunk alone
|
|
79
|
+
* cannot give it:
|
|
80
|
+
*
|
|
81
|
+
* - the cursor outlives the request, so a swap on the turn at step 3
|
|
82
|
+
* still describes steps 4..N, which emit no further chunk;
|
|
83
|
+
* - a side call that aggregates the stream through `collect()` — the
|
|
84
|
+
* compaction verifier and the forced-final summary both do — drops the
|
|
85
|
+
* `fallback` chunk on the floor, so a swap inside one is invisible to
|
|
86
|
+
* every chunk consumer. (The advisory executor calls its OWN advisor's
|
|
87
|
+
* provider, not the run's, so it is not one of these.)
|
|
88
|
+
*
|
|
89
|
+
* ## Fired when the replacement is ASKED, not when the cursor moves
|
|
90
|
+
*
|
|
91
|
+
* The two are not the same instant and the difference is observable. The
|
|
92
|
+
* cursor moves inside the catch; the notice chunk is then yielded, and the
|
|
93
|
+
* replacement request is only issued when the consumer comes back for
|
|
94
|
+
* another chunk. A consumer that stops there — a Stop, a `break`, a host
|
|
95
|
+
* that abandons the iterator — leaves a chain that selected a member and
|
|
96
|
+
* never asked it.
|
|
97
|
+
*
|
|
98
|
+
* Announcing at cursor-move would report that member as serving, and a
|
|
99
|
+
* ledger saying a provider served a turn it was never sent is the exact
|
|
100
|
+
* defect this callback exists to end, reintroduced one layer down. So the
|
|
101
|
+
* announcement sits at the top of the loop, immediately before the
|
|
102
|
+
* replacement's `chatStream` — the earliest moment at which the member is
|
|
103
|
+
* actually being asked.
|
|
104
|
+
*
|
|
105
|
+
* ## One stream at a time
|
|
106
|
+
*
|
|
107
|
+
* `cursor` is shared by every concurrent `chatStream` on this wrapper, so
|
|
108
|
+
* two overlapping calls can advance it under one another: one call's
|
|
109
|
+
* failure moves the cursor while the other is still being served by the
|
|
110
|
+
* head, and a listener would hear about a member that answered nothing for
|
|
111
|
+
* that call. Nothing here serializes or refuses concurrency — the
|
|
112
|
+
* property held before this option existed and is not introduced by it.
|
|
113
|
+
* `query()` issues its main turn and its side calls in sequence, which is
|
|
114
|
+
* what makes the reading exact there.
|
|
115
|
+
*/
|
|
116
|
+
readonly onSwap?: (to: ServingMember) => void
|
|
50
117
|
}
|
|
51
118
|
|
|
52
119
|
/**
|
|
@@ -199,10 +266,22 @@ export function withProviderFallback(
|
|
|
199
266
|
// A one-member chain is the identity. Returning the provider itself rather
|
|
200
267
|
// than a wrapper that can never advance keeps the no-chain path byte-identical
|
|
201
268
|
// to what it was before this file existed.
|
|
269
|
+
//
|
|
270
|
+
// `onSwap` is therefore never called on this path, and that is the correct
|
|
271
|
+
// reading rather than a hole: a listener starts at member 0 and a one-member
|
|
272
|
+
// chain never leaves it. Announcing member 0 here would say "the chain
|
|
273
|
+
// advanced" about a chain that cannot.
|
|
202
274
|
if (members.length === 1) return first.provider
|
|
203
275
|
|
|
204
276
|
const log = options.log
|
|
205
277
|
let cursor = 0
|
|
278
|
+
/**
|
|
279
|
+
* The last position {@link WithProviderFallbackOptions.onSwap} was told
|
|
280
|
+
* about. Lags `cursor` for exactly as long as the consumer has the notice
|
|
281
|
+
* chunk and has not come back for more — which is the window in which a
|
|
282
|
+
* selected member has not been asked anything. See that option's doc.
|
|
283
|
+
*/
|
|
284
|
+
let announced = 0
|
|
206
285
|
|
|
207
286
|
async function* chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
|
|
208
287
|
for (;;) {
|
|
@@ -213,6 +292,15 @@ export function withProviderFallback(
|
|
|
213
292
|
// the fix.
|
|
214
293
|
if (!member) throw new Error(`provider chain has no member at position ${cursor}`)
|
|
215
294
|
|
|
295
|
+
if (announced !== cursor) {
|
|
296
|
+
announced = cursor
|
|
297
|
+
options.onSwap?.({
|
|
298
|
+
index: cursor,
|
|
299
|
+
providerId: member.provider.id,
|
|
300
|
+
...(member.model !== undefined ? { model: member.model } : {}),
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
|
|
216
304
|
const request = member.model !== undefined ? { ...params, model: member.model } : params
|
|
217
305
|
let produced = false
|
|
218
306
|
try {
|
package/src/provider/index.ts
CHANGED
|
@@ -36,4 +36,8 @@ export {
|
|
|
36
36
|
export { DEFAULT_PROVIDER_RETRY, withProviderRetry } from './retry.js'
|
|
37
37
|
export type { ProviderRetryConfig, WithProviderRetryOptions } from './retry.js'
|
|
38
38
|
export { withProviderFallback } from './fallback.js'
|
|
39
|
-
export type {
|
|
39
|
+
export type {
|
|
40
|
+
ProviderChainMember,
|
|
41
|
+
ServingMember,
|
|
42
|
+
WithProviderFallbackOptions,
|
|
43
|
+
} from './fallback.js'
|
package/src/provider/retry.ts
CHANGED
|
@@ -14,6 +14,15 @@ export interface ProviderRetryConfig {
|
|
|
14
14
|
* Cap on a server-directed `Retry-After`. A provider asking for 15
|
|
15
15
|
* minutes should not silently park an interactive run for 15 minutes;
|
|
16
16
|
* past this we surface the error and let the caller decide.
|
|
17
|
+
*
|
|
18
|
+
* "Surface" is the whole of it: there is no shorter retry underneath. A
|
|
19
|
+
* server that named a wait has said something specific, and answering it
|
|
20
|
+
* with a half-second backoff neither honours the wait nor tells anyone it
|
|
21
|
+
* was refused. The error carries `retryAfterMs`, so a host that wants to
|
|
22
|
+
* come back in fifteen minutes can — that decision is above this loop.
|
|
23
|
+
*
|
|
24
|
+
* Raise it to let the run sleep longer; a request under the ceiling is
|
|
25
|
+
* still slept exactly as instructed.
|
|
17
26
|
*/
|
|
18
27
|
readonly maxRetryAfterMs: number
|
|
19
28
|
}
|
|
@@ -153,10 +162,48 @@ export function withProviderRetry(
|
|
|
153
162
|
}
|
|
154
163
|
|
|
155
164
|
const serverDirected = classified.retryAfterMs
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
165
|
+
|
|
166
|
+
// The server named a wait longer than the caller's ceiling, so the
|
|
167
|
+
// error goes to the caller — which is what `maxRetryAfterMs`
|
|
168
|
+
// documents and what it now does.
|
|
169
|
+
//
|
|
170
|
+
// It used to fall through to the jittered backoff instead, and that
|
|
171
|
+
// is degrading where the contract says refuse
|
|
172
|
+
// (`docs/conventions/refuse-do-not-degrade.md`). The ceiling was
|
|
173
|
+
// read as "how long may I sleep", so a provider asking for fifteen
|
|
174
|
+
// minutes was re-asked in half a second: the one instruction the
|
|
175
|
+
// server gave was the one thing discarded, and the retries that
|
|
176
|
+
// followed were sent to an endpoint that had already said it would
|
|
177
|
+
// not serve them. They cost the run its whole budget to rediscover
|
|
178
|
+
// a 429 it had been told about in advance.
|
|
179
|
+
//
|
|
180
|
+
// The caller loses nothing it had. This throws the SAME error the
|
|
181
|
+
// exhausted path throws, so the run settles exactly as it did
|
|
182
|
+
// before — only sooner, and with `retryAfterMs` intact for a host
|
|
183
|
+
// that wants to schedule against it. What it gains is the wait
|
|
184
|
+
// itself, which no backoff of ours can honour: fifteen minutes is
|
|
185
|
+
// not a number this loop is allowed to sleep for.
|
|
186
|
+
//
|
|
187
|
+
// With a chain declared it gains more than that. The error is a
|
|
188
|
+
// `rate_limit`, which is a fact about the MEMBER, so
|
|
189
|
+
// `withProviderFallback` moves to the next one — the run continues
|
|
190
|
+
// on another provider instead of spending its budget arguing with
|
|
191
|
+
// the first. Under the old behaviour the chain did not see the
|
|
192
|
+
// failure until those attempts were gone.
|
|
193
|
+
if (serverDirected !== undefined && serverDirected > config.maxRetryAfterMs) {
|
|
194
|
+
log?.warn('Provider call failed — server-directed wait exceeds the ceiling', {
|
|
195
|
+
provider: provider.id,
|
|
196
|
+
code: classified.code,
|
|
197
|
+
status: classified.status,
|
|
198
|
+
attempt: attempt + 1,
|
|
199
|
+
retryAfterMs: serverDirected,
|
|
200
|
+
maxRetryAfterMs: config.maxRetryAfterMs,
|
|
201
|
+
reason: 'surfacing rather than retrying — the caller decides how to wait',
|
|
202
|
+
})
|
|
203
|
+
throw isProviderRequestError(err) ? err : classified
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const delay = serverDirected ?? backoffWithJitter(attempt, config, random)
|
|
160
207
|
|
|
161
208
|
log?.warn('Provider call failed — retrying', {
|
|
162
209
|
provider: provider.id,
|
|
@@ -16,7 +16,11 @@ import type { CompactionConfig } from '../../config/runtime.js'
|
|
|
16
16
|
import { TOOL_OUTPUT_DIR_NAME } from '../../constants/tools/index.js'
|
|
17
17
|
import { EmergencySaveManager } from '../../manager/run/emergency.js'
|
|
18
18
|
import { resolveProviderCapabilities } from '../../provider/capabilities.js'
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
type ProviderChainMember,
|
|
21
|
+
type ServingMember,
|
|
22
|
+
withProviderFallback,
|
|
23
|
+
} from '../../provider/fallback.js'
|
|
20
24
|
import { type ProviderRetryConfig, withProviderRetry } from '../../provider/retry.js'
|
|
21
25
|
import type { PathBuilder } from '../../session/workspace/path-builder.js'
|
|
22
26
|
import {
|
|
@@ -566,9 +570,28 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
566
570
|
params.retry === false
|
|
567
571
|
? provider
|
|
568
572
|
: withProviderRetry(provider, { config: params.retry, log: getRootLogger() })
|
|
573
|
+
// Who is serving right now, for the run RECORD rather than for the request.
|
|
574
|
+
//
|
|
575
|
+
// It starts at the head and moves only when the chain does, which is the
|
|
576
|
+
// whole of the truth because the cursor never rewinds. The run cannot read
|
|
577
|
+
// this off `resilientProvider`: that wrapper reports the head's `id` on
|
|
578
|
+
// purpose, so asking it produces the declaration back — the defect this
|
|
579
|
+
// record exists to fix.
|
|
580
|
+
const serving: { current: ServingMember } = {
|
|
581
|
+
current: { index: 0, providerId: params.provider.id },
|
|
582
|
+
}
|
|
569
583
|
const resilientProvider = withProviderFallback(
|
|
570
584
|
chain.map((member) => ({ ...member, provider: withRetry(member.provider) })),
|
|
571
|
-
{
|
|
585
|
+
{
|
|
586
|
+
log: getRootLogger(),
|
|
587
|
+
onSwap: (to) => {
|
|
588
|
+
serving.current = to
|
|
589
|
+
// `ctx` is declared below and is initialized before anything can
|
|
590
|
+
// call the provider: this fires from inside a `chatStream`, and
|
|
591
|
+
// the first one is issued by the loop that `ctx` is built for.
|
|
592
|
+
ctx.runMgr.setServingProvider(to.providerId)
|
|
593
|
+
},
|
|
594
|
+
},
|
|
572
595
|
)
|
|
573
596
|
|
|
574
597
|
const ctx = RunContextFactory.build({
|
|
@@ -898,6 +921,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
898
921
|
|
|
899
922
|
const iterationOrchestrator = new IterationOrchestrator({
|
|
900
923
|
provider: resilientProvider,
|
|
924
|
+
servingMember: () => serving.current,
|
|
901
925
|
runConfig: params.runConfig,
|
|
902
926
|
...(params.stopWhen ? { stopWhen: params.stopWhen } : {}),
|
|
903
927
|
...(params.prepareStep ? { prepareStep: params.prepareStep } : {}),
|
|
@@ -32,6 +32,7 @@ import type { AnswerReview } from '../../../types/run/answer-review.js'
|
|
|
32
32
|
import type {
|
|
33
33
|
PrepareStepResult,
|
|
34
34
|
RunEvent,
|
|
35
|
+
StepProvenance,
|
|
35
36
|
StepResult,
|
|
36
37
|
StopReason,
|
|
37
38
|
} from '../../../types/run/index.js'
|
|
@@ -425,6 +426,40 @@ export class IterationOrchestrator {
|
|
|
425
426
|
iterSpan,
|
|
426
427
|
)
|
|
427
428
|
|
|
429
|
+
// Who answered THIS turn.
|
|
430
|
+
//
|
|
431
|
+
// The read is exact at this point and stays exact: a chain that
|
|
432
|
+
// has produced output cannot fall over again inside the same
|
|
433
|
+
// request, so the member at the cursor when the stream ends is
|
|
434
|
+
// the one whose bytes are in `response`.
|
|
435
|
+
//
|
|
436
|
+
// It is taken here rather than at `recordStep` several hundred
|
|
437
|
+
// lines below, and the honest account of that is defence in
|
|
438
|
+
// depth, not a defect it currently prevents. Moving it down
|
|
439
|
+
// fails no test, because nothing between the two asks this
|
|
440
|
+
// provider for anything: compaction and working memory run
|
|
441
|
+
// BEFORE the turn, the advisory phase runs after the step is
|
|
442
|
+
// already recorded, and the only thing in between is tool
|
|
443
|
+
// execution. That is a fact about today's phase order, which a
|
|
444
|
+
// later phase inserted here would change silently — and the
|
|
445
|
+
// symptom would be a step attributed to a member that first
|
|
446
|
+
// served the turn after it, which is the class of wrongness
|
|
447
|
+
// this whole field exists to end.
|
|
448
|
+
const servedBy: StepProvenance = ((): StepProvenance => {
|
|
449
|
+
const member = this.ctx.servingMember?.() ?? {
|
|
450
|
+
index: 0,
|
|
451
|
+
providerId: this.ctx.provider.id,
|
|
452
|
+
}
|
|
453
|
+
return {
|
|
454
|
+
providerId: member.providerId,
|
|
455
|
+
// A member declared without a model asked for the model the
|
|
456
|
+
// step named — which is what the decorator does with the
|
|
457
|
+
// request, so this is a reading of it and not a guess.
|
|
458
|
+
model: member.model ?? stepModel,
|
|
459
|
+
chainIndex: member.index,
|
|
460
|
+
}
|
|
461
|
+
})()
|
|
462
|
+
|
|
428
463
|
// Main-loop turn: also records the prompt size compaction reads.
|
|
429
464
|
runMgr.recordTurnUsage(response.usage)
|
|
430
465
|
|
|
@@ -746,7 +781,12 @@ export class IterationOrchestrator {
|
|
|
746
781
|
// and a caller reconstructing cost per step must see it.
|
|
747
782
|
this.recordStep({
|
|
748
783
|
stepNumber: iterationNum,
|
|
749
|
-
model
|
|
784
|
+
// The model this step ASKED for. It used to be `model`, the
|
|
785
|
+
// run's own — so a `prepareStep` that routed one step to a
|
|
786
|
+
// cheaper model was recorded as the expensive one, with no
|
|
787
|
+
// provider chain involved.
|
|
788
|
+
model: stepModel,
|
|
789
|
+
servedBy,
|
|
750
790
|
messageId,
|
|
751
791
|
response,
|
|
752
792
|
toolResults: reviewOutcome.results,
|
|
@@ -1201,6 +1241,7 @@ export class IterationOrchestrator {
|
|
|
1201
1241
|
private recordStep(input: {
|
|
1202
1242
|
stepNumber: number
|
|
1203
1243
|
model: string
|
|
1244
|
+
servedBy: StepProvenance
|
|
1204
1245
|
messageId: MessageId
|
|
1205
1246
|
response: ChatCompletionResponse
|
|
1206
1247
|
toolResults: readonly ToolCallOutcome[]
|
|
@@ -1216,6 +1257,7 @@ export class IterationOrchestrator {
|
|
|
1216
1257
|
const step: StepResult = {
|
|
1217
1258
|
stepNumber: input.stepNumber,
|
|
1218
1259
|
model: input.model,
|
|
1260
|
+
servedBy: input.servedBy,
|
|
1219
1261
|
messageId: input.messageId,
|
|
1220
1262
|
content: input.response.message.content,
|
|
1221
1263
|
toolCalls,
|
|
@@ -6,6 +6,7 @@ import type { CompactionConfig } from '../../../../config/runtime.js'
|
|
|
6
6
|
import type { CompletionInbox } from '../../../../gateway/completion-inbox.js'
|
|
7
7
|
import type { PlanManager } from '../../../../manager/plan/lifecycle.js'
|
|
8
8
|
import type { RunPersistence } from '../../../../manager/run/persistence.js'
|
|
9
|
+
import type { ServingMember } from '../../../../provider/fallback.js'
|
|
9
10
|
import type { ActivityStore } from '../../../../store/activity/memory.js'
|
|
10
11
|
import type { TaskGateway } from '../../../../types/agent/gateway.js'
|
|
11
12
|
import type { WorkingMemoryProvider } from '../../../../types/agent/working-memory.js'
|
|
@@ -37,6 +38,22 @@ import type { ToolGrantSet } from '../../tool-grants.js'
|
|
|
37
38
|
|
|
38
39
|
export interface IterationContext {
|
|
39
40
|
readonly provider: LLMProvider
|
|
41
|
+
/**
|
|
42
|
+
* Which chain member `provider` will route the NEXT request to.
|
|
43
|
+
*
|
|
44
|
+
* `provider` cannot answer this itself: `withProviderFallback` keeps its
|
|
45
|
+
* `id` transparently equal to the head's, deliberately, because that is
|
|
46
|
+
* what capability negotiation and the run's `gen_ai.system` attribute are
|
|
47
|
+
* about. Asking the wrapper who it is gets the declaration; this gets the
|
|
48
|
+
* observation.
|
|
49
|
+
*
|
|
50
|
+
* Optional because a host may build an `IterationContext` without a chain
|
|
51
|
+
* at all. Absent, the loop attributes each step to `provider.id` and the
|
|
52
|
+
* model it requested, which is exactly right when nothing can fall over —
|
|
53
|
+
* and exactly wrong when something can, so the wiring from `query()` is
|
|
54
|
+
* covered end-to-end rather than by a unit test on this accessor.
|
|
55
|
+
*/
|
|
56
|
+
readonly servingMember?: () => ServingMember
|
|
40
57
|
/**
|
|
41
58
|
* The run's `invoke_agent` span, so each iteration can parent itself to
|
|
42
59
|
* it. Explicit rather than ambient because this loop is an async
|
package/src/types/run/entity.ts
CHANGED
|
@@ -11,7 +11,45 @@ export interface RunStateMetadata {
|
|
|
11
11
|
agentId: string
|
|
12
12
|
agentName: string
|
|
13
13
|
config: AgentRunConfig
|
|
14
|
+
/**
|
|
15
|
+
* The provider the run was CONFIGURED with — the head of the chain, and
|
|
16
|
+
* the counterpart of `config.model` beside it.
|
|
17
|
+
*
|
|
18
|
+
* It is a declaration, not an observation, and it stays one. After a
|
|
19
|
+
* provider chain falls over it still names the head, which is correct for
|
|
20
|
+
* a field that answers "what was asked for"; it was only ever misleading
|
|
21
|
+
* because nothing else answered "what served". See {@link servingProvider}
|
|
22
|
+
* for the run-level answer and `steps[].servedBy` for the per-step one.
|
|
23
|
+
*/
|
|
14
24
|
provider: string
|
|
25
|
+
/**
|
|
26
|
+
* The chain member the run was routed to at the end, when that is not the
|
|
27
|
+
* one it was configured with.
|
|
28
|
+
*
|
|
29
|
+
* Absent means the declared provider served every call. That reading holds
|
|
30
|
+
* for stored records too, with one bounded exception: the sdk major that
|
|
31
|
+
* shipped the chain could fall over without recording it, so a run from
|
|
32
|
+
* that release reads as "no swap" whether or not there was one. Its
|
|
33
|
+
* transcript still carries the `provider_fallback` events. The exception
|
|
34
|
+
* is one release wide and it is stated rather than migrated away, because
|
|
35
|
+
* a migration would have to invent the answer for exactly the records that
|
|
36
|
+
* do not have it.
|
|
37
|
+
*
|
|
38
|
+
* **This is the durable half.** `RunDiskStore.writeRunMeta` writes
|
|
39
|
+
* `metadata` and not `steps`, so on the built-in store this field is the
|
|
40
|
+
* whole of what survives the process. `steps[].servedBy` is the finer
|
|
41
|
+
* record — the one to read for "which member answered turn 4" — and it
|
|
42
|
+
* reaches a host only on the returned `Run`.
|
|
43
|
+
*
|
|
44
|
+
* It also covers the case no step ledger can: a run that falls over and
|
|
45
|
+
* then dies before a single step is recorded still has to be able to say
|
|
46
|
+
* whose failure ended it. That is why the wording is "routed to" and not
|
|
47
|
+
* "served by" — this member was asked, and on that path it answered with
|
|
48
|
+
* an error. It is never a member that was merely selected: the chain
|
|
49
|
+
* announces a replacement when it issues its request, not when it picks
|
|
50
|
+
* it, so a run cancelled at the swap notice does not name one.
|
|
51
|
+
*/
|
|
52
|
+
servingProvider?: string
|
|
15
53
|
}
|
|
16
54
|
|
|
17
55
|
export type SessionMetadata = RunStateMetadata
|
package/src/types/run/step.ts
CHANGED
|
@@ -15,7 +15,46 @@ import type { ToolCall } from '../message/index.js'
|
|
|
15
15
|
export interface StepResult {
|
|
16
16
|
/** 1-based, matching `iteration` on the run events. */
|
|
17
17
|
stepNumber: number
|
|
18
|
+
/**
|
|
19
|
+
* The model this step ASKED for: the run's configured model, or the
|
|
20
|
+
* override a `prepareStep` hook returned for this step.
|
|
21
|
+
*
|
|
22
|
+
* It used to be the run's model unconditionally — the loop passed its own
|
|
23
|
+
* `model` here while building the request from `step.model ?? model` a few
|
|
24
|
+
* lines above — so a host that routed one step to a cheaper model read the
|
|
25
|
+
* expensive one back out of the ledger. No chain was needed to see it.
|
|
26
|
+
*
|
|
27
|
+
* What was asked for and what answered are two facts, and after a provider
|
|
28
|
+
* chain falls over they differ. This is the first; {@link servedBy} is the
|
|
29
|
+
* second.
|
|
30
|
+
*/
|
|
18
31
|
model: string
|
|
32
|
+
/**
|
|
33
|
+
* Who actually answered, and with which model.
|
|
34
|
+
*
|
|
35
|
+
* Equal to {@link model} and to `run.metadata.provider` on every run
|
|
36
|
+
* without a chain, which is most of them; it diverges exactly when
|
|
37
|
+
* `withProviderFallback` advanced. Recorded even when it agrees, because a
|
|
38
|
+
* ledger that carries the fact only when it is surprising cannot be read as
|
|
39
|
+
* evidence — a reader could not tell "the head served" from "nobody wrote
|
|
40
|
+
* it down".
|
|
41
|
+
*
|
|
42
|
+
* Optional only for records that predate the field. Absence means "not
|
|
43
|
+
* recorded", and it is left meaning that rather than backfilled: the sdk
|
|
44
|
+
* shipped a chain that could fall over one release before it recorded
|
|
45
|
+
* which member did, so filling those in from the declared head would state
|
|
46
|
+
* as fact the exact thing that release got wrong, on exactly the runs
|
|
47
|
+
* where it was wrong. Every step this build produces has it.
|
|
48
|
+
*
|
|
49
|
+
* **Reaches a host through the returned `Run`, not through `run.json`.**
|
|
50
|
+
* `RunDiskStore.writeRunMeta` persists the metadata and the counters and
|
|
51
|
+
* does not write `steps` at all, so the built-in store carries the
|
|
52
|
+
* run-level {@link
|
|
53
|
+
* import('./entity.js').RunStateMetadata.servingProvider} and none of
|
|
54
|
+
* this. A host that wants per-step provenance on disk persists the `Run`
|
|
55
|
+
* it is handed.
|
|
56
|
+
*/
|
|
57
|
+
servedBy?: StepProvenance
|
|
19
58
|
messageId: MessageId
|
|
20
59
|
/** Assistant text for this step, if any. */
|
|
21
60
|
content: string | null
|
|
@@ -34,6 +73,21 @@ export interface StepResult {
|
|
|
34
73
|
toolExecutionMs: number
|
|
35
74
|
}
|
|
36
75
|
|
|
76
|
+
/**
|
|
77
|
+
* The chain member that served one step.
|
|
78
|
+
*
|
|
79
|
+
* `chainIndex` is a position in the chain the host declared, and it is here
|
|
80
|
+
* rather than derived from `providerId` because a chain may legitimately name
|
|
81
|
+
* the same provider twice — two models, or two credentials, on one driver.
|
|
82
|
+
* `providerId` alone could not tell those apart.
|
|
83
|
+
*/
|
|
84
|
+
export interface StepProvenance {
|
|
85
|
+
readonly providerId: string
|
|
86
|
+
readonly model: string
|
|
87
|
+
/** 0 is the head, i.e. the provider the run was configured with. */
|
|
88
|
+
readonly chainIndex: number
|
|
89
|
+
}
|
|
90
|
+
|
|
37
91
|
export interface StepToolResult {
|
|
38
92
|
toolCallId: string
|
|
39
93
|
toolName: string
|