@providerkit/core 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -7
- package/dist/auth/flows.d.ts.map +1 -1
- package/dist/auth/flows.js +5 -5
- package/dist/auth/flows.js.map +1 -1
- package/dist/auth/index.d.ts +1 -0
- package/dist/auth/index.d.ts.map +1 -1
- package/dist/auth/index.js +1 -0
- package/dist/auth/index.js.map +1 -1
- package/dist/auth/oauth.d.ts +12 -0
- package/dist/auth/oauth.d.ts.map +1 -1
- package/dist/auth/oauth.js +19 -1
- package/dist/auth/oauth.js.map +1 -1
- package/dist/auth/token-source.d.ts +9 -3
- package/dist/auth/token-source.d.ts.map +1 -1
- package/dist/auth/token-source.js +24 -9
- package/dist/auth/token-source.js.map +1 -1
- package/dist/auth/with-token.d.ts +32 -0
- package/dist/auth/with-token.d.ts.map +1 -0
- package/dist/auth/with-token.js +88 -0
- package/dist/auth/with-token.js.map +1 -0
- package/dist/context.d.ts +39 -1
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +119 -0
- package/dist/context.js.map +1 -1
- package/dist/errors.d.ts +12 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +44 -5
- package/dist/errors.js.map +1 -1
- package/dist/fallback.d.ts +45 -1
- package/dist/fallback.d.ts.map +1 -1
- package/dist/fallback.js +138 -75
- package/dist/fallback.js.map +1 -1
- package/dist/gate.d.ts +29 -0
- package/dist/gate.d.ts.map +1 -0
- package/dist/gate.js +96 -0
- package/dist/gate.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/key-pool.d.ts +21 -4
- package/dist/key-pool.d.ts.map +1 -1
- package/dist/key-pool.js +43 -12
- package/dist/key-pool.js.map +1 -1
- package/dist/presets.d.ts +7 -2
- package/dist/presets.d.ts.map +1 -1
- package/dist/presets.js +12 -2
- package/dist/presets.js.map +1 -1
- package/dist/providers/anthropic.d.ts +7 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +83 -26
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/factory.d.ts +26 -0
- package/dist/providers/factory.d.ts.map +1 -1
- package/dist/providers/factory.js +41 -9
- package/dist/providers/factory.js.map +1 -1
- package/dist/providers/gemini.d.ts +6 -0
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +102 -27
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/openai.d.ts +7 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +66 -15
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/responses.d.ts +11 -0
- package/dist/providers/responses.d.ts.map +1 -1
- package/dist/providers/responses.js +101 -53
- package/dist/providers/responses.js.map +1 -1
- package/dist/rate-limit.d.ts +10 -0
- package/dist/rate-limit.d.ts.map +1 -1
- package/dist/rate-limit.js +7 -1
- package/dist/rate-limit.js.map +1 -1
- package/dist/registry.d.ts +105 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +291 -0
- package/dist/registry.js.map +1 -0
- package/dist/retry.d.ts +11 -0
- package/dist/retry.d.ts.map +1 -1
- package/dist/retry.js +25 -12
- package/dist/retry.js.map +1 -1
- package/dist/tool-args.d.ts +22 -0
- package/dist/tool-args.d.ts.map +1 -1
- package/dist/tool-args.js +28 -0
- package/dist/tool-args.js.map +1 -1
- package/dist/tools.d.ts +8 -0
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +2 -2
- package/dist/tools.js.map +1 -1
- package/dist/transport.d.ts +34 -4
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +77 -16
- package/dist/transport.js.map +1 -1
- package/dist/turn.d.ts +55 -0
- package/dist/turn.d.ts.map +1 -0
- package/dist/turn.js +125 -0
- package/dist/turn.js.map +1 -0
- package/dist/types.d.ts +21 -22
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +0 -53
- package/dist/types.js.map +1 -1
- package/dist/watchdog.d.ts +2 -0
- package/dist/watchdog.d.ts.map +1 -1
- package/dist/watchdog.js +25 -6
- package/dist/watchdog.js.map +1 -1
- package/package.json +1 -1
- package/src/auth/flows.ts +5 -4
- package/src/auth/index.ts +1 -0
- package/src/auth/oauth.ts +23 -1
- package/src/auth/token-source.ts +30 -11
- package/src/auth/with-token.ts +100 -0
- package/src/context.ts +137 -2
- package/src/errors.ts +46 -6
- package/src/fallback.ts +173 -75
- package/src/gate.ts +152 -0
- package/src/index.ts +3 -0
- package/src/key-pool.ts +45 -13
- package/src/presets.ts +12 -2
- package/src/providers/anthropic.ts +93 -25
- package/src/providers/factory.ts +66 -21
- package/src/providers/gemini.ts +102 -28
- package/src/providers/openai.ts +71 -14
- package/src/providers/responses.ts +116 -51
- package/src/rate-limit.ts +31 -3
- package/src/registry.ts +420 -0
- package/src/retry.ts +41 -10
- package/src/tool-args.ts +44 -0
- package/src/tools.ts +10 -2
- package/src/transport.ts +97 -17
- package/src/turn.ts +166 -0
- package/src/types.ts +21 -62
- package/src/watchdog.ts +25 -6
package/src/transport.ts
CHANGED
|
@@ -4,8 +4,12 @@
|
|
|
4
4
|
//
|
|
5
5
|
// Adapters keep only their per-event mapping; everything about being an HTTP
|
|
6
6
|
// client lives here once.
|
|
7
|
-
import { classifyHttp,
|
|
8
|
-
import {
|
|
7
|
+
import { classifyHttp, parseRetryAfterMs, ProviderError } from "./errors.ts";
|
|
8
|
+
import {
|
|
9
|
+
parseRateLimitReset,
|
|
10
|
+
parseRateLimitResponse,
|
|
11
|
+
type RateLimitReading,
|
|
12
|
+
} from "./rate-limit.ts";
|
|
9
13
|
|
|
10
14
|
export interface RequestInit_ {
|
|
11
15
|
url: string;
|
|
@@ -21,6 +25,11 @@ export interface RequestInit_ {
|
|
|
21
25
|
onActivity?: () => void;
|
|
22
26
|
/** Called when the body states `[DONE]`. See `SseHooks.onDone`. */
|
|
23
27
|
onDone?: () => void;
|
|
28
|
+
/** Called when the response headers report how full a subscription window
|
|
29
|
+
* is. See `StreamOptions.onRateLimit`. */
|
|
30
|
+
onRateLimit?: (reading: RateLimitReading) => void;
|
|
31
|
+
/** Whether the turn already stated its finish. See `SseHooks.finished`. */
|
|
32
|
+
finished?: () => boolean;
|
|
24
33
|
}
|
|
25
34
|
|
|
26
35
|
export interface SseHooks {
|
|
@@ -33,6 +42,18 @@ export interface SseHooks {
|
|
|
33
42
|
* was not simply cut off.
|
|
34
43
|
*/
|
|
35
44
|
onDone?: () => void;
|
|
45
|
+
/**
|
|
46
|
+
* Whether the turn already stated its finish, so only trailing frames (the
|
|
47
|
+
* usage record, `[DONE]`) are left. Once this says true, a read that fails
|
|
48
|
+
* ends the stream instead of throwing. Kimi, GLM and OpenCode Go drop the
|
|
49
|
+
* socket there, and the failure threw away an answer the reader had already
|
|
50
|
+
* been shown (cc-proxy, measured on live traffic). The usage may be lost;
|
|
51
|
+
* the answer is not.
|
|
52
|
+
*/
|
|
53
|
+
finished?: () => boolean;
|
|
54
|
+
/** The request's signal. A caller's Stop is never swallowed, finished or not:
|
|
55
|
+
* a loop that sees a clean end goes on to run the turn's tools. */
|
|
56
|
+
signal?: AbortSignal;
|
|
36
57
|
/** Largest partial frame held before giving up. Defaults to 8 MiB. */
|
|
37
58
|
maxFrameChars?: number;
|
|
38
59
|
/** Names the provider on the oversized-frame error. */
|
|
@@ -120,18 +141,25 @@ async function errorFor(provider: string, res: Response): Promise<ProviderError>
|
|
|
120
141
|
|
|
121
142
|
async function send(opts: RequestInit_): Promise<Response> {
|
|
122
143
|
const doFetch = opts.fetchImpl ?? globalThis.fetch;
|
|
144
|
+
// Serialized outside the catch below: a body that cannot become JSON is a
|
|
145
|
+
// bug of ours, and must not be dressed as a network blip and retried.
|
|
146
|
+
const body = typeof opts.body === "string" ? opts.body : JSON.stringify(opts.body);
|
|
123
147
|
try {
|
|
124
148
|
return await doFetch(opts.url, {
|
|
125
149
|
method: "POST",
|
|
126
150
|
headers: { "content-type": "application/json", ...opts.headers },
|
|
127
|
-
body
|
|
151
|
+
body,
|
|
128
152
|
signal: opts.signal,
|
|
129
153
|
});
|
|
130
154
|
} catch (err) {
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
|
|
155
|
+
// An aborted signal passes through untouched: the caller's Stop stays
|
|
156
|
+
// `aborted`, and the watchdog re-issues its own abort as `timeout`
|
|
157
|
+
// (invariant 2). Every other rejection means no response came back, so it
|
|
158
|
+
// is a transport failure whatever its wording. Matching known wordings
|
|
159
|
+
// missed each runtime's next one, and a missed one was never retried: a
|
|
160
|
+
// long run died on its first blip. cc-proxy reads every failed send the
|
|
161
|
+
// same way.
|
|
162
|
+
if (opts.signal?.aborted) throw err;
|
|
135
163
|
throw new ProviderError(opts.provider, "network", `no response from ${opts.url}`, {
|
|
136
164
|
cause: err,
|
|
137
165
|
});
|
|
@@ -173,10 +201,11 @@ function payloadOf(frame: string): string | null {
|
|
|
173
201
|
* (see `payloadOf`).
|
|
174
202
|
*
|
|
175
203
|
* Frames are split on the blank line the spec requires, so a payload containing
|
|
176
|
-
* a bare newline survives
|
|
177
|
-
* adapter,
|
|
178
|
-
*
|
|
179
|
-
*
|
|
204
|
+
* a bare newline survives. `[DONE]` is swallowed here rather than in every
|
|
205
|
+
* adapter, reported through `hooks.onDone`, and ends the stream: it is the
|
|
206
|
+
* body's last word, and a read after it can only fail a finished turn. CRLF and
|
|
207
|
+
* bare CR are normalized: some gateways send them, and a `\r` left on the end
|
|
208
|
+
* of a JSON payload is a parse error nobody enjoys debugging.
|
|
180
209
|
*
|
|
181
210
|
* Exported apart from `streamSse` because the envelope above it is the half an
|
|
182
211
|
* adopting app most often cannot take: an app with its own translated error
|
|
@@ -195,17 +224,36 @@ export async function* parseSseStream(
|
|
|
195
224
|
// A CR at the very end of a read may be the first half of a CRLF; it waits
|
|
196
225
|
// for the next read before it is turned into a line break.
|
|
197
226
|
let heldCR = false;
|
|
227
|
+
// Whether the body was read to its end. Any other exit (`[DONE]`, a failed
|
|
228
|
+
// read, a consumer that stopped at its own terminal event) cancels it, so the
|
|
229
|
+
// connection is let go now rather than whenever the body is collected.
|
|
230
|
+
let drained = false;
|
|
198
231
|
|
|
199
|
-
|
|
232
|
+
/** Yields the frame's payload, and returns true when the frame was `[DONE]`. */
|
|
233
|
+
function* emit(frame: string): Generator<string, boolean> {
|
|
200
234
|
const payload = payloadOf(frame);
|
|
201
|
-
if (payload === "[DONE]")
|
|
202
|
-
|
|
235
|
+
if (payload === "[DONE]") {
|
|
236
|
+
hooks.onDone?.();
|
|
237
|
+
return true;
|
|
238
|
+
}
|
|
239
|
+
if (payload !== null) yield payload;
|
|
240
|
+
return false;
|
|
203
241
|
}
|
|
204
242
|
|
|
205
243
|
try {
|
|
206
244
|
for (;;) {
|
|
207
|
-
|
|
208
|
-
|
|
245
|
+
let read: ReadableStreamReadResult<Uint8Array>;
|
|
246
|
+
try {
|
|
247
|
+
read = await reader.read();
|
|
248
|
+
} catch (err) {
|
|
249
|
+
if (hooks.finished?.() && !hooks.signal?.aborted) return;
|
|
250
|
+
throw err;
|
|
251
|
+
}
|
|
252
|
+
const { done, value } = read;
|
|
253
|
+
if (done) {
|
|
254
|
+
drained = true;
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
209
257
|
hooks.onActivity?.();
|
|
210
258
|
let text: string = (heldCR ? "\r" : "") + decoder.decode(value, { stream: true });
|
|
211
259
|
heldCR = text.endsWith("\r");
|
|
@@ -222,7 +270,7 @@ export async function* parseSseStream(
|
|
|
222
270
|
|
|
223
271
|
let boundary = buffer.indexOf("\n\n", from);
|
|
224
272
|
while (boundary !== -1) {
|
|
225
|
-
yield* emit(buffer.slice(0, boundary));
|
|
273
|
+
if (yield* emit(buffer.slice(0, boundary))) return;
|
|
226
274
|
buffer = buffer.slice(boundary + 2);
|
|
227
275
|
from = 0;
|
|
228
276
|
boundary = buffer.indexOf("\n\n");
|
|
@@ -240,10 +288,32 @@ export async function* parseSseStream(
|
|
|
240
288
|
// hand — dropping it loses the last delta, or the usage record.
|
|
241
289
|
yield* emit(buffer + decoder.decode() + (heldCR ? "\n" : ""));
|
|
242
290
|
} finally {
|
|
291
|
+
if (!drained) reader.cancel().catch(() => {});
|
|
243
292
|
reader.releaseLock();
|
|
244
293
|
}
|
|
245
294
|
}
|
|
246
295
|
|
|
296
|
+
/**
|
|
297
|
+
* Parse one SSE payload as the JSON event it must be.
|
|
298
|
+
*
|
|
299
|
+
* A frame that is not JSON used to be skipped as "a keep-alive or a frame we do
|
|
300
|
+
* not model". But `parseSseStream` already drops comments and empty frames,
|
|
301
|
+
* which is every keep-alive the vendors send, so what reaches here unparsable
|
|
302
|
+
* is a broken stream: a delta, a tool call or the finish itself, gone, with the
|
|
303
|
+
* turn still reading as whole. It fails as `network`, the same as a socket that
|
|
304
|
+
* dies mid-read (cc-proxy fails these too).
|
|
305
|
+
*/
|
|
306
|
+
export function parseFrame<T>(data: string, provider: string): T {
|
|
307
|
+
try {
|
|
308
|
+
return JSON.parse(data) as T;
|
|
309
|
+
} catch (err) {
|
|
310
|
+
throw new ProviderError(provider, "network", `${provider}: a stream frame was not JSON`, {
|
|
311
|
+
body: data.slice(0, 2_000),
|
|
312
|
+
cause: err,
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
247
317
|
/**
|
|
248
318
|
* POST an SSE request and stream its payloads: the envelope (auth, the
|
|
249
319
|
* classified error, the retry hints) plus `parseSseStream`.
|
|
@@ -254,6 +324,16 @@ export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
|
|
|
254
324
|
// the POST: a backend that withholds them until its first output token (the
|
|
255
325
|
// ChatGPT backend does, for minutes on a large high-effort turn) is working.
|
|
256
326
|
opts.onActivity?.();
|
|
327
|
+
// The window's fill rides every response, the ones that go through too.
|
|
328
|
+
// Read only off a refusal, a caller learns the window was filling once it
|
|
329
|
+
// is already full.
|
|
330
|
+
if (opts.onRateLimit) {
|
|
331
|
+
const reading = parseRateLimitReset(res.headers);
|
|
332
|
+
const { utilization, resetAtMs } = reading;
|
|
333
|
+
if (utilization !== undefined && resetAtMs !== undefined) {
|
|
334
|
+
opts.onRateLimit({ ...reading, utilization, resetAtMs });
|
|
335
|
+
}
|
|
336
|
+
}
|
|
257
337
|
if (!res.ok) throw await errorFor(opts.provider, res);
|
|
258
338
|
// A 2xx with no body at all is an upstream anomaly, not a request we got
|
|
259
339
|
// wrong — worth the same retry a 5xx gets.
|
package/src/turn.ts
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// One turn, drained. Every agent loop reads a stream the same way: text and
|
|
2
|
+
// reasoning joined, tool-call fragments put back together by index, usage and
|
|
3
|
+
// finish taken from the last chunk that states them. The apps that use this
|
|
4
|
+
// package each wrote that by hand, a little differently each time, and the
|
|
5
|
+
// one in `drainStream` dropped the tool calls altogether.
|
|
6
|
+
//
|
|
7
|
+
// It assembles and nothing else. Arguments stay the raw string the seam
|
|
8
|
+
// promises, because tool-args.ts heals them and tools.ts validates them. And a
|
|
9
|
+
// turn that said nothing is not judged here: that is `requireContent`'s call
|
|
10
|
+
// (invariant 13), made where the retry that can fix it still can.
|
|
11
|
+
import { ProviderError } from "./errors.ts";
|
|
12
|
+
import {
|
|
13
|
+
EMPTY_USAGE,
|
|
14
|
+
type ChatMessage,
|
|
15
|
+
type FinishReason,
|
|
16
|
+
type ProviderChunk,
|
|
17
|
+
type ReasoningItems,
|
|
18
|
+
type TokenUsage,
|
|
19
|
+
type ToolCall,
|
|
20
|
+
} from "./types.ts";
|
|
21
|
+
|
|
22
|
+
/** One drained turn: the message a loop appends, whose `toolCalls` are the
|
|
23
|
+
* calls it runs, and what the turn cost. */
|
|
24
|
+
export interface Turn {
|
|
25
|
+
/** Ready to append to history as it is. `toolCalls` holds the calls to run,
|
|
26
|
+
* in the order the provider indexed them, and is absent when there are none. */
|
|
27
|
+
message: Extract<ChatMessage, { role: "assistant" }>;
|
|
28
|
+
/** The last usage the stream reported, or zeros when it reported none. */
|
|
29
|
+
usage: TokenUsage;
|
|
30
|
+
/** As the provider stated it, and null when it stated none. */
|
|
31
|
+
finishReason: FinishReason | null;
|
|
32
|
+
/** The endpoint and model that answered, when a fallback provider says. */
|
|
33
|
+
source?: { provider: string; model: string };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Drain one turn into the assistant message it produced, plus its usage,
|
|
38
|
+
* finish reason and source.
|
|
39
|
+
*
|
|
40
|
+
* ```ts
|
|
41
|
+
* const turn = await collectTurn(provider.createStream(history, tools));
|
|
42
|
+
* history.push(turn.message);
|
|
43
|
+
* for (const call of turn.message.toolCalls ?? []) history.push(await run(call));
|
|
44
|
+
* ```
|
|
45
|
+
*
|
|
46
|
+
* @throws {ProviderError} `overload` when a tool call ends the stream with no
|
|
47
|
+
* name. Whatever the stream itself throws passes through unchanged.
|
|
48
|
+
*/
|
|
49
|
+
export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise<Turn> {
|
|
50
|
+
let content = "";
|
|
51
|
+
let reasoning = "";
|
|
52
|
+
// Not concatenated: this half of the record is a payload the provider owns,
|
|
53
|
+
// and it arrives whole on one delta rather than in fragments. Dropped here,
|
|
54
|
+
// a drained turn replays only half its own reasoning on the next round,
|
|
55
|
+
// which is the failure `reasoningDetails` exists to prevent.
|
|
56
|
+
let reasoningDetails: unknown[] | undefined;
|
|
57
|
+
let reasoningItems: ReasoningItems | undefined;
|
|
58
|
+
const calls = new Map<number, ToolCall>();
|
|
59
|
+
// Cumulative on some providers and final-only on others: the LAST one wins.
|
|
60
|
+
let usage: TokenUsage = EMPTY_USAGE;
|
|
61
|
+
let finishReason: FinishReason | null = null;
|
|
62
|
+
let source: Turn["source"];
|
|
63
|
+
|
|
64
|
+
// Read to the end, past the finish chunk: the OpenAI shape sends its usage
|
|
65
|
+
// after it, and a loop that stopped at the finish would bill every turn at zero.
|
|
66
|
+
for await (const chunk of stream) {
|
|
67
|
+
if (chunk.source) source = chunk.source;
|
|
68
|
+
if (chunk.content) content += chunk.content;
|
|
69
|
+
if (chunk.reasoning) reasoning += chunk.reasoning;
|
|
70
|
+
if (chunk.reasoningDetails?.length) reasoningDetails = chunk.reasoningDetails;
|
|
71
|
+
if (chunk.reasoningItems?.items.length) reasoningItems = chunk.reasoningItems;
|
|
72
|
+
for (const delta of chunk.toolCalls ?? []) {
|
|
73
|
+
const call = calls.get(delta.index) ?? { id: "", name: "", arguments: "" };
|
|
74
|
+
// Identity is the last value STATED, so an empty one never erases a real
|
|
75
|
+
// one: Anthropic opens a call with `""` for what it lacks, and the
|
|
76
|
+
// ChatGPT backend states the real call_id and name only on the call's
|
|
77
|
+
// closing event. A call that never states an id keeps `""`. The four
|
|
78
|
+
// adapters here mint one before their finish (`settleToolCalls`); for any
|
|
79
|
+
// other Provider, one made up here would be ours and not the wire's
|
|
80
|
+
// (invariant 10), and `repairHistory` already names it the same way on
|
|
81
|
+
// every send. Arguments are fragments, joined in arrival order.
|
|
82
|
+
if (delta.id) call.id = delta.id;
|
|
83
|
+
if (delta.name) call.name = delta.name;
|
|
84
|
+
if (delta.thoughtSignature) call.thoughtSignature = delta.thoughtSignature;
|
|
85
|
+
call.arguments += delta.arguments ?? "";
|
|
86
|
+
calls.set(delta.index, call);
|
|
87
|
+
}
|
|
88
|
+
if (chunk.usage) usage = chunk.usage;
|
|
89
|
+
if (chunk.finishReason) finishReason = chunk.finishReason;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// By index, not by arrival: parallel calls interleave their fragments.
|
|
93
|
+
const indexed = [...calls].sort(([a], [b]) => a - b);
|
|
94
|
+
const nameless = indexed.find(([, call]) => !call.name);
|
|
95
|
+
if (nameless) {
|
|
96
|
+
const [index, call] = nameless;
|
|
97
|
+
// `overload`, the kind for a provider's own anomaly inside a 200 (an empty
|
|
98
|
+
// body, a turn that completed with nothing): our request was fine, the
|
|
99
|
+
// vendor's frame was not. A retry may well get a whole call, and another
|
|
100
|
+
// model is a fair next try if this one keeps doing it. `invalid` would
|
|
101
|
+
// blame our request and stop the run over a fault that is not ours. The
|
|
102
|
+
// four adapters here fail such a call themselves, with the same kind
|
|
103
|
+
// (`settleToolCalls`); this gives any other Provider the same answer.
|
|
104
|
+
const provider = source?.provider ?? "turn";
|
|
105
|
+
const which = call.id || `at index ${index}`;
|
|
106
|
+
throw new ProviderError(
|
|
107
|
+
provider,
|
|
108
|
+
"overload",
|
|
109
|
+
`${provider}: tool call ${which} ended with no name, so there is no tool to run`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
const toolCalls = indexed.map(([, call]) => call);
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
message: {
|
|
116
|
+
role: "assistant",
|
|
117
|
+
content,
|
|
118
|
+
...(reasoning ? { reasoning } : {}),
|
|
119
|
+
...(reasoningDetails ? { reasoningDetails } : {}),
|
|
120
|
+
...(reasoningItems ? { reasoningItems } : {}),
|
|
121
|
+
...(toolCalls.length > 0 ? { toolCalls } : {}),
|
|
122
|
+
},
|
|
123
|
+
usage,
|
|
124
|
+
finishReason,
|
|
125
|
+
...(source ? { source } : {}),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface Completion {
|
|
130
|
+
text: string;
|
|
131
|
+
/** Present when the stream identifies the endpoint that answered. */
|
|
132
|
+
provider?: string;
|
|
133
|
+
reasoning: string;
|
|
134
|
+
/** Present only where the provider sent one. See ChatMessage.reasoningDetails. */
|
|
135
|
+
reasoningDetails?: unknown[];
|
|
136
|
+
/** Present only where the provider sent them. See ChatMessage.reasoningItems. */
|
|
137
|
+
reasoningItems?: ReasoningItems;
|
|
138
|
+
/** The calls the model asked for, in index order, and empty when it asked
|
|
139
|
+
* for none. Before this field a drained turn that called a tool read as a
|
|
140
|
+
* turn that said nothing. */
|
|
141
|
+
toolCalls: ToolCall[];
|
|
142
|
+
usage: TokenUsage;
|
|
143
|
+
finishReason: FinishReason | null;
|
|
144
|
+
model: string;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Drain a stream into a Completion: `collectTurn`, flattened for a caller
|
|
148
|
+
* that wants the answer rather than a message to append. `model` names the
|
|
149
|
+
* model unless a fallback provider stamped the one that answered. */
|
|
150
|
+
export async function drainStream(
|
|
151
|
+
stream: AsyncIterable<ProviderChunk>,
|
|
152
|
+
model: string,
|
|
153
|
+
): Promise<Completion> {
|
|
154
|
+
const { message, usage, finishReason, source } = await collectTurn(stream);
|
|
155
|
+
return {
|
|
156
|
+
text: message.content,
|
|
157
|
+
reasoning: message.reasoning ?? "",
|
|
158
|
+
...(message.reasoningDetails ? { reasoningDetails: message.reasoningDetails } : {}),
|
|
159
|
+
...(message.reasoningItems ? { reasoningItems: message.reasoningItems } : {}),
|
|
160
|
+
toolCalls: message.toolCalls ?? [],
|
|
161
|
+
usage,
|
|
162
|
+
finishReason,
|
|
163
|
+
model: source?.model ?? model,
|
|
164
|
+
...(source ? { provider: source.provider } : {}),
|
|
165
|
+
};
|
|
166
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// the lingua franca forces the other adapters to round-trip through a dialect
|
|
7
7
|
// that isn't theirs, and every quirk of that dialect then leaks into callers
|
|
8
8
|
// who never asked for it.
|
|
9
|
+
import type { RateLimitReading } from "./rate-limit.ts";
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* How hard the model thinks before answering. Absent = the provider's own
|
|
@@ -70,8 +71,9 @@ export type ContentPart = TextPart | ImagePart | FilePart;
|
|
|
70
71
|
|
|
71
72
|
/**
|
|
72
73
|
* A tool the model asked to run. `arguments` is the RAW JSON string, not a
|
|
73
|
-
* parsed object: it arrives in fragments and can be truncated mid-stream
|
|
74
|
-
*
|
|
74
|
+
* parsed object: it arrives in fragments and can be truncated mid-stream.
|
|
75
|
+
* `collectTurn` joins the fragments; healing and validating them is the
|
|
76
|
+
* consumer's job (see tools/define).
|
|
75
77
|
*/
|
|
76
78
|
export interface ToolCall {
|
|
77
79
|
id: string;
|
|
@@ -150,7 +152,7 @@ export interface ToolDefinition {
|
|
|
150
152
|
|
|
151
153
|
export type FinishReason = "stop" | "length" | "tool_calls" | "content_filter";
|
|
152
154
|
|
|
153
|
-
/** Incremental tool-call fragment, assembled by `index`
|
|
155
|
+
/** Incremental tool-call fragment, assembled by `index`. `collectTurn` does it. */
|
|
154
156
|
export interface ToolCallDelta {
|
|
155
157
|
index: number;
|
|
156
158
|
id?: string;
|
|
@@ -384,9 +386,10 @@ export interface StreamOptions {
|
|
|
384
386
|
*/
|
|
385
387
|
jsonWithTools?: JsonWithTools;
|
|
386
388
|
/**
|
|
387
|
-
* Stable id for the conversation or job this call belongs to. Sent
|
|
388
|
-
*
|
|
389
|
-
*
|
|
389
|
+
* Stable id for the conversation or job this call belongs to. Sent in the
|
|
390
|
+
* `sessionHeader` of endpoints that name one (OpenCode Go routes and caches
|
|
391
|
+
* by it), and as `prompt_cache_key` on the Responses shape. A call without
|
|
392
|
+
* one sends the provider's `instanceSessionId` in that header instead.
|
|
390
393
|
*/
|
|
391
394
|
sessionId?: string;
|
|
392
395
|
/**
|
|
@@ -397,6 +400,18 @@ export interface StreamOptions {
|
|
|
397
400
|
* works, on the watchdog's progress clock alone.
|
|
398
401
|
*/
|
|
399
402
|
onActivity?: () => void;
|
|
403
|
+
/**
|
|
404
|
+
* Called once per response whose headers report how full a subscription
|
|
405
|
+
* window is, whatever its status: `utilization` (1 is full), `resetAtMs`,
|
|
406
|
+
* and `window` when the headers name it. A caller sees the window fill
|
|
407
|
+
* turn by turn, instead of learning about it from the 429 that ends a run.
|
|
408
|
+
*
|
|
409
|
+
* Every adapter forwards it. Two backends send the headers today: Claude
|
|
410
|
+
* subscriptions (Anthropic's unified 5h/7d headers) and the ChatGPT backend
|
|
411
|
+
* (`x-codex-*`). A response that reports no window never calls it, which
|
|
412
|
+
* covers API-key accounts on every vendor and all of Gemini.
|
|
413
|
+
*/
|
|
414
|
+
onRateLimit?: (reading: RateLimitReading) => void;
|
|
400
415
|
/**
|
|
401
416
|
* `false` asks for at most one tool call per turn, for tools that must run
|
|
402
417
|
* in order, or a backend lane that refuses parallel calls. Absent or `true`
|
|
@@ -427,62 +442,6 @@ export interface Provider {
|
|
|
427
442
|
): AsyncIterable<ProviderChunk>;
|
|
428
443
|
}
|
|
429
444
|
|
|
430
|
-
export interface Completion {
|
|
431
|
-
text: string;
|
|
432
|
-
/** Present when the stream identifies the endpoint that answered. */
|
|
433
|
-
provider?: string;
|
|
434
|
-
reasoning: string;
|
|
435
|
-
/** Present only where the provider sent one. See ChatMessage.reasoningDetails. */
|
|
436
|
-
reasoningDetails?: unknown[];
|
|
437
|
-
/** Present only where the provider sent them. See ChatMessage.reasoningItems. */
|
|
438
|
-
reasoningItems?: ReasoningItems;
|
|
439
|
-
usage: TokenUsage;
|
|
440
|
-
finishReason: FinishReason | null;
|
|
441
|
-
model: string;
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
/** Drain a no-tools stream into a Completion. Usage chunks are cumulative on
|
|
445
|
-
* some providers and final-only on others: the LAST one wins. */
|
|
446
|
-
export async function drainStream(
|
|
447
|
-
stream: AsyncIterable<ProviderChunk>,
|
|
448
|
-
model: string,
|
|
449
|
-
): Promise<Completion> {
|
|
450
|
-
let text = "";
|
|
451
|
-
let reasoning = "";
|
|
452
|
-
// Not concatenated: this half of the record is a payload the provider owns,
|
|
453
|
-
// and it arrives whole on one delta rather than in fragments. Dropped here,
|
|
454
|
-
// a drained turn replays only half its own reasoning on the next round —
|
|
455
|
-
// which is the failure `reasoningDetails` exists to prevent.
|
|
456
|
-
let reasoningDetails: unknown[] | undefined;
|
|
457
|
-
let reasoningItems: ReasoningItems | undefined;
|
|
458
|
-
let usage: TokenUsage = EMPTY_USAGE;
|
|
459
|
-
let finishReason: FinishReason | null = null;
|
|
460
|
-
let source: ProviderChunk["source"];
|
|
461
|
-
for await (const chunk of stream) {
|
|
462
|
-
if (chunk.source) source = chunk.source;
|
|
463
|
-
if (chunk.type === "delta") {
|
|
464
|
-
if (chunk.content) text += chunk.content;
|
|
465
|
-
if (chunk.reasoning) reasoning += chunk.reasoning;
|
|
466
|
-
if (chunk.reasoningDetails?.length) reasoningDetails = chunk.reasoningDetails;
|
|
467
|
-
if (chunk.reasoningItems?.items.length) reasoningItems = chunk.reasoningItems;
|
|
468
|
-
} else if (chunk.type === "usage" && chunk.usage) {
|
|
469
|
-
usage = chunk.usage;
|
|
470
|
-
} else if (chunk.type === "finish" && chunk.finishReason) {
|
|
471
|
-
finishReason = chunk.finishReason;
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
return {
|
|
475
|
-
text,
|
|
476
|
-
reasoning,
|
|
477
|
-
...(reasoningDetails ? { reasoningDetails } : {}),
|
|
478
|
-
...(reasoningItems ? { reasoningItems } : {}),
|
|
479
|
-
usage,
|
|
480
|
-
finishReason,
|
|
481
|
-
model: source?.model ?? model,
|
|
482
|
-
...(source ? { provider: source.provider } : {}),
|
|
483
|
-
};
|
|
484
|
-
}
|
|
485
|
-
|
|
486
445
|
/**
|
|
487
446
|
* Prepare a history for a thinking-DISABLED turn: strip `reasoning` from every
|
|
488
447
|
* assistant message.
|
package/src/watchdog.ts
CHANGED
|
@@ -24,10 +24,14 @@
|
|
|
24
24
|
// scratch, which re-ran the reasoning into the same deadline.
|
|
25
25
|
//
|
|
26
26
|
// When either fires, the watchdog aborts ITS OWN controller and the caller's
|
|
27
|
-
// signal is only bridged in
|
|
28
|
-
// distinguishable from our timeout. One is their cancel and is never retried
|
|
29
|
-
//
|
|
30
|
-
//
|
|
27
|
+
// signal is only bridged in, which is what keeps a person's Stop
|
|
28
|
+
// distinguishable from our timeout. One is their cancel and is never retried.
|
|
29
|
+
// The other is ours and transient, with one exception: a progress timeout
|
|
30
|
+
// before the response started carries `shouldRetry: false`. That wait was a
|
|
31
|
+
// backend thinking with its headers held back, and a replay sends the same
|
|
32
|
+
// prompt into the same 300s and pays for the reasoning again, up to three
|
|
33
|
+
// times (cc-proxy never replays slow headers). Its kind stays `timeout`, so a
|
|
34
|
+
// fallback pool still moves on to its next candidate.
|
|
31
35
|
import { ProviderError } from "./errors.ts";
|
|
32
36
|
import type {
|
|
33
37
|
ChatMessage,
|
|
@@ -91,6 +95,9 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
|
|
|
91
95
|
let firstChunk: number | null = null;
|
|
92
96
|
let fired: ProviderError | null = null;
|
|
93
97
|
let disposed = false;
|
|
98
|
+
// The first sign of life (the headers, a keep-alive, a chunk). The idle
|
|
99
|
+
// clock waits for it, and a timeout that fires before it is not replayed.
|
|
100
|
+
let responded = false;
|
|
94
101
|
|
|
95
102
|
// The bridge is structural rather than an event listener: AbortSignal.any
|
|
96
103
|
// aborts synchronously when an input is ALREADY aborted, which is the race
|
|
@@ -101,7 +108,13 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
|
|
|
101
108
|
|
|
102
109
|
function deadline(ms: number, what: string): ReturnType<typeof setTimeout> {
|
|
103
110
|
const timer = setTimeout(() => {
|
|
104
|
-
|
|
111
|
+
// Only the progress clock runs before the response starts, so this is
|
|
112
|
+
// the backend that thinks with its headers held back (see the header).
|
|
113
|
+
fired = responded
|
|
114
|
+
? new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`)
|
|
115
|
+
: new ProviderError(provider, "timeout", `no response in ${ms / 1000}s`, {
|
|
116
|
+
shouldRetry: false,
|
|
117
|
+
});
|
|
105
118
|
timeout.abort(fired);
|
|
106
119
|
}, ms);
|
|
107
120
|
// An orphaned watch — its consumer gone, dispose never called — must not
|
|
@@ -117,6 +130,7 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
|
|
|
117
130
|
const live = () => !disposed && !signal.aborted;
|
|
118
131
|
|
|
119
132
|
function sawActivity() {
|
|
133
|
+
responded = true;
|
|
120
134
|
clearTimeout(idle);
|
|
121
135
|
if (live()) idle = deadline(idleMs, "a byte");
|
|
122
136
|
}
|
|
@@ -134,7 +148,10 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
|
|
|
134
148
|
classify(err: unknown) {
|
|
135
149
|
// Our deadline, not theirs — and not the caller's Stop.
|
|
136
150
|
if (fired && !(callerSignal?.aborted ?? false)) {
|
|
137
|
-
return new ProviderError(fired.provider, "timeout", fired.message, {
|
|
151
|
+
return new ProviderError(fired.provider, "timeout", fired.message, {
|
|
152
|
+
shouldRetry: fired.shouldRetry,
|
|
153
|
+
cause: err,
|
|
154
|
+
});
|
|
138
155
|
}
|
|
139
156
|
return err;
|
|
140
157
|
},
|
|
@@ -278,6 +295,8 @@ export interface WatchdogOptions {
|
|
|
278
295
|
* copies. The result is still a `Provider`, so it composes unchanged with
|
|
279
296
|
* `withStreamRetry` and `streamWithBackupModels`. Timeouts and empty turns
|
|
280
297
|
* without a `length` finish are transient, so wrapping it in a retry is safe.
|
|
298
|
+
* The one timeout a replay would make worse, no response at all within the
|
|
299
|
+
* progress window, says so itself with `shouldRetry: false`.
|
|
281
300
|
*/
|
|
282
301
|
export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Provider {
|
|
283
302
|
return {
|