@providerkit/core 0.27.0 → 0.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/with-token.d.ts +3 -1
- package/dist/auth/with-token.d.ts.map +1 -1
- package/dist/auth/with-token.js +23 -4
- package/dist/auth/with-token.js.map +1 -1
- package/dist/errors.d.ts +2 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +6 -0
- package/dist/errors.js.map +1 -1
- package/dist/fallback.d.ts +6 -4
- package/dist/fallback.d.ts.map +1 -1
- package/dist/fallback.js +13 -7
- package/dist/fallback.js.map +1 -1
- package/dist/key-pool.d.ts +12 -1
- package/dist/key-pool.d.ts.map +1 -1
- package/dist/key-pool.js +15 -3
- package/dist/key-pool.js.map +1 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +2 -1
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/factory.js +4 -1
- package/dist/providers/factory.js.map +1 -1
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +4 -1
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +2 -1
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/responses.d.ts.map +1 -1
- package/dist/providers/responses.js +2 -1
- package/dist/providers/responses.js.map +1 -1
- package/dist/types.d.ts +10 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
- package/src/auth/with-token.ts +24 -7
- package/src/errors.ts +7 -0
- package/src/fallback.ts +40 -18
- package/src/key-pool.ts +58 -35
- package/src/providers/anthropic.ts +8 -1
- package/src/providers/factory.ts +4 -1
- package/src/providers/gemini.ts +10 -1
- package/src/providers/openai.ts +8 -1
- package/src/providers/responses.ts +8 -1
- package/src/types.ts +10 -0
package/src/fallback.ts
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
classify,
|
|
3
|
+
parseRetryAfterMs,
|
|
4
|
+
ProviderError,
|
|
5
|
+
validateMaxAttempts,
|
|
6
|
+
type ErrorKind,
|
|
7
|
+
} from "./errors.ts";
|
|
2
8
|
import type { RateLimitWindow } from "./rate-limit.ts";
|
|
3
9
|
import type { ProviderPresetId } from "./presets.ts";
|
|
4
10
|
import { sleep } from "./retry.ts";
|
|
5
|
-
import type { Provider, ProviderChunk } from "./types.ts";
|
|
11
|
+
import type { Provider, ProviderChunk, StreamOptions } from "./types.ts";
|
|
6
12
|
import { STREAM_PROGRESS_MS, withWatchdog, type WatchdogOptions } from "./watchdog.ts";
|
|
7
13
|
import { createPresetProvider, type PresetProviderConfig } from "./providers/factory.ts";
|
|
8
14
|
|
|
@@ -240,21 +246,32 @@ export class FallbackPool<T> {
|
|
|
240
246
|
async with<R>(
|
|
241
247
|
attempt: (candidate: T, signal: AbortSignal) => Promise<R>,
|
|
242
248
|
signal?: AbortSignal,
|
|
249
|
+
opts: Pick<StreamOptions, "maxAttempts"> = {},
|
|
243
250
|
): Promise<R> {
|
|
244
|
-
for await (const result of this.stream(
|
|
245
|
-
|
|
246
|
-
|
|
251
|
+
for await (const result of this.stream(
|
|
252
|
+
async function* (candidate, inner) {
|
|
253
|
+
yield await attempt(candidate, inner);
|
|
254
|
+
},
|
|
255
|
+
signal,
|
|
256
|
+
opts,
|
|
257
|
+
))
|
|
247
258
|
return result;
|
|
248
259
|
throw new Error("providerkit: fallback attempt ended without a result");
|
|
249
260
|
}
|
|
250
261
|
|
|
251
262
|
/** Only a pre-output failure can switch provider. A later failure still
|
|
252
|
-
* cools the endpoint for FUTURE calls but never restarts the current answer.
|
|
263
|
+
* cools the endpoint for FUTURE calls but never restarts the current answer.
|
|
264
|
+
* `maxAttempts` counts callback invocations, not cooled candidates or waits
|
|
265
|
+
* for a probe. Each nested recovery layer has its own limit. */
|
|
253
266
|
async *stream<R>(
|
|
254
267
|
attempt: (candidate: T, signal: AbortSignal) => AsyncIterable<R>,
|
|
255
268
|
signal?: AbortSignal,
|
|
269
|
+
opts: Pick<StreamOptions, "maxAttempts"> = {},
|
|
256
270
|
): AsyncGenerator<R> {
|
|
271
|
+
const { maxAttempts } = opts;
|
|
272
|
+
validateMaxAttempts(maxAttempts);
|
|
257
273
|
signal?.throwIfAborted();
|
|
274
|
+
let attempts = 0;
|
|
258
275
|
let hasFailed = false;
|
|
259
276
|
let lastError: unknown;
|
|
260
277
|
for (let round = 0; ; round++) {
|
|
@@ -276,6 +293,7 @@ export class FallbackPool<T> {
|
|
|
276
293
|
const inner = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
|
|
277
294
|
let emitted = false;
|
|
278
295
|
try {
|
|
296
|
+
attempts++;
|
|
279
297
|
for await (const chunk of attempt(candidate, inner)) {
|
|
280
298
|
inner.throwIfAborted();
|
|
281
299
|
if (!emitted) {
|
|
@@ -311,7 +329,7 @@ export class FallbackPool<T> {
|
|
|
311
329
|
this.options.onCooldown?.({ candidate, error, kind, retryAtMs, window });
|
|
312
330
|
hasFailed = true;
|
|
313
331
|
lastError = error;
|
|
314
|
-
if (emitted) throw error;
|
|
332
|
+
if (emitted || (maxAttempts !== undefined && attempts >= maxAttempts)) throw error;
|
|
315
333
|
} finally {
|
|
316
334
|
controller.abort();
|
|
317
335
|
// Only the owner releases its probe. Clearing whatever probe was set
|
|
@@ -380,17 +398,21 @@ export function withFallbackProviders(
|
|
|
380
398
|
model: primary.model,
|
|
381
399
|
fallbacks,
|
|
382
400
|
createStream(messages, tools, opts = {}) {
|
|
383
|
-
return fallbacks.stream(
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
401
|
+
return fallbacks.stream(
|
|
402
|
+
async function* (provider, signal): AsyncGenerator<ProviderChunk> {
|
|
403
|
+
const { model: override, ...rest } = opts;
|
|
404
|
+
const model = provider === primary ? (override ?? provider.model) : provider.model;
|
|
405
|
+
for await (const chunk of provider.createStream(messages, tools, {
|
|
406
|
+
...rest,
|
|
407
|
+
model,
|
|
408
|
+
signal,
|
|
409
|
+
})) {
|
|
410
|
+
yield { ...chunk, source: chunk.source ?? { provider: provider.id, model } };
|
|
411
|
+
}
|
|
412
|
+
},
|
|
413
|
+
opts.signal,
|
|
414
|
+
opts,
|
|
415
|
+
);
|
|
394
416
|
},
|
|
395
417
|
};
|
|
396
418
|
}
|
package/src/key-pool.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// error propagates untouched: a transient 503 must not switch a single-key
|
|
14
14
|
// deployment off for a minute, where the caller's own retry is the whole
|
|
15
15
|
// recovery there is.
|
|
16
|
-
import { isTransient, ProviderError } from "./errors.ts";
|
|
16
|
+
import { isTransient, ProviderError, validateMaxAttempts } from "./errors.ts";
|
|
17
17
|
import type { ErrorKind } from "./errors.ts";
|
|
18
18
|
import type {
|
|
19
19
|
ChatMessage,
|
|
@@ -55,6 +55,12 @@ export interface KeyPoolOptions {
|
|
|
55
55
|
now?: () => number;
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
export interface KeyPoolCallOptions {
|
|
59
|
+
/** Maximum callback invocations, including the first. Must be a positive
|
|
60
|
+
* integer. Omit to try every available key as usual. */
|
|
61
|
+
maxAttempts?: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
58
64
|
/** No key can serve right now. `retryAtMs` = when the soonest one is back.
|
|
59
65
|
*
|
|
60
66
|
* A `ProviderError` of kind `rate` that carries that moment as `resetAtMs` and
|
|
@@ -117,14 +123,27 @@ export class KeyPool {
|
|
|
117
123
|
* transient failures. Everything else (a bad request, a content block) throws
|
|
118
124
|
* straight through: it would fail identically on every key, and spending the
|
|
119
125
|
* pool on it only turns one bad request into an outage.
|
|
126
|
+
*
|
|
127
|
+
* Set `maxAttempts: 1` for a paid request that must not be sent again. Only
|
|
128
|
+
* actual callback invocations count, not keys skipped during cooldown. At
|
|
129
|
+
* the cap the key is still benched as usual, but its original error is thrown
|
|
130
|
+
* instead of trying another key. This does not limit retries inside `fn`
|
|
131
|
+
* or in a wrapper around `with`.
|
|
120
132
|
*/
|
|
121
|
-
async with<T>(
|
|
133
|
+
async with<T>(
|
|
134
|
+
fn: (apiKey: string, tier: KeyTier) => Promise<T>,
|
|
135
|
+
opts: KeyPoolCallOptions = {},
|
|
136
|
+
): Promise<T> {
|
|
137
|
+
const { maxAttempts } = opts;
|
|
138
|
+
validateMaxAttempts(maxAttempts);
|
|
122
139
|
const candidates = this.candidates(this.now());
|
|
123
140
|
if (candidates.length === 0) throw this.noneAvailable();
|
|
124
141
|
|
|
142
|
+
let attempts = 0;
|
|
125
143
|
let last: unknown;
|
|
126
144
|
for (const key of candidates) {
|
|
127
145
|
try {
|
|
146
|
+
attempts++;
|
|
128
147
|
return await fn(key.apiKey, key.tier);
|
|
129
148
|
} catch (err) {
|
|
130
149
|
last = err;
|
|
@@ -135,6 +154,7 @@ export class KeyPool {
|
|
|
135
154
|
if (cooldown === null) throw err;
|
|
136
155
|
key.evictedUntil = this.now() + cooldown.forMs;
|
|
137
156
|
this.onEvict?.({ tier: key.tier, kind: cooldown.kind, forMs: cooldown.forMs });
|
|
157
|
+
if (maxAttempts !== undefined && attempts >= maxAttempts) throw err;
|
|
138
158
|
}
|
|
139
159
|
}
|
|
140
160
|
|
|
@@ -238,41 +258,44 @@ export function withKeyPool(pool: KeyPool, factory: (apiKey: string) => Provider
|
|
|
238
258
|
tools: ToolDefinition[],
|
|
239
259
|
opts: StreamOptions = {},
|
|
240
260
|
): AsyncIterable<ProviderChunk> {
|
|
241
|
-
const opened = await pool.with(
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
261
|
+
const opened = await pool.with(
|
|
262
|
+
async (apiKey) => {
|
|
263
|
+
// One AbortController per attempt, chained to the caller's own signal
|
|
264
|
+
// (retry.ts's rule). Finalizing a generator is not cancellation:
|
|
265
|
+
// `return()` unwinds the adapter down to `streamSse`'s finalizer, which
|
|
266
|
+
// only releases the reader's lock — the response body stays live and
|
|
267
|
+
// the request is never aborted. Without this controller an abandoned
|
|
268
|
+
// attempt keeps the provider generating the rest of the answer, holding
|
|
269
|
+
// a connection and a concurrency slot on the very key the pool is
|
|
270
|
+
// rotating away from.
|
|
271
|
+
const controller = new AbortController();
|
|
272
|
+
const onAbort = () => controller.abort(opts.signal?.reason);
|
|
273
|
+
if (opts.signal?.aborted) onAbort();
|
|
274
|
+
else opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
254
275
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
276
|
+
const stream = factory(apiKey).createStream(messages, tools, {
|
|
277
|
+
...opts,
|
|
278
|
+
signal: controller.signal,
|
|
279
|
+
});
|
|
280
|
+
const iterator = stream[Symbol.asyncIterator]();
|
|
281
|
+
const release = async () => {
|
|
282
|
+
opts.signal?.removeEventListener("abort", onAbort);
|
|
283
|
+
await close(iterator);
|
|
284
|
+
controller.abort();
|
|
285
|
+
};
|
|
265
286
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
287
|
+
try {
|
|
288
|
+
return { iterator, first: await iterator.next(), release };
|
|
289
|
+
} catch (err) {
|
|
290
|
+
// The request failed under the pool, which is about to try the next
|
|
291
|
+
// key: release this attempt before a second one opens against the
|
|
292
|
+
// same rate limit.
|
|
293
|
+
await release();
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
296
|
+
},
|
|
297
|
+
{ maxAttempts: opts.maxAttempts },
|
|
298
|
+
);
|
|
276
299
|
|
|
277
300
|
const { iterator, first, release } = opened;
|
|
278
301
|
try {
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// Anthropic-shape adapter — SSE from POST /v1/messages.
|
|
2
2
|
import { repairHistory } from "../context.ts";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
fileRefused,
|
|
5
|
+
ProviderError,
|
|
6
|
+
streamCut,
|
|
7
|
+
streamError,
|
|
8
|
+
validateMaxAttempts,
|
|
9
|
+
} from "../errors.ts";
|
|
4
10
|
import { schemaPrompt, toAnthropicToolSchema } from "../schema.ts";
|
|
5
11
|
import { parseToolArgs, settleToolCalls, type ToolCallIdentity } from "../tool-args.ts";
|
|
6
12
|
import { streamSse, apiUrl, parseFrame } from "../transport.ts";
|
|
@@ -422,6 +428,7 @@ export function createAnthropicProvider(config: AnthropicConfig): Provider {
|
|
|
422
428
|
tools: ToolDefinition[],
|
|
423
429
|
opts: StreamOptions = {},
|
|
424
430
|
): AsyncIterable<ProviderChunk> {
|
|
431
|
+
validateMaxAttempts(opts.maxAttempts);
|
|
425
432
|
const model = opts.model ?? config.model;
|
|
426
433
|
const maxTokens = opts.maxTokens ?? config.maxTokens ?? DEFAULT_MAX_TOKENS;
|
|
427
434
|
const effort = opts.effort ?? config.effort;
|
package/src/providers/factory.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// `effort` passes through to the adapter unchanged — the per-endpoint
|
|
11
11
|
// thinking dialects (Z.ai's explicit disabled marker, OpenRouter's floors)
|
|
12
12
|
// are the adapters' knowledge, and this factory must not re-derive them.
|
|
13
|
-
import { ProviderError } from "../errors.ts";
|
|
13
|
+
import { ProviderError, validateMaxAttempts } from "../errors.ts";
|
|
14
14
|
import { createAnthropicProvider } from "./anthropic.ts";
|
|
15
15
|
import { createGeminiProvider } from "./gemini.ts";
|
|
16
16
|
import { createOpenAIProvider } from "./openai.ts";
|
|
@@ -360,6 +360,8 @@ function wrapOpenCodeZen(
|
|
|
360
360
|
return {
|
|
361
361
|
...baseProvider,
|
|
362
362
|
async *createStream(messages, tools, opts = {}) {
|
|
363
|
+
const { maxAttempts } = opts;
|
|
364
|
+
validateMaxAttempts(maxAttempts);
|
|
363
365
|
const activeModel = opts.model ?? defaultModel;
|
|
364
366
|
const free = isZenFreeModel(activeModel);
|
|
365
367
|
const ses = await forgeOpenCodeSessionId(opts.sessionId ?? instanceSessionId);
|
|
@@ -431,6 +433,7 @@ function wrapOpenCodeZen(
|
|
|
431
433
|
// and refuse a stale key with 401. Retry anonymously when that happens.
|
|
432
434
|
if (
|
|
433
435
|
free &&
|
|
436
|
+
maxAttempts !== 1 &&
|
|
434
437
|
!startAnonymous &&
|
|
435
438
|
!emitted &&
|
|
436
439
|
err instanceof ProviderError &&
|
package/src/providers/gemini.ts
CHANGED
|
@@ -6,7 +6,13 @@
|
|
|
6
6
|
// wire rather than @google/genai, because this package ships zero dependencies
|
|
7
7
|
// and the SDK is a Node-shaped one.
|
|
8
8
|
import { repairHistory } from "../context.ts";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
ProviderError,
|
|
11
|
+
streamCut,
|
|
12
|
+
streamError,
|
|
13
|
+
validateMaxAttempts,
|
|
14
|
+
type ErrorKind,
|
|
15
|
+
} from "../errors.ts";
|
|
10
16
|
import { schemaPrompt, toAntigravityToolSchema, toGeminiToolSchema } from "../schema.ts";
|
|
11
17
|
import { parseToolArgs, settleToolCalls } from "../tool-args.ts";
|
|
12
18
|
import { streamSse, apiUrl } from "../transport.ts";
|
|
@@ -413,6 +419,8 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
|
|
|
413
419
|
tools: ToolDefinition[],
|
|
414
420
|
opts: StreamOptions = {},
|
|
415
421
|
): AsyncIterable<ProviderChunk> {
|
|
422
|
+
const { maxAttempts } = opts;
|
|
423
|
+
validateMaxAttempts(maxAttempts);
|
|
416
424
|
const effort = opts.effort ?? config.effort;
|
|
417
425
|
const maxTokens = opts.maxTokens ?? config.maxTokens;
|
|
418
426
|
// Repaired first: a result with no call, or a call with no result, is a
|
|
@@ -690,6 +698,7 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
|
|
|
690
698
|
} catch (err) {
|
|
691
699
|
const canFallback =
|
|
692
700
|
!emitted &&
|
|
701
|
+
maxAttempts !== 1 &&
|
|
693
702
|
baseIdx === 0 &&
|
|
694
703
|
bases.length > 1 &&
|
|
695
704
|
err instanceof ProviderError &&
|
package/src/providers/openai.ts
CHANGED
|
@@ -4,7 +4,13 @@
|
|
|
4
4
|
// OpenRouter, DeepSeek, GLM, Kimi, Groq, Together, vLLM, Ollama and LM Studio.
|
|
5
5
|
// Their divergences are small and named where they appear.
|
|
6
6
|
import { repairHistory } from "../context.ts";
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
fileRefused,
|
|
9
|
+
ProviderError,
|
|
10
|
+
streamCut,
|
|
11
|
+
streamError,
|
|
12
|
+
validateMaxAttempts,
|
|
13
|
+
} from "../errors.ts";
|
|
8
14
|
import { streamSse, apiUrl, parseFrame } from "../transport.ts";
|
|
9
15
|
import { settleToolCalls, type ToolCallIdentity } from "../tool-args.ts";
|
|
10
16
|
import { attributionHeaders } from "../attribution.ts";
|
|
@@ -563,6 +569,7 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
563
569
|
tools: ToolDefinition[],
|
|
564
570
|
opts: StreamOptions = {},
|
|
565
571
|
): AsyncIterable<ProviderChunk> {
|
|
572
|
+
validateMaxAttempts(opts.maxAttempts);
|
|
566
573
|
const effort = opts.effort ?? config.effort;
|
|
567
574
|
const model = opts.model ?? config.model;
|
|
568
575
|
|
|
@@ -11,7 +11,13 @@
|
|
|
11
11
|
// payload's own `type`. The transport yields only `data:` payloads, so this
|
|
12
12
|
// adapter reads `type` — which is what survives, and what gateways agree on.
|
|
13
13
|
import { repairHistory } from "../context.ts";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
fileRefused,
|
|
16
|
+
ProviderError,
|
|
17
|
+
streamCut,
|
|
18
|
+
streamError,
|
|
19
|
+
validateMaxAttempts,
|
|
20
|
+
} from "../errors.ts";
|
|
15
21
|
import { streamSse, apiUrl, parseFrame } from "../transport.ts";
|
|
16
22
|
import { parseRateLimitReset } from "../rate-limit.ts";
|
|
17
23
|
import { settleToolCalls } from "../tool-args.ts";
|
|
@@ -419,6 +425,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
419
425
|
tools: ToolDefinition[],
|
|
420
426
|
opts: StreamOptions = {},
|
|
421
427
|
): AsyncIterable<ProviderChunk> {
|
|
428
|
+
validateMaxAttempts(opts.maxAttempts);
|
|
422
429
|
const effort = opts.effort ?? config.effort;
|
|
423
430
|
const model = opts.model ?? config.model;
|
|
424
431
|
const museContributor = isMuseSparkContributor(model);
|
package/src/types.ts
CHANGED
|
@@ -356,6 +356,16 @@ export type JsonWithTools = "response_format" | "prompt";
|
|
|
356
356
|
export interface StreamOptions {
|
|
357
357
|
/** Override the provider's bound model for this call. */
|
|
358
358
|
model?: string;
|
|
359
|
+
/**
|
|
360
|
+
* Maximum attempts at each SDK recovery layer, including the first. Must
|
|
361
|
+
* be a positive integer. Omit to keep normal recovery. `1` prevents another
|
|
362
|
+
* inference request through model, key, host or token fallback, keeping the
|
|
363
|
+
* original failure and normal cooldown or credential renewal for later calls.
|
|
364
|
+
* Values above `1` are PER LAYER, not a total HTTP budget across nested pools.
|
|
365
|
+
* Initial token acquisition and credential refresh are not inference attempts.
|
|
366
|
+
* App-written outer retries need their own limit (`RetryOptions.maxAttempts`).
|
|
367
|
+
*/
|
|
368
|
+
maxAttempts?: number;
|
|
359
369
|
effort?: Effort;
|
|
360
370
|
/** Output ceiling. On most providers thinking and answer SHARE it, so a
|
|
361
371
|
* task emitting a large artifact must raise it or the tool-call JSON is
|