@providerkit/core 0.35.0 → 0.35.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/decisions.js +1 -1
- package/dist/decisions.js.map +1 -1
- package/dist/errors.d.ts +2 -7
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +98 -68
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +5 -4
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +5 -4
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/openai.d.ts +3 -34
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +10 -40
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/openrouter-billing.d.ts +27 -0
- package/dist/providers/openrouter-billing.d.ts.map +1 -0
- package/dist/providers/openrouter-billing.js +37 -0
- package/dist/providers/openrouter-billing.js.map +1 -0
- package/dist/providers/responses.d.ts +2 -1
- package/dist/providers/responses.d.ts.map +1 -1
- package/dist/providers/responses.js +26 -13
- package/dist/providers/responses.js.map +1 -1
- package/dist/rate-limit.d.ts +11 -8
- package/dist/rate-limit.d.ts.map +1 -1
- package/dist/rate-limit.js +123 -77
- package/dist/rate-limit.js.map +1 -1
- package/dist/transport.d.ts +6 -0
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +12 -9
- package/dist/transport.js.map +1 -1
- package/dist/watchdog.d.ts.map +1 -1
- package/dist/watchdog.js +4 -1
- package/dist/watchdog.js.map +1 -1
- package/package.json +1 -1
- package/src/decisions.ts +1 -1
- package/src/errors.ts +112 -69
- package/src/index.ts +9 -2
- package/src/providers/anthropic.ts +5 -4
- package/src/providers/gemini.ts +5 -4
- package/src/providers/openai.ts +29 -63
- package/src/providers/openrouter-billing.ts +42 -0
- package/src/providers/responses.ts +29 -15
- package/src/rate-limit.ts +132 -79
- package/src/transport.ts +14 -13
- package/src/watchdog.ts +4 -0
package/src/providers/openai.ts
CHANGED
|
@@ -4,6 +4,11 @@
|
|
|
4
4
|
// OpenRouter, DeepSeek, GLM, Kimi, Groq, Together, vLLM, Ollama and LM Studio.
|
|
5
5
|
// Their divergences are small and named where they appear.
|
|
6
6
|
import { repairHistory } from "../context.ts";
|
|
7
|
+
import {
|
|
8
|
+
isOpenRouterBillingHost,
|
|
9
|
+
openRouterCostUsd,
|
|
10
|
+
type OpenRouterBill,
|
|
11
|
+
} from "./openrouter-billing.ts";
|
|
7
12
|
import {
|
|
8
13
|
fileRefused,
|
|
9
14
|
ProviderError,
|
|
@@ -13,7 +18,7 @@ import {
|
|
|
13
18
|
} from "../errors.ts";
|
|
14
19
|
import type { HostedToolType } from "../hosted/types.ts";
|
|
15
20
|
import { refuseHostedTools, type HostedCodecs } from "./hosted-codecs.ts";
|
|
16
|
-
import {
|
|
21
|
+
import { openSse, apiUrl, parseFrame, withRedactedErrors, type FetchLike } from "../transport.ts";
|
|
17
22
|
import { settleToolCalls, type ToolCallIdentity } from "../tool-args.ts";
|
|
18
23
|
import { attributionHeaders } from "../attribution.ts";
|
|
19
24
|
import type {
|
|
@@ -319,7 +324,9 @@ export function effortParams(
|
|
|
319
324
|
case "opencode-go":
|
|
320
325
|
return goEffortParams(effort, model);
|
|
321
326
|
case "openai":
|
|
322
|
-
|
|
327
|
+
// The pinned OpenAI host above refuses `none` for gpt-5-mini. Only that measured id
|
|
328
|
+
// gets its floor; an unrequested effort still sends nothing.
|
|
329
|
+
return { reasoning_effort: level ?? (model === "gpt-5-mini" ? "minimal" : "none") };
|
|
323
330
|
case "off":
|
|
324
331
|
return {};
|
|
325
332
|
}
|
|
@@ -508,65 +515,26 @@ export interface OpenAIChunk {
|
|
|
508
515
|
};
|
|
509
516
|
finish_reason?: string | null;
|
|
510
517
|
}[];
|
|
511
|
-
usage?:
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
/** The server tool calls OpenRouter ran for the turn. */
|
|
527
|
-
server_tool_use_details?: { web_search_requests?: number } | null;
|
|
528
|
-
} | null;
|
|
518
|
+
usage?:
|
|
519
|
+
| (OpenRouterBill & {
|
|
520
|
+
prompt_tokens?: number;
|
|
521
|
+
completion_tokens?: number;
|
|
522
|
+
prompt_tokens_details?: { cached_tokens?: number };
|
|
523
|
+
completion_tokens_details?: { reasoning_tokens?: number };
|
|
524
|
+
/** DeepSeek's native API reports the cache-hit count here instead of in
|
|
525
|
+
* `prompt_tokens_details`, and it is absent from every OpenAI SDK type. */
|
|
526
|
+
prompt_cache_hit_tokens?: number;
|
|
527
|
+
/** Kimi (Moonshot) puts it at the top level of `usage`. */
|
|
528
|
+
cached_tokens?: number;
|
|
529
|
+
/** The server tool calls OpenRouter ran for the turn. */
|
|
530
|
+
server_tool_use_details?: { web_search_requests?: number } | null;
|
|
531
|
+
})
|
|
532
|
+
| null;
|
|
529
533
|
/** Present only on the in-band failure below. Never beside a choice.
|
|
530
534
|
* `code` is the numeric HTTP status on the gateways, a slug on OpenAI. */
|
|
531
535
|
error?: { message?: string; code?: string | number; type?: string };
|
|
532
536
|
}
|
|
533
537
|
|
|
534
|
-
/** A price off the wire, or undefined when it is not one. A negative or a
|
|
535
|
-
* string falls back to the caller's rate rather than billing a nonsense
|
|
536
|
-
* number. 0 is a real price: the one a free model charges. */
|
|
537
|
-
function price(value: unknown): number | undefined {
|
|
538
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
/**
|
|
542
|
-
* What an OpenRouter call cost, in USD, or undefined when it did not say.
|
|
543
|
-
*
|
|
544
|
-
* Nothing has to ask for it. OpenRouter's usage-accounting docs (read
|
|
545
|
-
* 2026-09-26) say every response now carries the full usage record, cost
|
|
546
|
-
* included, on the last frame of a stream, and that `usage: { include: true }`
|
|
547
|
-
* is deprecated and does nothing. Measured the same day: a streamed call and a
|
|
548
|
-
* plain one, neither sending the flag, both came back with `usage.cost`, the
|
|
549
|
-
* stream on its last frame. So the request does not send it.
|
|
550
|
-
*
|
|
551
|
-
* With the caller's own provider key (BYOK), `cost` is only OpenRouter's fee.
|
|
552
|
-
* The inference is billed to the caller's provider account and arrives as
|
|
553
|
-
* `cost_details.upstream_inference_cost`, so the call cost the two together.
|
|
554
|
-
* Without that second figure there is no whole bill to report, and the
|
|
555
|
-
* caller's rate prices the call instead.
|
|
556
|
-
*
|
|
557
|
-
* Exported for the other OpenRouter wire that bills the same way: Jev's.
|
|
558
|
-
*/
|
|
559
|
-
export function openRouterCostUsd(usage: {
|
|
560
|
-
cost?: unknown;
|
|
561
|
-
is_byok?: boolean;
|
|
562
|
-
cost_details?: { upstream_inference_cost?: unknown } | null;
|
|
563
|
-
}): number | undefined {
|
|
564
|
-
const cost = price(usage.cost);
|
|
565
|
-
if (cost === undefined || usage.is_byok !== true) return cost;
|
|
566
|
-
const upstream = price(usage.cost_details?.upstream_inference_cost);
|
|
567
|
-
return upstream === undefined ? undefined : cost + upstream;
|
|
568
|
-
}
|
|
569
|
-
|
|
570
538
|
export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
571
539
|
return openaiProvider(config);
|
|
572
540
|
}
|
|
@@ -575,10 +543,7 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
575
543
|
export function openaiProvider(config: OpenAIConfig, codecs?: HostedCodecs): Provider {
|
|
576
544
|
const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
|
|
577
545
|
const id = config.id ?? "openai";
|
|
578
|
-
|
|
579
|
-
// `usage.cost` is its field in its unit. Another gateway that sends a
|
|
580
|
-
// `cost` has not said what it means.
|
|
581
|
-
const reportsCost = isOpenRouter(baseUrl);
|
|
546
|
+
const reportsCost = isOpenRouterBillingHost(baseUrl);
|
|
582
547
|
const instanceSessionId = config.sessionHeader
|
|
583
548
|
? (config.instanceSessionId ?? crypto.randomUUID())
|
|
584
549
|
: undefined;
|
|
@@ -720,7 +685,7 @@ export function openaiProvider(config: OpenAIConfig, codecs?: HostedCodecs): Pro
|
|
|
720
685
|
// before it goes out, and they are what says the turn called a tool.
|
|
721
686
|
const calls = new Map<number, ToolCallIdentity>();
|
|
722
687
|
const hostedStream = codecs?.openai.stream();
|
|
723
|
-
|
|
688
|
+
const stream = await openSse({
|
|
724
689
|
url: apiUrl(baseUrl, config.path ?? defaultPath),
|
|
725
690
|
headers: {
|
|
726
691
|
...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}),
|
|
@@ -746,7 +711,8 @@ export function openaiProvider(config: OpenAIConfig, codecs?: HostedCodecs): Pro
|
|
|
746
711
|
// to `[DONE]` (invariant 5). But a read that dies in between costs only
|
|
747
712
|
// the usage frame: the answer is whole once its finish is stated.
|
|
748
713
|
finished: () => ended,
|
|
749
|
-
})
|
|
714
|
+
});
|
|
715
|
+
for await (const data of stream.frames) {
|
|
750
716
|
const chunk = parseFrame<OpenAIChunk>(data, id);
|
|
751
717
|
|
|
752
718
|
// A failure the backend reports after its headers went out. The
|
|
@@ -755,7 +721,7 @@ export function openaiProvider(config: OpenAIConfig, codecs?: HostedCodecs): Pro
|
|
|
755
721
|
// line, and a frame carrying `error` carries no choices: unread, it
|
|
756
722
|
// falls through both branches below and the turn ends as a successful
|
|
757
723
|
// zero-token completion nobody retries.
|
|
758
|
-
if (chunk.error) throw streamError(id, chunk);
|
|
724
|
+
if (chunk.error) throw streamError(id, chunk, stream.headers);
|
|
759
725
|
|
|
760
726
|
// A usage-only frame carries no choices — this shape sends it last.
|
|
761
727
|
if (chunk.usage) {
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface OpenRouterBill {
|
|
2
|
+
cost?: unknown;
|
|
3
|
+
is_byok?: boolean;
|
|
4
|
+
cost_details?: { upstream_inference_cost?: unknown } | null;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
/** Other gateways may speak the same wire format without billing in US dollars. */
|
|
8
|
+
export function isOpenRouterBillingHost(baseUrl: string): boolean {
|
|
9
|
+
try {
|
|
10
|
+
return /(?:^|\.)openrouter\.ai\.?$/.test(new URL(baseUrl).hostname);
|
|
11
|
+
} catch {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** A negative or a string falls back to the caller's rate. Zero is a real bill. */
|
|
17
|
+
function price(value: unknown): number | undefined {
|
|
18
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* What an OpenRouter call cost, in USD, or undefined when it did not say.
|
|
23
|
+
*
|
|
24
|
+
* Nothing has to ask for it. OpenRouter's usage-accounting docs (read
|
|
25
|
+
* 2026-09-26) say every response now carries the full usage record, cost
|
|
26
|
+
* included, on the last frame of a stream, and that `usage: { include: true }`
|
|
27
|
+
* is deprecated and does nothing. Measured the same day: a streamed call and a
|
|
28
|
+
* plain one, neither sending the flag, both came back with `usage.cost`, the
|
|
29
|
+
* stream on its last frame. So the request does not send it.
|
|
30
|
+
*
|
|
31
|
+
* With the caller's own provider key (BYOK), `cost` is only OpenRouter's fee.
|
|
32
|
+
* The inference is billed to the caller's provider account and arrives as
|
|
33
|
+
* `cost_details.upstream_inference_cost`, so the call cost the two together.
|
|
34
|
+
* Without that second figure there is no whole bill to report, and the
|
|
35
|
+
* caller's rate prices the call instead.
|
|
36
|
+
*/
|
|
37
|
+
export function openRouterCostUsd(usage: OpenRouterBill): number | undefined {
|
|
38
|
+
const cost = price(usage.cost);
|
|
39
|
+
if (cost === undefined || usage.is_byok !== true) return cost;
|
|
40
|
+
const upstream = price(usage.cost_details?.upstream_inference_cost);
|
|
41
|
+
return upstream === undefined ? undefined : price(cost + upstream);
|
|
42
|
+
}
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
} from "../errors.ts";
|
|
26
26
|
import type { HostedToolType } from "../hosted/types.ts";
|
|
27
27
|
import { refuseHostedTools, type HostedCodecs } from "./hosted-codecs.ts";
|
|
28
|
-
import {
|
|
28
|
+
import { openSse, apiUrl, parseFrame, withRedactedErrors, type FetchLike } from "../transport.ts";
|
|
29
29
|
import { parseRateLimitReset } from "../rate-limit.ts";
|
|
30
30
|
import { settleToolCalls } from "../tool-args.ts";
|
|
31
31
|
import type {
|
|
@@ -44,6 +44,11 @@ import { toDataUri } from "../types.ts";
|
|
|
44
44
|
import { isStrictSchema, withoutPatterns } from "../schema.ts";
|
|
45
45
|
import { configuredFallbacks, type ProviderFallbackConfig } from "../fallback.ts";
|
|
46
46
|
import { gateImages, type ImageLimits } from "../image.ts";
|
|
47
|
+
import {
|
|
48
|
+
isOpenRouterBillingHost,
|
|
49
|
+
openRouterCostUsd,
|
|
50
|
+
type OpenRouterBill,
|
|
51
|
+
} from "./openrouter-billing.ts";
|
|
47
52
|
|
|
48
53
|
export interface ResponsesConfig extends ProviderFallbackConfig {
|
|
49
54
|
apiKey: string;
|
|
@@ -285,7 +290,7 @@ export function toResponsesInput(
|
|
|
285
290
|
|
|
286
291
|
// ── stream events ─────────────────────────────────────────────────────────
|
|
287
292
|
|
|
288
|
-
interface ResponsesUsage {
|
|
293
|
+
interface ResponsesUsage extends OpenRouterBill {
|
|
289
294
|
input_tokens?: number;
|
|
290
295
|
input_tokens_details?: { cached_tokens?: number };
|
|
291
296
|
output_tokens?: number;
|
|
@@ -418,8 +423,10 @@ function codexHeaders(limits: CodexRateLimits): Headers {
|
|
|
418
423
|
* them per call and states no count of its own. */
|
|
419
424
|
function usageChunk(
|
|
420
425
|
usage: ResponsesUsage,
|
|
426
|
+
reportsCost: boolean,
|
|
421
427
|
hostedCalls?: Partial<Record<HostedToolType, number>>,
|
|
422
428
|
): ProviderChunk {
|
|
429
|
+
const reportedCostUsd = reportsCost ? openRouterCostUsd(usage) : undefined;
|
|
423
430
|
return {
|
|
424
431
|
type: "usage",
|
|
425
432
|
usage: {
|
|
@@ -432,6 +439,7 @@ function usageChunk(
|
|
|
432
439
|
...(usage.output_tokens_details?.reasoning_tokens !== undefined
|
|
433
440
|
? { reasoningTokens: usage.output_tokens_details.reasoning_tokens }
|
|
434
441
|
: {}),
|
|
442
|
+
...(reportedCostUsd !== undefined ? { reportedCostUsd } : {}),
|
|
435
443
|
...(hostedCalls ? { hostedCalls } : {}),
|
|
436
444
|
},
|
|
437
445
|
};
|
|
@@ -463,6 +471,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
463
471
|
/** @internal Shared by the plain and hosted entry points. */
|
|
464
472
|
export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs): Provider {
|
|
465
473
|
const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
|
|
474
|
+
const reportsCost = isOpenRouterBillingHost(baseUrl);
|
|
466
475
|
const id = config.id ?? "openai-responses";
|
|
467
476
|
const instanceSessionId = config.sessionHeader
|
|
468
477
|
? (config.instanceSessionId ?? crypto.randomUUID())
|
|
@@ -637,7 +646,10 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
637
646
|
? [{ type: "delta", reasoningItems: { provider: id, items: reasoningItems } }]
|
|
638
647
|
: [];
|
|
639
648
|
const hostedStream = codecs?.responses.stream(hosted?.hosted ?? []);
|
|
640
|
-
|
|
649
|
+
function* reportUsage(usage?: ResponsesUsage): Generator<ProviderChunk> {
|
|
650
|
+
if (usage) yield usageChunk(usage, reportsCost, hostedStream?.counts());
|
|
651
|
+
}
|
|
652
|
+
const stream = await openSse({
|
|
641
653
|
url: apiUrl(baseUrl, config.path ?? DEFAULT_PATH),
|
|
642
654
|
headers: {
|
|
643
655
|
authorization: `Bearer ${config.apiKey}`,
|
|
@@ -652,7 +664,8 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
652
664
|
...(config.fetchImpl ? { fetchImpl: config.fetchImpl } : {}),
|
|
653
665
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
654
666
|
...(opts.onRateLimit ? { onRateLimit: opts.onRateLimit } : {}),
|
|
655
|
-
})
|
|
667
|
+
});
|
|
668
|
+
for await (const data of stream.frames) {
|
|
656
669
|
const event = parseFrame<ResponsesEvent>(data, id);
|
|
657
670
|
if (event.type?.endsWith(".delta") || event.type === "response.output_item.added") {
|
|
658
671
|
produced = true;
|
|
@@ -806,7 +819,7 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
806
819
|
// error failed, and one that states `incomplete` was cut. Read as
|
|
807
820
|
// a clean stop, either is a broken turn reported as a finished one.
|
|
808
821
|
if (response?.status === "failed" || response?.error) {
|
|
809
|
-
throw streamError(id,
|
|
822
|
+
throw streamError(id, event, stream.headers);
|
|
810
823
|
}
|
|
811
824
|
// Any other status (`cancelled`, `queued`, one added later) says
|
|
812
825
|
// nothing about the answer being whole. Read as a clean stop, it
|
|
@@ -823,12 +836,19 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
823
836
|
} else {
|
|
824
837
|
// Checked on a whole turn only: a cut one may have lost a call's
|
|
825
838
|
// name to the cut, and nobody runs a cut turn's calls.
|
|
826
|
-
|
|
839
|
+
let minted;
|
|
840
|
+
try {
|
|
841
|
+
minted = settleToolCalls(id, calls);
|
|
842
|
+
} catch (error) {
|
|
843
|
+
// Opening the call already emitted a delta. Its bill is due even if no tool can run.
|
|
844
|
+
yield* reportUsage(response?.usage);
|
|
845
|
+
throw error;
|
|
846
|
+
}
|
|
827
847
|
if (minted.length > 0) yield { type: "delta", toolCalls: minted };
|
|
828
848
|
}
|
|
829
849
|
if (hostedStream) yield* hostedStream.close();
|
|
830
850
|
yield* reasoningChunk();
|
|
831
|
-
|
|
851
|
+
yield* reportUsage(response?.usage);
|
|
832
852
|
yield { type: "finish", finishReason };
|
|
833
853
|
return;
|
|
834
854
|
}
|
|
@@ -843,8 +863,7 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
843
863
|
const finishReason = incompleteFinish(id, event.response?.incomplete_details?.reason);
|
|
844
864
|
if (hostedStream) yield* hostedStream.close();
|
|
845
865
|
yield* reasoningChunk();
|
|
846
|
-
|
|
847
|
-
if (usage) yield usageChunk(usage, hostedStream?.counts());
|
|
866
|
+
yield* reportUsage(event.response?.usage);
|
|
848
867
|
yield { type: "finish", finishReason };
|
|
849
868
|
return;
|
|
850
869
|
}
|
|
@@ -853,18 +872,13 @@ export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs
|
|
|
853
872
|
// out: a terminal `response.failed`, or a bare error frame from a
|
|
854
873
|
// gateway in front of it.
|
|
855
874
|
case "response.failed":
|
|
856
|
-
throw streamError(
|
|
857
|
-
id,
|
|
858
|
-
event.response?.error ? event.response : { message: "response failed" },
|
|
859
|
-
);
|
|
860
|
-
|
|
861
875
|
case "error":
|
|
862
876
|
case "response.error":
|
|
863
877
|
// The status and the window clocks ride beside `error`. Without
|
|
864
878
|
// them a spent Codex window reads as an unnamed in-stream failure:
|
|
865
879
|
// floored to overload, then retried and walked across every backup
|
|
866
880
|
// on the same account wall.
|
|
867
|
-
throw streamError(id, event);
|
|
881
|
+
throw streamError(id, event, stream.headers);
|
|
868
882
|
}
|
|
869
883
|
}
|
|
870
884
|
|
package/src/rate-limit.ts
CHANGED
|
@@ -253,13 +253,6 @@ function windowForMs(ms: number): RateLimitWindow {
|
|
|
253
253
|
return "monthly";
|
|
254
254
|
}
|
|
255
255
|
|
|
256
|
-
/** A wait read off a body, named as a window once it is long enough to be one. */
|
|
257
|
-
function resetAfter(waitMs: number, now: number): RateLimitReset {
|
|
258
|
-
const result: RateLimitReset = { retryAfterMs: waitMs, resetAtMs: now + waitMs };
|
|
259
|
-
if (waitMs > WINDOW_FLOOR_MS) result.window = windowForMs(waitMs);
|
|
260
|
-
return result;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
256
|
/** A finite number off a JSON field, the string form a relay may stringify it
|
|
264
257
|
* into included. Bare `Number()` is the trap `numeric()` guards against one
|
|
265
258
|
* layer up: `Number(null)`, `Number("")`, `Number([])` and `Number(false)` are
|
|
@@ -279,7 +272,16 @@ const PROSE_RETRY_PATTERNS = [
|
|
|
279
272
|
/retry[- ]after[:\s]+(\d+(?:\.\d+)?)\s*(s(?:ec(?:ond)?s?)?|m(?:in(?:ute)?s?)?|h(?:(?:ou)?rs?)?)?/i,
|
|
280
273
|
];
|
|
281
274
|
|
|
275
|
+
function secondsMs(value: unknown): number | undefined {
|
|
276
|
+
const seconds = jsonSeconds(value);
|
|
277
|
+
if (seconds === undefined || seconds < 0) return undefined;
|
|
278
|
+
const ms = Math.round(seconds * 1000);
|
|
279
|
+
return Number.isFinite(ms) ? ms : undefined;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
282
|
function parseProseRetryMs(text: string): number | undefined {
|
|
283
|
+
const google = text.match(/\bplease retry in (\d+(?:\.\d+)?)\s*s\b/i);
|
|
284
|
+
if (google) return secondsMs(google[1]);
|
|
283
285
|
for (const pattern of PROSE_RETRY_PATTERNS) {
|
|
284
286
|
const match = text.match(pattern);
|
|
285
287
|
if (!match?.[1]) continue;
|
|
@@ -287,76 +289,128 @@ function parseProseRetryMs(text: string): number | undefined {
|
|
|
287
289
|
if (!Number.isFinite(val) || val <= 0) continue;
|
|
288
290
|
const unit = (match[2] ?? "s").toLowerCase();
|
|
289
291
|
const multiplier = unit.startsWith("h") ? 3_600_000 : unit.startsWith("m") ? 60_000 : 1_000;
|
|
290
|
-
|
|
292
|
+
const ms = Math.ceil(val * multiplier);
|
|
293
|
+
if (Number.isFinite(ms)) return ms;
|
|
291
294
|
}
|
|
292
295
|
return undefined;
|
|
293
296
|
}
|
|
294
297
|
|
|
295
|
-
/**
|
|
296
|
-
*
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
298
|
+
/** SDKs may append a serialized reply to a status line. Only the text before it is
|
|
299
|
+
* prose: a partial JSON body cannot tell us which words belong to the provider. */
|
|
300
|
+
function plainMessageDelay(text: string): number | undefined {
|
|
301
|
+
const jsonAt = text.search(/\{|\[/u);
|
|
302
|
+
return parseProseRetryMs(jsonAt < 0 ? text : text.slice(0, jsonAt));
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function fields(value: unknown): Record<string, unknown> | undefined {
|
|
306
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
307
|
+
? (value as Record<string, unknown>)
|
|
308
|
+
: undefined;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
interface BodyHints {
|
|
312
|
+
accountAt?: number;
|
|
313
|
+
delay?: number;
|
|
314
|
+
prose?: number;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** A real account reset survives a shorter Retry-After. A guess from prose does not. Keep
|
|
318
|
+
* those sources separate until the headers are known. Never scan a serialized request: a
|
|
319
|
+
* gateway may echo the entire prompt alongside its error. */
|
|
320
|
+
function bodyHints(text: string, now: number): BodyHints {
|
|
321
|
+
let decoded: unknown;
|
|
305
322
|
try {
|
|
306
|
-
|
|
323
|
+
decoded = JSON.parse(text);
|
|
307
324
|
} catch {
|
|
308
|
-
|
|
325
|
+
return { prose: plainMessageDelay(text) };
|
|
309
326
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
const
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
327
|
+
if (typeof decoded === "string") return { prose: parseProseRetryMs(decoded) };
|
|
328
|
+
const hints: BodyHints = {};
|
|
329
|
+
for (const row of Array.isArray(decoded) ? decoded : [decoded]) {
|
|
330
|
+
const outer = fields(row);
|
|
331
|
+
if (!outer) continue;
|
|
332
|
+
const error = fields(outer.error) ?? outer;
|
|
333
|
+
const message = typeof outer.error === "string" ? outer.error : error.message;
|
|
334
|
+
if (typeof message === "string") hints.prose ??= parseProseRetryMs(message);
|
|
335
|
+
// Codex's relative reset is authoritative when it also gives an epoch. Gateways
|
|
336
|
+
// can add a later account reset in metadata, including on a Google ErrorInfo detail.
|
|
337
|
+
const relative = secondsMs(error.resets_in_seconds);
|
|
338
|
+
const epoch = secondsMs(error.resets_at);
|
|
339
|
+
const at = relative !== undefined ? now + relative : epoch;
|
|
340
|
+
if (at !== undefined && at > 0) hints.accountAt = Math.max(hints.accountAt ?? now, at);
|
|
341
|
+
hints.delay ??=
|
|
342
|
+
secondsMs(error.retry_after) ??
|
|
343
|
+
secondsMs(error.retry_after_seconds) ??
|
|
344
|
+
secondsMs(outer.retry_after_seconds);
|
|
345
|
+
const details: unknown[] = Array.isArray(error.details) ? error.details : [];
|
|
346
|
+
for (const value of [outer, error, ...details]) {
|
|
347
|
+
const item = fields(value);
|
|
348
|
+
const deadline = fields(item?.metadata)?.resetsAt;
|
|
349
|
+
if (typeof deadline === "string") {
|
|
350
|
+
const ms = Date.parse(deadline);
|
|
351
|
+
if (Number.isFinite(ms)) hints.accountAt = Math.max(hints.accountAt ?? now, ms);
|
|
352
|
+
}
|
|
353
|
+
// RetryInfo spells even whole seconds as a string. A mention of "retryDelay=1s"
|
|
354
|
+
// inside the model's error message is not this field.
|
|
355
|
+
if (typeof item?.retryDelay === "string") {
|
|
356
|
+
hints.delay ??= secondsMs(/^(\d+(?:\.\d+)?)s$/.exec(item.retryDelay)?.[1]);
|
|
357
|
+
}
|
|
338
358
|
}
|
|
359
|
+
}
|
|
360
|
+
return hints;
|
|
361
|
+
}
|
|
339
362
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
363
|
+
interface HintOptions {
|
|
364
|
+
headers?: Headers;
|
|
365
|
+
accountWindow?: boolean;
|
|
366
|
+
normalizedDelay?: number;
|
|
367
|
+
fallbackMessage?: string;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function mergedReset(text: string, options: HintOptions, now: number): RateLimitReset {
|
|
371
|
+
const header = options.headers ? parseRateLimitReset(options.headers, now) : {};
|
|
372
|
+
const body = bodyHints(text, now);
|
|
373
|
+
const aboutWindow = options.accountWindow !== false;
|
|
374
|
+
const explicit = options.normalizedDelay ?? header.retryAfterMs;
|
|
375
|
+
// A Retry-After header alone supplies no independent account horizon. In particular,
|
|
376
|
+
// it cannot restore a longer delay that the caller has already normalized.
|
|
377
|
+
let headerAt = aboutWindow ? header.resetAtMs : undefined;
|
|
378
|
+
if (
|
|
379
|
+
header.retryAfterMs !== undefined &&
|
|
380
|
+
headerAt === now + header.retryAfterMs &&
|
|
381
|
+
!header.window
|
|
382
|
+
) {
|
|
383
|
+
headerAt = undefined;
|
|
384
|
+
}
|
|
385
|
+
const accountAt = aboutWindow ? body.accountAt : undefined;
|
|
386
|
+
let bodyWait = body.delay;
|
|
387
|
+
if (accountAt !== undefined) bodyWait = Math.max(accountAt - now, bodyWait ?? 0);
|
|
388
|
+
if (explicit === undefined && headerAt === undefined && bodyWait === undefined) {
|
|
389
|
+
bodyWait = body.prose ?? parseProseRetryMs(options.fallbackMessage ?? "");
|
|
355
390
|
}
|
|
391
|
+
const delay = explicit ?? bodyWait;
|
|
392
|
+
if (!aboutWindow) return delay === undefined ? {} : { retryAfterMs: delay };
|
|
393
|
+
const end = Math.max(headerAt ?? 0, accountAt ?? 0, delay === undefined ? 0 : now + delay);
|
|
394
|
+
const bodySetsEnd =
|
|
395
|
+
end > 0 &&
|
|
396
|
+
(end === accountAt ||
|
|
397
|
+
(explicit === undefined && bodyWait !== undefined && end === now + bodyWait));
|
|
398
|
+
let window: RateLimitWindow | undefined;
|
|
399
|
+
if (header.window && headerAt === end) {
|
|
400
|
+
window = header.window;
|
|
401
|
+
} else if (bodySetsEnd && end - now > WINDOW_FLOOR_MS) {
|
|
402
|
+
window = windowForMs(end - now);
|
|
403
|
+
}
|
|
404
|
+
return {
|
|
405
|
+
...(end > 0 ? { resetAtMs: end } : {}),
|
|
406
|
+
...(delay !== undefined ? { retryAfterMs: delay } : {}),
|
|
407
|
+
...(window ? { window } : {}),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
356
410
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
return
|
|
411
|
+
/** Read a usage deadline, RetryInfo field or wait stated in the provider's own message. */
|
|
412
|
+
export function parseUsageLimitBody(bodyText: string, now = Date.now()): RateLimitReset {
|
|
413
|
+
return mergedReset(bodyText, {}, now);
|
|
360
414
|
}
|
|
361
415
|
|
|
362
416
|
/** Read both places an endpoint can report its reset. A short Retry-After is
|
|
@@ -366,17 +420,16 @@ export function parseRateLimitResponse(
|
|
|
366
420
|
bodyText: string,
|
|
367
421
|
now = Date.now(),
|
|
368
422
|
): RateLimitReset {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
};
|
|
423
|
+
return mergedReset(bodyText, { headers }, now);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** @internal A failed rate-limited call can use a known account deadline as its wait even
|
|
427
|
+
* without Retry-After. Standing account headers say nothing about unrelated failures. */
|
|
428
|
+
export function failureReset(bodyText: string, options: HintOptions): RateLimitReset {
|
|
429
|
+
const now = Date.now();
|
|
430
|
+
const reset = mergedReset(bodyText, options, now);
|
|
431
|
+
if (reset.retryAfterMs === undefined && reset.resetAtMs !== undefined) {
|
|
432
|
+
reset.retryAfterMs = Math.max(0, reset.resetAtMs - now);
|
|
433
|
+
}
|
|
434
|
+
return reset;
|
|
382
435
|
}
|
package/src/transport.ts
CHANGED
|
@@ -8,16 +8,11 @@ import {
|
|
|
8
8
|
classify,
|
|
9
9
|
classifyHttp,
|
|
10
10
|
parseContextOverflow,
|
|
11
|
-
parseRetryAfterMs,
|
|
12
11
|
ProviderError,
|
|
13
12
|
type ContextOverflowInfo,
|
|
14
13
|
type ErrorKind,
|
|
15
14
|
} from "./errors.ts";
|
|
16
|
-
import {
|
|
17
|
-
parseRateLimitReset,
|
|
18
|
-
parseRateLimitResponse,
|
|
19
|
-
type RateLimitReading,
|
|
20
|
-
} from "./rate-limit.ts";
|
|
15
|
+
import { failureReset, parseRateLimitReset, type RateLimitReading } from "./rate-limit.ts";
|
|
21
16
|
import type { Provider } from "./types.ts";
|
|
22
17
|
|
|
23
18
|
/**
|
|
@@ -159,11 +154,10 @@ async function errorFor(provider: string, res: Response, redact = false): Promis
|
|
|
159
154
|
// three days": past every retry budget, and the backup walker benched the
|
|
160
155
|
// model until the weekly window rolled — the overload that most wants another
|
|
161
156
|
// attempt got none. A server's own Retry-After is honoured on any status.
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
: parseRateLimitReset(res.headers).retryAfterMs;
|
|
157
|
+
const reset = failureReset(text, {
|
|
158
|
+
headers: res.headers,
|
|
159
|
+
accountWindow: kind === "rate" || kind === "quota",
|
|
160
|
+
});
|
|
167
161
|
const message = redact
|
|
168
162
|
? `${provider} ${res.status}`
|
|
169
163
|
: text
|
|
@@ -176,7 +170,6 @@ async function errorFor(provider: string, res: Response, redact = false): Promis
|
|
|
176
170
|
status: res.status,
|
|
177
171
|
...reset,
|
|
178
172
|
shouldRetry,
|
|
179
|
-
retryAfterMs: asked ?? parseRetryAfterMs({}, text),
|
|
180
173
|
...(redact ? {} : { body: text.slice(0, 2_000) || undefined }),
|
|
181
174
|
});
|
|
182
175
|
}
|
|
@@ -666,6 +659,14 @@ export function parseFrame<T>(data: string, provider: string): T {
|
|
|
666
659
|
* classified error, the retry hints) plus `parseSseStream`.
|
|
667
660
|
*/
|
|
668
661
|
export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
|
|
662
|
+
yield* (await openSse(opts)).frames;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/** @internal A stream can fail after HTTP 200. Its adapter needs the response
|
|
666
|
+
* headers alongside that error frame to recover the server's retry delay. */
|
|
667
|
+
export async function openSse(
|
|
668
|
+
opts: RequestInit_,
|
|
669
|
+
): Promise<{ headers: Headers; frames: AsyncGenerator<string> }> {
|
|
669
670
|
const res = await send(opts);
|
|
670
671
|
// Headers are the first sign of life — the idle clock starts here, not at
|
|
671
672
|
// the POST: a backend that withholds them until its first output token (the
|
|
@@ -689,5 +690,5 @@ export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
|
|
|
689
690
|
status: res.status,
|
|
690
691
|
});
|
|
691
692
|
}
|
|
692
|
-
|
|
693
|
+
return { headers: res.headers, frames: parseSseStream(res.body, opts) };
|
|
693
694
|
}
|