@starci/ai-router 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +12 -3
package/dist/headers.js
CHANGED
|
@@ -31,12 +31,18 @@ function resetAt(headers, names, now) {
|
|
|
31
31
|
if (raw === null)
|
|
32
32
|
continue;
|
|
33
33
|
const plain = Number(raw.trim());
|
|
34
|
-
// A plain number past
|
|
34
|
+
// A plain number past 1e12 is an epoch in milliseconds (OpenRouter), past 1e9 one in seconds; smaller is a delta.
|
|
35
|
+
if (Number.isFinite(plain) && plain > 1e12)
|
|
36
|
+
return plain;
|
|
35
37
|
if (Number.isFinite(plain) && plain > 1e9)
|
|
36
38
|
return plain * 1000;
|
|
37
39
|
const ms = parseDuration(raw);
|
|
38
40
|
if (ms !== null)
|
|
39
41
|
return now + ms;
|
|
42
|
+
// Anthropic's anthropic-ratelimit-*-reset is an RFC 3339 time.
|
|
43
|
+
const date = /^\d{4}-\d{2}-\d{2}T/.test(raw.trim()) ? Date.parse(raw.trim()) : Number.NaN;
|
|
44
|
+
if (!Number.isNaN(date))
|
|
45
|
+
return date;
|
|
40
46
|
}
|
|
41
47
|
return 0;
|
|
42
48
|
}
|
|
@@ -54,13 +60,13 @@ export function retryAt(headers, now) {
|
|
|
54
60
|
const date = Date.parse(after);
|
|
55
61
|
return Number.isNaN(date) ? 0 : date;
|
|
56
62
|
}
|
|
57
|
-
/** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
63
|
+
/** Reads the `x-ratelimit-*` (OpenAI, Groq, OpenRouter), `anthropic-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
58
64
|
export function readRateLimit(headers, now) {
|
|
59
65
|
return {
|
|
60
|
-
remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "x-ratelimit-remaining", "ratelimit-remaining"]),
|
|
61
|
-
requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "x-ratelimit-reset", "ratelimit-reset"], now),
|
|
62
|
-
remainingTokens: count(headers, ["x-ratelimit-remaining-tokens"]),
|
|
63
|
-
tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens"], now),
|
|
66
|
+
remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "anthropic-ratelimit-requests-remaining", "x-ratelimit-remaining", "ratelimit-remaining"]),
|
|
67
|
+
requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "anthropic-ratelimit-requests-reset", "x-ratelimit-reset", "ratelimit-reset"], now),
|
|
68
|
+
remainingTokens: count(headers, ["x-ratelimit-remaining-tokens", "anthropic-ratelimit-tokens-remaining"]),
|
|
69
|
+
tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens", "anthropic-ratelimit-tokens-reset"], now),
|
|
64
70
|
retryAt: retryAt(headers, now),
|
|
65
71
|
};
|
|
66
72
|
}
|
package/dist/pool.d.ts
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
import type { AttemptError } from "./attempt.js";
|
|
1
2
|
import type { ResolvedPolicy, Settings } from "./config.js";
|
|
2
3
|
import type { Emitter } from "./events.js";
|
|
3
4
|
import type { RateLimitInfo } from "./headers.js";
|
|
4
5
|
import type { StateStore } from "./store.js";
|
|
5
|
-
import type { Capability,
|
|
6
|
-
/** One routable (provider, key, model) triple
|
|
6
|
+
import type { Capability, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } from "./types.js";
|
|
7
|
+
/** One routable (provider, key, model) triple, the rank of its tier in the call's policy and its weight there. */
|
|
7
8
|
export interface Lane {
|
|
8
9
|
readonly provider: ProviderConfig;
|
|
9
10
|
readonly key: KeyConfig;
|
|
10
11
|
readonly model: ModelConfig;
|
|
11
12
|
readonly rank: number;
|
|
13
|
+
/** The candidate's weight in its tier (1 for a model named by `model` or `fallbacks`). */
|
|
14
|
+
readonly weight: number;
|
|
12
15
|
/** `provider/key/model`, the lane's name in local counters. */
|
|
13
16
|
readonly id: string;
|
|
14
17
|
/** `provider/key`, the pair a retry moves away from. */
|
|
@@ -21,6 +24,10 @@ export interface Need {
|
|
|
21
24
|
readonly promptTokens: number;
|
|
22
25
|
/** `max_tokens` or `max_completion_tokens`, 0 when unbounded. */
|
|
23
26
|
readonly outputTokens: number;
|
|
27
|
+
/** True keeps every lane on the first lane's model identity (an embeddings call, a pinned profile). */
|
|
28
|
+
readonly oneIdentity: boolean;
|
|
29
|
+
/** The vector length an embeddings call asked for; a model declared at another is no candidate. */
|
|
30
|
+
readonly dimensions: number | null;
|
|
24
31
|
}
|
|
25
32
|
/** What a call has tried so far: lanes, key pairs and providers. */
|
|
26
33
|
export interface Tried {
|
|
@@ -33,17 +40,27 @@ export interface Pick {
|
|
|
33
40
|
readonly lane: Lane;
|
|
34
41
|
readonly leases: readonly string[];
|
|
35
42
|
}
|
|
43
|
+
/** Why the lanes that are out are out, when every one of them is out for its key or its provider's budget. */
|
|
44
|
+
export interface Blacklisted {
|
|
45
|
+
/** `quota` when any lane is out for a spent quota or budget, else `auth` (a refused or missing secret). */
|
|
46
|
+
readonly reason: "quota" | "auth";
|
|
47
|
+
/** The aliases, `provider/key`, that are blacklisted, and the providers past their budget. */
|
|
48
|
+
readonly keys: readonly string[];
|
|
49
|
+
readonly budgets: readonly string[];
|
|
50
|
+
}
|
|
36
51
|
/** No lane is usable now: `waitUntil` is when the earliest one may be (Infinity: never within reach). */
|
|
37
52
|
export interface Blocked {
|
|
38
53
|
readonly waitUntil: number;
|
|
39
54
|
/** True when a lane waits only on in-process concurrency or a probe another attempt holds. */
|
|
40
55
|
readonly busy: boolean;
|
|
56
|
+
/** Set when every lane is out because its key is blacklisted or its provider's budget is spent. */
|
|
57
|
+
readonly blacklisted: Blacklisted | null;
|
|
41
58
|
}
|
|
42
59
|
/** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
|
|
43
60
|
export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
|
|
44
61
|
/**
|
|
45
|
-
* The pool's state: shared breaker,
|
|
46
|
-
* round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
|
|
62
|
+
* The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
|
|
63
|
+
* EWMAs and round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
|
|
47
64
|
*/
|
|
48
65
|
export declare class PoolState {
|
|
49
66
|
#private;
|
|
@@ -55,16 +72,18 @@ export declare class PoolState {
|
|
|
55
72
|
};
|
|
56
73
|
/** Binds the state to a pool's providers, settings, store and emitter. */
|
|
57
74
|
constructor(providers: readonly ProviderConfig[], settings: Settings, store: StateStore, emitter: Emitter);
|
|
58
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Every lane that serves one of the policy's candidates with the capabilities, context window and budget the call
|
|
77
|
+
* needs, tier by tier. With `need.oneIdentity`, a lane whose model identity differs from the first lane's is left out.
|
|
78
|
+
*/
|
|
59
79
|
lanes(policy: ResolvedPolicy, need: Need): Lane[];
|
|
60
80
|
/**
|
|
61
81
|
* Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
|
|
62
|
-
* call, the policy's
|
|
63
|
-
* to another key or provider of the same model first, then to the next
|
|
64
|
-
* Within the first non-empty tier:
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
* overshoot a cap.
|
|
82
|
+
* call, the policy's candidate order, a key not yet tried, a provider not yet tried, enough headroom. So a retry
|
|
83
|
+
* moves to another key or provider of the same model first, then to the next tier, and only then back to a tried
|
|
84
|
+
* lane. Within the first non-empty tier: the fewest in-flight requests, then latency within the tolerance of the
|
|
85
|
+
* best EWMA, then smooth weighted round-robin. The concurrency check and the hold run without an await between
|
|
86
|
+
* them, so concurrent calls never overshoot a cap.
|
|
68
87
|
*/
|
|
69
88
|
pick(lanes: readonly Lane[], need: Need, tried: Tried, leaseMs: number): Promise<Pick | Blocked>;
|
|
70
89
|
/** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
|
|
@@ -73,14 +92,16 @@ export declare class PoolState {
|
|
|
73
92
|
release(lane: Lane, leases: readonly string[]): Promise<void>;
|
|
74
93
|
/** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
|
|
75
94
|
waitForCapacity(ms: number, signal: AbortSignal): Promise<void>;
|
|
76
|
-
/** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
|
|
95
|
+
/** Records a success: closes the lane's breakers, stores its headroom and spend, and feeds its latency EWMA. */
|
|
77
96
|
succeeded(lane: Lane, info: RateLimitInfo | null, latencyMs: number, usage: Usage | undefined): Promise<number | undefined>;
|
|
78
97
|
/**
|
|
79
|
-
* Records a failure by class: a 429 cools the lane down (retry-after or the default)
|
|
80
|
-
* key
|
|
81
|
-
*
|
|
98
|
+
* Records a failure by class: a 429 cools the lane down (retry-after or the default); a spent quota blacklists the
|
|
99
|
+
* key until its window ends (the provider's, a retry-after, else `quotaBlacklistMs`) or its secret changes; a
|
|
100
|
+
* refused or missing secret blacklists it until the secret changes (or `authDisableMs`); a transient failure counts
|
|
101
|
+
* toward the key's and the provider's breakers (honouring retry-after). Any other class touches no shared state.
|
|
102
|
+
* `secret` is the value the attempt sent, kept only as its fingerprint.
|
|
82
103
|
*/
|
|
83
|
-
failed(lane: Lane,
|
|
104
|
+
failed(lane: Lane, error: AttemptError, secret: string | undefined): Promise<void>;
|
|
84
105
|
/** Reads every record into a health snapshot. */
|
|
85
106
|
health(): Promise<HealthSnapshot>;
|
|
86
107
|
/** Returns this process's counters. */
|
package/dist/pool.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { windowEnd } from "./classify.js";
|
|
2
|
+
import { DEFAULT_CAPABILITIES, identityOf, modelMatches } from "./config.js";
|
|
3
|
+
import { presetOf } from "./providers.js";
|
|
4
|
+
import { fingerprintOf, resolveSecret } from "./secrets.js";
|
|
5
|
+
import { EMPTY_RECORD, toHealthRecord } from "./store.js";
|
|
3
6
|
const FOREVER = Number.MAX_SAFE_INTEGER;
|
|
4
7
|
/** The latency samples a lane needs before its EWMA ranks it: one cold call (a JIT warm-up, a first TLS handshake) never decides. */
|
|
5
8
|
const LATENCY_SAMPLES = 3;
|
|
@@ -17,9 +20,28 @@ export function costOf(model, usage) {
|
|
|
17
20
|
const completion = usage.completion_tokens ?? 0;
|
|
18
21
|
return (usage.prompt_tokens * model.cost.input + completion * model.cost.output) / 1_000_000;
|
|
19
22
|
}
|
|
23
|
+
/** The UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`) of a time. */
|
|
24
|
+
function periods(now) {
|
|
25
|
+
const day = new Date(now).toISOString().slice(0, 10);
|
|
26
|
+
return { day, month: day.slice(0, 7) };
|
|
27
|
+
}
|
|
28
|
+
/** When a provider past its budget is a candidate again: the next UTC day or month, or 0 when it is within it. */
|
|
29
|
+
function budgetUntil(provider, record, now) {
|
|
30
|
+
const budget = provider.budget;
|
|
31
|
+
if (budget === undefined)
|
|
32
|
+
return 0;
|
|
33
|
+
const { day, month } = periods(now);
|
|
34
|
+
const date = new Date(now);
|
|
35
|
+
let until = 0;
|
|
36
|
+
if (budget.costPerMonth !== undefined && record.spendMonth === month && record.spendMonthCost >= budget.costPerMonth)
|
|
37
|
+
until = Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1);
|
|
38
|
+
else if (budget.costPerDay !== undefined && record.spendDay === day && record.spendDayCost >= budget.costPerDay)
|
|
39
|
+
until = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate() + 1);
|
|
40
|
+
return until;
|
|
41
|
+
}
|
|
20
42
|
/**
|
|
21
|
-
* The pool's state: shared breaker,
|
|
22
|
-
* round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
|
|
43
|
+
* The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
|
|
44
|
+
* EWMAs and round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
|
|
23
45
|
*/
|
|
24
46
|
export class PoolState {
|
|
25
47
|
#providers;
|
|
@@ -51,33 +73,50 @@ export class PoolState {
|
|
|
51
73
|
return `${this.#settings.namespace}:lane:${provider}/${alias}/${model}`;
|
|
52
74
|
}
|
|
53
75
|
async #read(id) {
|
|
54
|
-
|
|
76
|
+
// A record an older version or another adapter wrote is read field by field, its missing fields empty.
|
|
77
|
+
const stored = await this.#store.get(id);
|
|
78
|
+
return stored === undefined ? EMPTY_RECORD : (toHealthRecord(stored) ?? EMPTY_RECORD);
|
|
55
79
|
}
|
|
56
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Every lane that serves one of the policy's candidates with the capabilities, context window and budget the call
|
|
82
|
+
* needs, tier by tier. With `need.oneIdentity`, a lane whose model identity differs from the first lane's is left out.
|
|
83
|
+
*/
|
|
57
84
|
lanes(policy, need) {
|
|
58
85
|
const lanes = [];
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (!modelMatches(model, name))
|
|
86
|
+
let identity;
|
|
87
|
+
policy.tiers.forEach((tier, rank) => {
|
|
88
|
+
for (const ref of tier) {
|
|
89
|
+
for (const provider of this.#providers) {
|
|
90
|
+
if (policy.providers !== null && !policy.providers.has(provider.id))
|
|
65
91
|
continue;
|
|
66
|
-
|
|
67
|
-
if (!need.capabilities.every((capability) => capabilities.includes(capability)))
|
|
92
|
+
if (ref.provider !== null && ref.provider !== provider.id)
|
|
68
93
|
continue;
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
if (policy.maxCost !== null && model.cost !== undefined) {
|
|
72
|
-
const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
|
|
73
|
-
if (worst > policy.maxCost)
|
|
94
|
+
for (const model of provider.models) {
|
|
95
|
+
if (!modelMatches(model, ref.model))
|
|
74
96
|
continue;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
if (
|
|
97
|
+
const capabilities = model.capabilities ?? DEFAULT_CAPABILITIES;
|
|
98
|
+
if (!need.capabilities.every((capability) => capabilities.includes(capability)))
|
|
99
|
+
continue;
|
|
100
|
+
if (model.contextWindow !== undefined && need.promptTokens + need.outputTokens > model.contextWindow)
|
|
101
|
+
continue;
|
|
102
|
+
if (policy.maxCost !== null && model.cost !== undefined) {
|
|
103
|
+
const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
|
|
104
|
+
if (worst > policy.maxCost)
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (need.dimensions !== null && model.dimensions !== undefined && model.dimensions !== need.dimensions)
|
|
79
108
|
continue;
|
|
80
|
-
|
|
109
|
+
if (need.oneIdentity) {
|
|
110
|
+
identity ??= identityOf(model);
|
|
111
|
+
if (identityOf(model) !== identity)
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
for (const key of provider.keys) {
|
|
115
|
+
const id = `${provider.id}/${key.alias}/${model.id}`;
|
|
116
|
+
if (lanes.some((lane) => lane.id === id))
|
|
117
|
+
continue;
|
|
118
|
+
lanes.push({ provider, key, model, rank, weight: ref.weight, id, pair: `${provider.id}/${key.alias}` });
|
|
119
|
+
}
|
|
81
120
|
}
|
|
82
121
|
}
|
|
83
122
|
}
|
|
@@ -91,10 +130,23 @@ export class PoolState {
|
|
|
91
130
|
this.#read(this.#laneId(lane.provider.id, lane.key.alias, lane.model.id)),
|
|
92
131
|
]);
|
|
93
132
|
const blocks = [];
|
|
94
|
-
|
|
133
|
+
let block = null;
|
|
134
|
+
if (key.disabledUntil > now) {
|
|
95
135
|
blocks.push(key.disabledUntil >= FOREVER ? Infinity : key.disabledUntil);
|
|
136
|
+
block = key.disabledReason === "quota" ? "blacklist-quota" : "blacklist-auth";
|
|
137
|
+
}
|
|
138
|
+
const budget = budgetUntil(lane.provider, provider, now);
|
|
139
|
+
if (budget > now) {
|
|
140
|
+
blocks.push(budget);
|
|
141
|
+
block ??= "budget";
|
|
142
|
+
}
|
|
143
|
+
const soon = blocks.length;
|
|
96
144
|
const probes = [];
|
|
97
|
-
|
|
145
|
+
const breakers = [
|
|
146
|
+
{ record: key, id: this.#keyId(lane.provider.id, lane.key.alias) },
|
|
147
|
+
{ record: provider, id: this.#providerId(lane.provider.id) },
|
|
148
|
+
];
|
|
149
|
+
for (const { record, id } of breakers) {
|
|
98
150
|
const state = breakerState(record, now);
|
|
99
151
|
if (state === "open")
|
|
100
152
|
blocks.push(record.openUntil);
|
|
@@ -108,10 +160,32 @@ export class PoolState {
|
|
|
108
160
|
if (headroom.remainingTokens !== null && headroom.remainingTokens < need.promptTokens + need.outputTokens && headroom.tokensResetAt > now) {
|
|
109
161
|
blocks.push(headroom.tokensResetAt);
|
|
110
162
|
}
|
|
163
|
+
if (block === null && blocks.length > soon)
|
|
164
|
+
block = "soon";
|
|
111
165
|
if (blocks.length > 0)
|
|
112
|
-
return { usable: false, until: Math.max(...blocks), low: false, probes };
|
|
166
|
+
return { usable: false, until: Math.max(...blocks), low: false, probes, block };
|
|
113
167
|
const low = headroom.remainingRequests !== null && headroom.remainingRequests <= this.#settings.lowHeadroom && headroom.requestsResetAt > now;
|
|
114
|
-
return { usable: true, until: now, low, probes };
|
|
168
|
+
return { usable: true, until: now, low, probes, block: null };
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Lifts the blacklist of every key among the lanes whose secret changed since it was blacklisted (a missing secret
|
|
172
|
+
* that is now set counts): a new secret is a new key, and deserves a try at once.
|
|
173
|
+
*/
|
|
174
|
+
async #restore(lanes, now) {
|
|
175
|
+
const seen = new Set();
|
|
176
|
+
for (const lane of lanes) {
|
|
177
|
+
if (seen.has(lane.pair))
|
|
178
|
+
continue;
|
|
179
|
+
seen.add(lane.pair);
|
|
180
|
+
const id = this.#keyId(lane.provider.id, lane.key.alias);
|
|
181
|
+
const record = await this.#read(id);
|
|
182
|
+
if (record.disabledUntil <= now || record.disabledReason === null || record.fingerprint === null)
|
|
183
|
+
continue;
|
|
184
|
+
if (fingerprintOf(await resolveSecret(lane.key)) === record.fingerprint)
|
|
185
|
+
continue;
|
|
186
|
+
await this.#store.set(id, { ...record, disabledUntil: 0, disabledReason: null, disabledCode: null, fingerprint: null });
|
|
187
|
+
this.#emitter.emit("key-restored", { type: "key-restored", provider: lane.provider.id, key: lane.key.alias, was: record.disabledReason, reason: "secret_changed" });
|
|
188
|
+
}
|
|
115
189
|
}
|
|
116
190
|
#busy(lane) {
|
|
117
191
|
return ((this.#inflightKey.get(lane.pair) ?? 0) >= (lane.key.maxConcurrency ?? Infinity) ||
|
|
@@ -123,24 +197,33 @@ export class PoolState {
|
|
|
123
197
|
}
|
|
124
198
|
/**
|
|
125
199
|
* Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
|
|
126
|
-
* call, the policy's
|
|
127
|
-
* to another key or provider of the same model first, then to the next
|
|
128
|
-
* Within the first non-empty tier:
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* overshoot a cap.
|
|
200
|
+
* call, the policy's candidate order, a key not yet tried, a provider not yet tried, enough headroom. So a retry
|
|
201
|
+
* moves to another key or provider of the same model first, then to the next tier, and only then back to a tried
|
|
202
|
+
* lane. Within the first non-empty tier: the fewest in-flight requests, then latency within the tolerance of the
|
|
203
|
+
* best EWMA, then smooth weighted round-robin. The concurrency check and the hold run without an await between
|
|
204
|
+
* them, so concurrent calls never overshoot a cap.
|
|
132
205
|
*/
|
|
133
206
|
async pick(lanes, need, tried, leaseMs) {
|
|
134
207
|
const now = Date.now();
|
|
208
|
+
await this.#restore(lanes, now);
|
|
135
209
|
const statuses = await Promise.all(lanes.map((lane) => this.#status(lane, need, now)));
|
|
136
210
|
let usable = [];
|
|
137
211
|
let waitUntil = Infinity;
|
|
212
|
+
const out = { keys: new Set(), budgets: new Set(), quota: false, other: false };
|
|
138
213
|
lanes.forEach((lane, index) => {
|
|
139
214
|
const status = statuses[index];
|
|
140
215
|
if (status === undefined)
|
|
141
216
|
return;
|
|
142
217
|
if (!status.usable) {
|
|
143
218
|
waitUntil = Math.min(waitUntil, status.until);
|
|
219
|
+
if (status.block === "blacklist-quota" || status.block === "blacklist-auth")
|
|
220
|
+
out.keys.add(lane.pair);
|
|
221
|
+
if (status.block === "budget")
|
|
222
|
+
out.budgets.add(lane.provider.id);
|
|
223
|
+
if (status.block === "blacklist-quota" || status.block === "budget")
|
|
224
|
+
out.quota = true;
|
|
225
|
+
if (status.block === "soon")
|
|
226
|
+
out.other = true;
|
|
144
227
|
return;
|
|
145
228
|
}
|
|
146
229
|
const tier = [
|
|
@@ -154,6 +237,7 @@ export class PoolState {
|
|
|
154
237
|
});
|
|
155
238
|
usable.sort((a, b) => compareTiers(a.tier, b.tier));
|
|
156
239
|
let busy = false;
|
|
240
|
+
const anyUsable = usable.length > 0;
|
|
157
241
|
for (;;) {
|
|
158
242
|
const free = usable.filter((entry) => !this.#busy(entry.lane));
|
|
159
243
|
if (free.length < usable.length)
|
|
@@ -181,7 +265,9 @@ export class PoolState {
|
|
|
181
265
|
busy = true;
|
|
182
266
|
usable = usable.filter((candidate) => candidate !== entry);
|
|
183
267
|
}
|
|
184
|
-
|
|
268
|
+
const reason = out.quota ? "quota" : "auth";
|
|
269
|
+
const blacklisted = !anyUsable && !out.other && (out.keys.size > 0 || out.budgets.size > 0) ? { reason, keys: [...out.keys], budgets: [...out.budgets] } : null;
|
|
270
|
+
return { waitUntil: busy ? Math.min(waitUntil, now) : waitUntil, busy, blacklisted };
|
|
185
271
|
}
|
|
186
272
|
#choose(group) {
|
|
187
273
|
const load = (lane) => this.#inflightKey.get(lane.pair) ?? 0;
|
|
@@ -205,7 +291,7 @@ export class PoolState {
|
|
|
205
291
|
let best;
|
|
206
292
|
let bestWeight = -Infinity;
|
|
207
293
|
for (const lane of pool) {
|
|
208
|
-
const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1);
|
|
294
|
+
const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1) * lane.weight;
|
|
209
295
|
total += weight;
|
|
210
296
|
const current = (this.#roundRobin.get(lane.id) ?? 0) + weight;
|
|
211
297
|
this.#roundRobin.set(lane.id, current);
|
|
@@ -243,7 +329,7 @@ export class PoolState {
|
|
|
243
329
|
const aborted = () => {
|
|
244
330
|
clearTimeout(timer);
|
|
245
331
|
this.#waiters.delete(done);
|
|
246
|
-
reject(
|
|
332
|
+
reject(new Error("aborted"));
|
|
247
333
|
};
|
|
248
334
|
const timer = setTimeout(done, Math.max(0, ms));
|
|
249
335
|
if (signal.aborted) {
|
|
@@ -312,7 +398,21 @@ export class PoolState {
|
|
|
312
398
|
to,
|
|
313
399
|
});
|
|
314
400
|
}
|
|
315
|
-
|
|
401
|
+
async #spend(lane, cost, now) {
|
|
402
|
+
if (lane.provider.budget === undefined || cost === undefined || cost <= 0)
|
|
403
|
+
return;
|
|
404
|
+
const id = this.#providerId(lane.provider.id);
|
|
405
|
+
const record = await this.#read(id);
|
|
406
|
+
const { day, month } = periods(now);
|
|
407
|
+
await this.#store.set(id, {
|
|
408
|
+
...record,
|
|
409
|
+
spendDay: day,
|
|
410
|
+
spendDayCost: (record.spendDay === day ? record.spendDayCost : 0) + cost,
|
|
411
|
+
spendMonth: month,
|
|
412
|
+
spendMonthCost: (record.spendMonth === month ? record.spendMonthCost : 0) + cost,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
/** Records a success: closes the lane's breakers, stores its headroom and spend, and feeds its latency EWMA. */
|
|
316
416
|
async succeeded(lane, info, latencyMs, usage) {
|
|
317
417
|
const now = Date.now();
|
|
318
418
|
await this.#breakerSuccess(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, now);
|
|
@@ -328,29 +428,51 @@ export class PoolState {
|
|
|
328
428
|
counters.promptTokens += usage?.prompt_tokens ?? 0;
|
|
329
429
|
counters.completionTokens += usage?.completion_tokens ?? 0;
|
|
330
430
|
counters.cost += cost ?? 0;
|
|
431
|
+
await this.#spend(lane, cost, now);
|
|
331
432
|
return cost;
|
|
332
433
|
}
|
|
434
|
+
/** Blacklists a key: until a time, or (FOREVER) until its secret changes, and says so on `key-disabled`. */
|
|
435
|
+
async #blacklist(lane, reason, until, fingerprint, code) {
|
|
436
|
+
const id = this.#keyId(lane.provider.id, lane.key.alias);
|
|
437
|
+
const record = await this.#read(id);
|
|
438
|
+
await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason, disabledCode: code ?? null, fingerprint });
|
|
439
|
+
this.#emitter.emit("key-disabled", {
|
|
440
|
+
type: "key-disabled",
|
|
441
|
+
provider: lane.provider.id,
|
|
442
|
+
key: lane.key.alias,
|
|
443
|
+
reason,
|
|
444
|
+
until: until >= FOREVER ? null : until,
|
|
445
|
+
...(code === undefined ? {} : { code }),
|
|
446
|
+
});
|
|
447
|
+
}
|
|
333
448
|
/**
|
|
334
|
-
* Records a failure by class: a 429 cools the lane down (retry-after or the default)
|
|
335
|
-
* key
|
|
336
|
-
*
|
|
449
|
+
* Records a failure by class: a 429 cools the lane down (retry-after or the default); a spent quota blacklists the
|
|
450
|
+
* key until its window ends (the provider's, a retry-after, else `quotaBlacklistMs`) or its secret changes; a
|
|
451
|
+
* refused or missing secret blacklists it until the secret changes (or `authDisableMs`); a transient failure counts
|
|
452
|
+
* toward the key's and the provider's breakers (honouring retry-after). Any other class touches no shared state.
|
|
453
|
+
* `secret` is the value the attempt sent, kept only as its fingerprint.
|
|
337
454
|
*/
|
|
338
|
-
async failed(lane,
|
|
455
|
+
async failed(lane, error, secret) {
|
|
339
456
|
const now = Date.now();
|
|
340
457
|
const counters = this.#count(lane);
|
|
458
|
+
const errorClass = error.errorClass;
|
|
341
459
|
counters.failures[errorClass] = (counters.failures[errorClass] ?? 0) + 1;
|
|
460
|
+
const info = error.info;
|
|
342
461
|
const retryAt = info?.retryAt ?? 0;
|
|
343
462
|
if (errorClass === "rate_limit") {
|
|
344
463
|
await this.#headroom(lane, info, retryAt > now ? retryAt : now + this.#settings.rateLimitCooldownMs);
|
|
345
464
|
return;
|
|
346
465
|
}
|
|
466
|
+
if (errorClass === "quota") {
|
|
467
|
+
// The provider's window, else its request-limit reset (Groq's daily requests), else the pool's TTL; a later retry-after wins.
|
|
468
|
+
const reset = info !== null && info.remainingRequests === 0 && info.requestsResetAt > now ? info.requestsResetAt : now + this.#settings.quotaBlacklistMs;
|
|
469
|
+
const window = error.window === undefined ? reset : windowEnd(error.window, now);
|
|
470
|
+
await this.#blacklist(lane, "quota", Math.max(window, retryAt), fingerprintOf(secret), error.code);
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
347
473
|
if (errorClass === "auth") {
|
|
348
|
-
const id = this.#keyId(lane.provider.id, lane.key.alias);
|
|
349
|
-
const record = await this.#read(id);
|
|
350
474
|
const until = this.#settings.authDisableMs === null ? FOREVER : now + this.#settings.authDisableMs;
|
|
351
|
-
|
|
352
|
-
await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason });
|
|
353
|
-
this.#emitter.emit("key-disabled", { type: "key-disabled", provider: lane.provider.id, key: lane.key.alias, reason, until: until >= FOREVER ? null : until });
|
|
475
|
+
await this.#blacklist(lane, error.missingKey ? "missing" : "auth", until, fingerprintOf(error.missingKey ? undefined : secret), error.code);
|
|
354
476
|
return;
|
|
355
477
|
}
|
|
356
478
|
if (errorClass === "transient" || errorClass === "timeout" || errorClass === "network" || errorClass === "stream") {
|
|
@@ -384,11 +506,14 @@ export class PoolState {
|
|
|
384
506
|
failures: keyRecord.failures,
|
|
385
507
|
disabled,
|
|
386
508
|
disabledReason: disabled ? keyRecord.disabledReason : null,
|
|
509
|
+
disabledUntil: disabled && keyRecord.disabledUntil < FOREVER ? keyRecord.disabledUntil : null,
|
|
387
510
|
inflight: this.#inflightKey.get(`${provider.id}/${key.alias}`) ?? 0,
|
|
388
511
|
models,
|
|
389
512
|
};
|
|
390
513
|
}));
|
|
391
|
-
|
|
514
|
+
const { day, month } = periods(now);
|
|
515
|
+
const spend = provider.budget === undefined ? {} : { spend: { day: record.spendDay === day ? record.spendDayCost : 0, month: record.spendMonth === month ? record.spendMonthCost : 0 } };
|
|
516
|
+
return { id: provider.id, kind: presetOf(provider).kind, breaker: breakerState(record, now), failures: record.failures, inflight: this.#inflightProvider.get(provider.id) ?? 0, ...spend, keys };
|
|
392
517
|
}));
|
|
393
518
|
return { at: now, providers };
|
|
394
519
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ProviderConfig, ProviderKind } from "./types.js";
|
|
2
|
+
/** The wire protocol a provider speaks: OpenAI chat-completions and embeddings, or Anthropic Messages. */
|
|
3
|
+
export type Protocol = "openai" | "anthropic";
|
|
4
|
+
/** When a provider's exhausted quota comes back when its answer does not say: never on its own, the next UTC month, or a calendar day in a time zone. */
|
|
5
|
+
export type QuotaWindow = {
|
|
6
|
+
readonly kind: "month-utc";
|
|
7
|
+
} | {
|
|
8
|
+
readonly kind: "day";
|
|
9
|
+
readonly timeZone: string;
|
|
10
|
+
};
|
|
11
|
+
/** What a provider kind fixes: its protocol, default base URL, auth header, embeddings support and env prefix. */
|
|
12
|
+
export interface ProviderPreset {
|
|
13
|
+
readonly kind: ProviderKind;
|
|
14
|
+
readonly protocol: Protocol;
|
|
15
|
+
/** Null: the base URL is required. */
|
|
16
|
+
readonly baseUrl: string | null;
|
|
17
|
+
/** `bearer` sends `Authorization: Bearer <key>`; `x-api-key` sends `x-api-key: <key>` (Anthropic). */
|
|
18
|
+
readonly auth: "bearer" | "x-api-key";
|
|
19
|
+
/** False when the provider has no embeddings API: a model there declaring `embeddings` is refused. */
|
|
20
|
+
readonly embeddings: boolean;
|
|
21
|
+
/** The prefix of its canonical env variables (`<PREFIX>_API_KEY`, `_2`, ..., `<PREFIX>_BASE_URL`); null: the app names one. */
|
|
22
|
+
readonly envPrefix: string | null;
|
|
23
|
+
/** The official docs this preset was read from (docs/design.md lists the date). */
|
|
24
|
+
readonly docs: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The provider matrix (docs/design.md): read from each provider's official documentation on 2026-10-31. A base URL is
|
|
28
|
+
* the default only; `baseUrl` (or `<PREFIX>_BASE_URL`) moves it, e.g. Qwen to another region or workspace host.
|
|
29
|
+
*/
|
|
30
|
+
export declare const PROVIDER_PRESETS: Readonly<Record<ProviderKind, ProviderPreset>>;
|
|
31
|
+
/** The preset of a provider: its kind's, `openai-compatible` when it names none. */
|
|
32
|
+
export declare function presetOf(provider: ProviderConfig): ProviderPreset;
|
|
33
|
+
/** The base URL a provider is called at, without a trailing slash; empty when neither the provider nor its kind has one. */
|
|
34
|
+
export declare function baseUrlOf(provider: ProviderConfig): string;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provider matrix (docs/design.md): read from each provider's official documentation on 2026-10-31. A base URL is
|
|
3
|
+
* the default only; `baseUrl` (or `<PREFIX>_BASE_URL`) moves it, e.g. Qwen to another region or workspace host.
|
|
4
|
+
*/
|
|
5
|
+
export const PROVIDER_PRESETS = Object.freeze({
|
|
6
|
+
openai: { kind: "openai", protocol: "openai", baseUrl: "https://api.openai.com/v1", auth: "bearer", embeddings: true, envPrefix: "OPENAI", docs: "https://developers.openai.com/api/docs/guides/error-codes" },
|
|
7
|
+
anthropic: { kind: "anthropic", protocol: "anthropic", baseUrl: "https://api.anthropic.com/v1", auth: "x-api-key", embeddings: false, envPrefix: "ANTHROPIC", docs: "https://platform.claude.com/docs/en/api/errors" },
|
|
8
|
+
openrouter: { kind: "openrouter", protocol: "openai", baseUrl: "https://openrouter.ai/api/v1", auth: "bearer", embeddings: true, envPrefix: "OPENROUTER", docs: "https://openrouter.ai/docs/api/reference/errors-and-debugging" },
|
|
9
|
+
qwen: { kind: "qwen", protocol: "openai", baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", auth: "bearer", embeddings: true, envPrefix: "DASHSCOPE", docs: "https://www.alibabacloud.com/help/en/model-studio/error-code" },
|
|
10
|
+
deepseek: { kind: "deepseek", protocol: "openai", baseUrl: "https://api.deepseek.com", auth: "bearer", embeddings: false, envPrefix: "DEEPSEEK", docs: "https://api-docs.deepseek.com/quick_start/error_codes" },
|
|
11
|
+
gemini: { kind: "gemini", protocol: "openai", baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai", auth: "bearer", embeddings: true, envPrefix: "GEMINI", docs: "https://ai.google.dev/gemini-api/docs/api-errors" },
|
|
12
|
+
groq: { kind: "groq", protocol: "openai", baseUrl: "https://api.groq.com/openai/v1", auth: "bearer", embeddings: false, envPrefix: "GROQ", docs: "https://console.groq.com/docs/errors" },
|
|
13
|
+
"openai-compatible": { kind: "openai-compatible", protocol: "openai", baseUrl: null, auth: "bearer", embeddings: true, envPrefix: null, docs: "https://docs.vllm.ai/en/latest/serving/online_serving/" },
|
|
14
|
+
});
|
|
15
|
+
/** The preset of a provider: its kind's, `openai-compatible` when it names none. */
|
|
16
|
+
export function presetOf(provider) {
|
|
17
|
+
return PROVIDER_PRESETS[provider.kind ?? "openai-compatible"];
|
|
18
|
+
}
|
|
19
|
+
/** The base URL a provider is called at, without a trailing slash; empty when neither the provider nor its kind has one. */
|
|
20
|
+
export function baseUrlOf(provider) {
|
|
21
|
+
return (provider.baseUrl ?? presetOf(provider).baseUrl ?? "").replace(/\/+$/, "");
|
|
22
|
+
}
|