@starci/ai-router 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +133 -40
  3. package/dist/ai-router.service.d.ts +2 -2
  4. package/dist/ai-router.service.js +4 -2
  5. package/dist/attempt.d.ts +26 -0
  6. package/dist/attempt.js +29 -0
  7. package/dist/classify.d.ts +40 -0
  8. package/dist/classify.js +145 -0
  9. package/dist/config.d.ts +30 -5
  10. package/dist/config.js +184 -24
  11. package/dist/env.d.ts +70 -0
  12. package/dist/env.js +152 -0
  13. package/dist/errors.d.ts +1 -3
  14. package/dist/errors.js +4 -23
  15. package/dist/fake.d.ts +624 -5
  16. package/dist/fake.js +177 -21
  17. package/dist/gateway/config-file.d.ts +8 -3
  18. package/dist/gateway/config-file.js +53 -8
  19. package/dist/gateway/gateway.d.ts +2 -0
  20. package/dist/gateway/gateway.js +19 -3
  21. package/dist/gateway/index.d.ts +10 -1
  22. package/dist/gateway/index.js +5 -0
  23. package/dist/gateway/serve.js +7 -2
  24. package/dist/headers.d.ts +1 -1
  25. package/dist/headers.js +12 -6
  26. package/dist/pool.d.ts +37 -16
  27. package/dist/pool.js +173 -48
  28. package/dist/providers.d.ts +34 -0
  29. package/dist/providers.js +22 -0
  30. package/dist/router.js +108 -105
  31. package/dist/secrets.d.ts +8 -0
  32. package/dist/secrets.js +22 -0
  33. package/dist/shapes.d.ts +11 -0
  34. package/dist/shapes.js +124 -0
  35. package/dist/state-stores.d.ts +38 -0
  36. package/dist/state-stores.js +101 -0
  37. package/dist/store.d.ts +26 -7
  38. package/dist/store.js +64 -2
  39. package/dist/testing.d.ts +2 -2
  40. package/dist/testing.js +1 -1
  41. package/dist/types.d.ts +126 -38
  42. package/dist/wire/anthropic.d.ts +21 -0
  43. package/dist/wire/anthropic.js +322 -0
  44. package/dist/wire/index.d.ts +5 -0
  45. package/dist/wire/index.js +7 -0
  46. package/dist/wire/wire.d.ts +42 -0
  47. package/dist/wire/wire.js +35 -0
  48. package/docs/design.md +279 -0
  49. package/package.json +13 -3
package/dist/headers.js CHANGED
@@ -31,12 +31,18 @@ function resetAt(headers, names, now) {
31
31
  if (raw === null)
32
32
  continue;
33
33
  const plain = Number(raw.trim());
34
- // A plain number past 1e9 is an epoch in seconds (x-ratelimit-reset of some providers); smaller is a delta.
34
+ // A plain number past 1e12 is an epoch in milliseconds (OpenRouter), past 1e9 one in seconds; smaller is a delta.
35
+ if (Number.isFinite(plain) && plain > 1e12)
36
+ return plain;
35
37
  if (Number.isFinite(plain) && plain > 1e9)
36
38
  return plain * 1000;
37
39
  const ms = parseDuration(raw);
38
40
  if (ms !== null)
39
41
  return now + ms;
42
+ // Anthropic's anthropic-ratelimit-*-reset is an RFC 3339 time.
43
+ const date = /^\d{4}-\d{2}-\d{2}T/.test(raw.trim()) ? Date.parse(raw.trim()) : Number.NaN;
44
+ if (!Number.isNaN(date))
45
+ return date;
40
46
  }
41
47
  return 0;
42
48
  }
@@ -54,13 +60,13 @@ export function retryAt(headers, now) {
54
60
  const date = Date.parse(after);
55
61
  return Number.isNaN(date) ? 0 : date;
56
62
  }
57
- /** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
63
+ /** Reads the `x-ratelimit-*` (OpenAI, Groq, OpenRouter), `anthropic-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
58
64
  export function readRateLimit(headers, now) {
59
65
  return {
60
- remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "x-ratelimit-remaining", "ratelimit-remaining"]),
61
- requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "x-ratelimit-reset", "ratelimit-reset"], now),
62
- remainingTokens: count(headers, ["x-ratelimit-remaining-tokens"]),
63
- tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens"], now),
66
+ remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "anthropic-ratelimit-requests-remaining", "x-ratelimit-remaining", "ratelimit-remaining"]),
67
+ requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "anthropic-ratelimit-requests-reset", "x-ratelimit-reset", "ratelimit-reset"], now),
68
+ remainingTokens: count(headers, ["x-ratelimit-remaining-tokens", "anthropic-ratelimit-tokens-remaining"]),
69
+ tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens", "anthropic-ratelimit-tokens-reset"], now),
64
70
  retryAt: retryAt(headers, now),
65
71
  };
66
72
  }
package/dist/pool.d.ts CHANGED
@@ -1,14 +1,17 @@
1
+ import type { AttemptError } from "./attempt.js";
1
2
  import type { ResolvedPolicy, Settings } from "./config.js";
2
3
  import type { Emitter } from "./events.js";
3
4
  import type { RateLimitInfo } from "./headers.js";
4
5
  import type { StateStore } from "./store.js";
5
- import type { Capability, ErrorClass, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } from "./types.js";
6
- /** One routable (provider, key, model) triple and the rank of its model in the call's policy. */
6
+ import type { Capability, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } from "./types.js";
7
+ /** One routable (provider, key, model) triple, the rank of its tier in the call's policy and its weight there. */
7
8
  export interface Lane {
8
9
  readonly provider: ProviderConfig;
9
10
  readonly key: KeyConfig;
10
11
  readonly model: ModelConfig;
11
12
  readonly rank: number;
13
+ /** The candidate's weight in its tier (1 for a model named by `model` or `fallbacks`). */
14
+ readonly weight: number;
12
15
  /** `provider/key/model`, the lane's name in local counters. */
13
16
  readonly id: string;
14
17
  /** `provider/key`, the pair a retry moves away from. */
@@ -21,6 +24,10 @@ export interface Need {
21
24
  readonly promptTokens: number;
22
25
  /** `max_tokens` or `max_completion_tokens`, 0 when unbounded. */
23
26
  readonly outputTokens: number;
27
+ /** True keeps every lane on the first lane's model identity (an embeddings call, a pinned profile). */
28
+ readonly oneIdentity: boolean;
29
+ /** The vector length an embeddings call asked for; a model declared at another is no candidate. */
30
+ readonly dimensions: number | null;
24
31
  }
25
32
  /** What a call has tried so far: lanes, key pairs and providers. */
26
33
  export interface Tried {
@@ -33,17 +40,27 @@ export interface Pick {
33
40
  readonly lane: Lane;
34
41
  readonly leases: readonly string[];
35
42
  }
43
+ /** Why the lanes that are out are out, when every one of them is out for its key or its provider's budget. */
44
+ export interface Blacklisted {
45
+ /** `quota` when any lane is out for a spent quota or budget, else `auth` (a refused or missing secret). */
46
+ readonly reason: "quota" | "auth";
47
+ /** The aliases, `provider/key`, that are blacklisted, and the providers past their budget. */
48
+ readonly keys: readonly string[];
49
+ readonly budgets: readonly string[];
50
+ }
36
51
  /** No lane is usable now: `waitUntil` is when the earliest one may be (Infinity: never within reach). */
37
52
  export interface Blocked {
38
53
  readonly waitUntil: number;
39
54
  /** True when a lane waits only on in-process concurrency or a probe another attempt holds. */
40
55
  readonly busy: boolean;
56
+ /** Set when every lane is out because its key is blacklisted or its provider's budget is spent. */
57
+ readonly blacklisted: Blacklisted | null;
41
58
  }
42
59
  /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
43
60
  export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
44
61
  /**
45
- * The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
46
- * round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
62
+ * The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
63
+ * EWMAs and round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
47
64
  */
48
65
  export declare class PoolState {
49
66
  #private;
@@ -55,16 +72,18 @@ export declare class PoolState {
55
72
  };
56
73
  /** Binds the state to a pool's providers, settings, store and emitter. */
57
74
  constructor(providers: readonly ProviderConfig[], settings: Settings, store: StateStore, emitter: Emitter);
58
- /** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
75
+ /**
76
+ * Every lane that serves one of the policy's candidates with the capabilities, context window and budget the call
77
+ * needs, tier by tier. With `need.oneIdentity`, a lane whose model identity differs from the first lane's is left out.
78
+ */
59
79
  lanes(policy: ResolvedPolicy, need: Need): Lane[];
60
80
  /**
61
81
  * Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
62
- * call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
63
- * to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
64
- * Within the first non-empty tier:
65
- * the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
66
- * round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
67
- * overshoot a cap.
82
+ * call, the policy's candidate order, a key not yet tried, a provider not yet tried, enough headroom. So a retry
83
+ * moves to another key or provider of the same model first, then to the next tier, and only then back to a tried
84
+ * lane. Within the first non-empty tier: the fewest in-flight requests, then latency within the tolerance of the
85
+ * best EWMA, then smooth weighted round-robin. The concurrency check and the hold run without an await between
86
+ * them, so concurrent calls never overshoot a cap.
68
87
  */
69
88
  pick(lanes: readonly Lane[], need: Need, tried: Tried, leaseMs: number): Promise<Pick | Blocked>;
70
89
  /** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
@@ -73,14 +92,16 @@ export declare class PoolState {
73
92
  release(lane: Lane, leases: readonly string[]): Promise<void>;
74
93
  /** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
75
94
  waitForCapacity(ms: number, signal: AbortSignal): Promise<void>;
76
- /** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
95
+ /** Records a success: closes the lane's breakers, stores its headroom and spend, and feeds its latency EWMA. */
77
96
  succeeded(lane: Lane, info: RateLimitInfo | null, latencyMs: number, usage: Usage | undefined): Promise<number | undefined>;
78
97
  /**
79
- * Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
80
- * key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
81
- * class touches no shared state.
98
+ * Records a failure by class: a 429 cools the lane down (retry-after or the default); a spent quota blacklists the
99
+ * key until its window ends (the provider's, a retry-after, else `quotaBlacklistMs`) or its secret changes; a
100
+ * refused or missing secret blacklists it until the secret changes (or `authDisableMs`); a transient failure counts
101
+ * toward the key's and the provider's breakers (honouring retry-after). Any other class touches no shared state.
102
+ * `secret` is the value the attempt sent, kept only as its fingerprint.
82
103
  */
83
- failed(lane: Lane, errorClass: ErrorClass, info: RateLimitInfo | null, missingKey: boolean): Promise<void>;
104
+ failed(lane: Lane, error: AttemptError, secret: string | undefined): Promise<void>;
84
105
  /** Reads every record into a health snapshot. */
85
106
  health(): Promise<HealthSnapshot>;
86
107
  /** Returns this process's counters. */
package/dist/pool.js CHANGED
@@ -1,5 +1,8 @@
1
- import { DEFAULT_CAPABILITIES, modelMatches } from "./config.js";
2
- import { EMPTY_RECORD } from "./store.js";
1
+ import { windowEnd } from "./classify.js";
2
+ import { DEFAULT_CAPABILITIES, identityOf, modelMatches } from "./config.js";
3
+ import { presetOf } from "./providers.js";
4
+ import { fingerprintOf, resolveSecret } from "./secrets.js";
5
+ import { EMPTY_RECORD, toHealthRecord } from "./store.js";
3
6
  const FOREVER = Number.MAX_SAFE_INTEGER;
4
7
  /** The latency samples a lane needs before its EWMA ranks it: one cold call (a JIT warm-up, a first TLS handshake) never decides. */
5
8
  const LATENCY_SAMPLES = 3;
@@ -17,9 +20,28 @@ export function costOf(model, usage) {
17
20
  const completion = usage.completion_tokens ?? 0;
18
21
  return (usage.prompt_tokens * model.cost.input + completion * model.cost.output) / 1_000_000;
19
22
  }
23
+ /** The UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`) of a time. */
24
+ function periods(now) {
25
+ const day = new Date(now).toISOString().slice(0, 10);
26
+ return { day, month: day.slice(0, 7) };
27
+ }
28
+ /** When a provider past its budget is a candidate again: the next UTC day or month, or 0 when it is within it. */
29
+ function budgetUntil(provider, record, now) {
30
+ const budget = provider.budget;
31
+ if (budget === undefined)
32
+ return 0;
33
+ const { day, month } = periods(now);
34
+ const date = new Date(now);
35
+ let until = 0;
36
+ if (budget.costPerMonth !== undefined && record.spendMonth === month && record.spendMonthCost >= budget.costPerMonth)
37
+ until = Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1);
38
+ else if (budget.costPerDay !== undefined && record.spendDay === day && record.spendDayCost >= budget.costPerDay)
39
+ until = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate() + 1);
40
+ return until;
41
+ }
20
42
  /**
21
- * The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
22
- * round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
43
+ * The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
44
+ * EWMAs and round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
23
45
  */
24
46
  export class PoolState {
25
47
  #providers;
@@ -51,33 +73,50 @@ export class PoolState {
51
73
  return `${this.#settings.namespace}:lane:${provider}/${alias}/${model}`;
52
74
  }
53
75
  async #read(id) {
54
- return (await this.#store.get(id)) ?? EMPTY_RECORD;
76
+ // A record an older version or another adapter wrote is read field by field, its missing fields empty.
77
+ const stored = await this.#store.get(id);
78
+ return stored === undefined ? EMPTY_RECORD : (toHealthRecord(stored) ?? EMPTY_RECORD);
55
79
  }
56
- /** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
80
+ /**
81
+ * Every lane that serves one of the policy's candidates with the capabilities, context window and budget the call
82
+ * needs, tier by tier. With `need.oneIdentity`, a lane whose model identity differs from the first lane's is left out.
83
+ */
57
84
  lanes(policy, need) {
58
85
  const lanes = [];
59
- policy.models.forEach((name, rank) => {
60
- for (const provider of this.#providers) {
61
- if (policy.providers !== null && !policy.providers.has(provider.id))
62
- continue;
63
- for (const model of provider.models) {
64
- if (!modelMatches(model, name))
86
+ let identity;
87
+ policy.tiers.forEach((tier, rank) => {
88
+ for (const ref of tier) {
89
+ for (const provider of this.#providers) {
90
+ if (policy.providers !== null && !policy.providers.has(provider.id))
65
91
  continue;
66
- const capabilities = model.capabilities ?? DEFAULT_CAPABILITIES;
67
- if (!need.capabilities.every((capability) => capabilities.includes(capability)))
92
+ if (ref.provider !== null && ref.provider !== provider.id)
68
93
  continue;
69
- if (model.contextWindow !== undefined && need.promptTokens + need.outputTokens > model.contextWindow)
70
- continue;
71
- if (policy.maxCost !== null && model.cost !== undefined) {
72
- const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
73
- if (worst > policy.maxCost)
94
+ for (const model of provider.models) {
95
+ if (!modelMatches(model, ref.model))
74
96
  continue;
75
- }
76
- for (const key of provider.keys) {
77
- const id = `${provider.id}/${key.alias}/${model.id}`;
78
- if (lanes.some((lane) => lane.id === id))
97
+ const capabilities = model.capabilities ?? DEFAULT_CAPABILITIES;
98
+ if (!need.capabilities.every((capability) => capabilities.includes(capability)))
99
+ continue;
100
+ if (model.contextWindow !== undefined && need.promptTokens + need.outputTokens > model.contextWindow)
101
+ continue;
102
+ if (policy.maxCost !== null && model.cost !== undefined) {
103
+ const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
104
+ if (worst > policy.maxCost)
105
+ continue;
106
+ }
107
+ if (need.dimensions !== null && model.dimensions !== undefined && model.dimensions !== need.dimensions)
79
108
  continue;
80
- lanes.push({ provider, key, model, rank, id, pair: `${provider.id}/${key.alias}` });
109
+ if (need.oneIdentity) {
110
+ identity ??= identityOf(model);
111
+ if (identityOf(model) !== identity)
112
+ continue;
113
+ }
114
+ for (const key of provider.keys) {
115
+ const id = `${provider.id}/${key.alias}/${model.id}`;
116
+ if (lanes.some((lane) => lane.id === id))
117
+ continue;
118
+ lanes.push({ provider, key, model, rank, weight: ref.weight, id, pair: `${provider.id}/${key.alias}` });
119
+ }
81
120
  }
82
121
  }
83
122
  }
@@ -91,10 +130,23 @@ export class PoolState {
91
130
  this.#read(this.#laneId(lane.provider.id, lane.key.alias, lane.model.id)),
92
131
  ]);
93
132
  const blocks = [];
94
- if (key.disabledUntil > now)
133
+ let block = null;
134
+ if (key.disabledUntil > now) {
95
135
  blocks.push(key.disabledUntil >= FOREVER ? Infinity : key.disabledUntil);
136
+ block = key.disabledReason === "quota" ? "blacklist-quota" : "blacklist-auth";
137
+ }
138
+ const budget = budgetUntil(lane.provider, provider, now);
139
+ if (budget > now) {
140
+ blocks.push(budget);
141
+ block ??= "budget";
142
+ }
143
+ const soon = blocks.length;
96
144
  const probes = [];
97
- for (const [record, id] of [[key, this.#keyId(lane.provider.id, lane.key.alias)], [provider, this.#providerId(lane.provider.id)]]) {
145
+ const breakers = [
146
+ { record: key, id: this.#keyId(lane.provider.id, lane.key.alias) },
147
+ { record: provider, id: this.#providerId(lane.provider.id) },
148
+ ];
149
+ for (const { record, id } of breakers) {
98
150
  const state = breakerState(record, now);
99
151
  if (state === "open")
100
152
  blocks.push(record.openUntil);
@@ -108,10 +160,32 @@ export class PoolState {
108
160
  if (headroom.remainingTokens !== null && headroom.remainingTokens < need.promptTokens + need.outputTokens && headroom.tokensResetAt > now) {
109
161
  blocks.push(headroom.tokensResetAt);
110
162
  }
163
+ if (block === null && blocks.length > soon)
164
+ block = "soon";
111
165
  if (blocks.length > 0)
112
- return { usable: false, until: Math.max(...blocks), low: false, probes };
166
+ return { usable: false, until: Math.max(...blocks), low: false, probes, block };
113
167
  const low = headroom.remainingRequests !== null && headroom.remainingRequests <= this.#settings.lowHeadroom && headroom.requestsResetAt > now;
114
- return { usable: true, until: now, low, probes };
168
+ return { usable: true, until: now, low, probes, block: null };
169
+ }
170
+ /**
171
+ * Lifts the blacklist of every key among the lanes whose secret changed since it was blacklisted (a missing secret
172
+ * that is now set counts): a new secret is a new key, and deserves a try at once.
173
+ */
174
+ async #restore(lanes, now) {
175
+ const seen = new Set();
176
+ for (const lane of lanes) {
177
+ if (seen.has(lane.pair))
178
+ continue;
179
+ seen.add(lane.pair);
180
+ const id = this.#keyId(lane.provider.id, lane.key.alias);
181
+ const record = await this.#read(id);
182
+ if (record.disabledUntil <= now || record.disabledReason === null || record.fingerprint === null)
183
+ continue;
184
+ if (fingerprintOf(await resolveSecret(lane.key)) === record.fingerprint)
185
+ continue;
186
+ await this.#store.set(id, { ...record, disabledUntil: 0, disabledReason: null, disabledCode: null, fingerprint: null });
187
+ this.#emitter.emit("key-restored", { type: "key-restored", provider: lane.provider.id, key: lane.key.alias, was: record.disabledReason, reason: "secret_changed" });
188
+ }
115
189
  }
116
190
  #busy(lane) {
117
191
  return ((this.#inflightKey.get(lane.pair) ?? 0) >= (lane.key.maxConcurrency ?? Infinity) ||
@@ -123,24 +197,33 @@ export class PoolState {
123
197
  }
124
198
  /**
125
199
  * Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
126
- * call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
127
- * to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
128
- * Within the first non-empty tier:
129
- * the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
130
- * round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
131
- * overshoot a cap.
200
+ * call, the policy's candidate order, a key not yet tried, a provider not yet tried, enough headroom. So a retry
201
+ * moves to another key or provider of the same model first, then to the next tier, and only then back to a tried
202
+ * lane. Within the first non-empty tier: the fewest in-flight requests, then latency within the tolerance of the
203
+ * best EWMA, then smooth weighted round-robin. The concurrency check and the hold run without an await between
204
+ * them, so concurrent calls never overshoot a cap.
132
205
  */
133
206
  async pick(lanes, need, tried, leaseMs) {
134
207
  const now = Date.now();
208
+ await this.#restore(lanes, now);
135
209
  const statuses = await Promise.all(lanes.map((lane) => this.#status(lane, need, now)));
136
210
  let usable = [];
137
211
  let waitUntil = Infinity;
212
+ const out = { keys: new Set(), budgets: new Set(), quota: false, other: false };
138
213
  lanes.forEach((lane, index) => {
139
214
  const status = statuses[index];
140
215
  if (status === undefined)
141
216
  return;
142
217
  if (!status.usable) {
143
218
  waitUntil = Math.min(waitUntil, status.until);
219
+ if (status.block === "blacklist-quota" || status.block === "blacklist-auth")
220
+ out.keys.add(lane.pair);
221
+ if (status.block === "budget")
222
+ out.budgets.add(lane.provider.id);
223
+ if (status.block === "blacklist-quota" || status.block === "budget")
224
+ out.quota = true;
225
+ if (status.block === "soon")
226
+ out.other = true;
144
227
  return;
145
228
  }
146
229
  const tier = [
@@ -154,6 +237,7 @@ export class PoolState {
154
237
  });
155
238
  usable.sort((a, b) => compareTiers(a.tier, b.tier));
156
239
  let busy = false;
240
+ const anyUsable = usable.length > 0;
157
241
  for (;;) {
158
242
  const free = usable.filter((entry) => !this.#busy(entry.lane));
159
243
  if (free.length < usable.length)
@@ -181,7 +265,9 @@ export class PoolState {
181
265
  busy = true;
182
266
  usable = usable.filter((candidate) => candidate !== entry);
183
267
  }
184
- return { waitUntil: busy ? Math.min(waitUntil, now) : waitUntil, busy };
268
+ const reason = out.quota ? "quota" : "auth";
269
+ const blacklisted = !anyUsable && !out.other && (out.keys.size > 0 || out.budgets.size > 0) ? { reason, keys: [...out.keys], budgets: [...out.budgets] } : null;
270
+ return { waitUntil: busy ? Math.min(waitUntil, now) : waitUntil, busy, blacklisted };
185
271
  }
186
272
  #choose(group) {
187
273
  const load = (lane) => this.#inflightKey.get(lane.pair) ?? 0;
@@ -205,7 +291,7 @@ export class PoolState {
205
291
  let best;
206
292
  let bestWeight = -Infinity;
207
293
  for (const lane of pool) {
208
- const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1);
294
+ const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1) * lane.weight;
209
295
  total += weight;
210
296
  const current = (this.#roundRobin.get(lane.id) ?? 0) + weight;
211
297
  this.#roundRobin.set(lane.id, current);
@@ -243,7 +329,7 @@ export class PoolState {
243
329
  const aborted = () => {
244
330
  clearTimeout(timer);
245
331
  this.#waiters.delete(done);
246
- reject(signal.reason);
332
+ reject(new Error("aborted"));
247
333
  };
248
334
  const timer = setTimeout(done, Math.max(0, ms));
249
335
  if (signal.aborted) {
@@ -312,7 +398,21 @@ export class PoolState {
312
398
  to,
313
399
  });
314
400
  }
315
- /** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
401
+ async #spend(lane, cost, now) {
402
+ if (lane.provider.budget === undefined || cost === undefined || cost <= 0)
403
+ return;
404
+ const id = this.#providerId(lane.provider.id);
405
+ const record = await this.#read(id);
406
+ const { day, month } = periods(now);
407
+ await this.#store.set(id, {
408
+ ...record,
409
+ spendDay: day,
410
+ spendDayCost: (record.spendDay === day ? record.spendDayCost : 0) + cost,
411
+ spendMonth: month,
412
+ spendMonthCost: (record.spendMonth === month ? record.spendMonthCost : 0) + cost,
413
+ });
414
+ }
415
+ /** Records a success: closes the lane's breakers, stores its headroom and spend, and feeds its latency EWMA. */
316
416
  async succeeded(lane, info, latencyMs, usage) {
317
417
  const now = Date.now();
318
418
  await this.#breakerSuccess(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, now);
@@ -328,29 +428,51 @@ export class PoolState {
328
428
  counters.promptTokens += usage?.prompt_tokens ?? 0;
329
429
  counters.completionTokens += usage?.completion_tokens ?? 0;
330
430
  counters.cost += cost ?? 0;
431
+ await this.#spend(lane, cost, now);
331
432
  return cost;
332
433
  }
434
+ /** Blacklists a key: until a time, or (FOREVER) until its secret changes, and says so on `key-disabled`. */
435
+ async #blacklist(lane, reason, until, fingerprint, code) {
436
+ const id = this.#keyId(lane.provider.id, lane.key.alias);
437
+ const record = await this.#read(id);
438
+ await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason, disabledCode: code ?? null, fingerprint });
439
+ this.#emitter.emit("key-disabled", {
440
+ type: "key-disabled",
441
+ provider: lane.provider.id,
442
+ key: lane.key.alias,
443
+ reason,
444
+ until: until >= FOREVER ? null : until,
445
+ ...(code === undefined ? {} : { code }),
446
+ });
447
+ }
333
448
  /**
334
- * Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
335
- * key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
336
- * class touches no shared state.
449
+ * Records a failure by class: a 429 cools the lane down (retry-after or the default); a spent quota blacklists the
450
+ * key until its window ends (the provider's, a retry-after, else `quotaBlacklistMs`) or its secret changes; a
451
+ * refused or missing secret blacklists it until the secret changes (or `authDisableMs`); a transient failure counts
452
+ * toward the key's and the provider's breakers (honouring retry-after). Any other class touches no shared state.
453
+ * `secret` is the value the attempt sent, kept only as its fingerprint.
337
454
  */
338
- async failed(lane, errorClass, info, missingKey) {
455
+ async failed(lane, error, secret) {
339
456
  const now = Date.now();
340
457
  const counters = this.#count(lane);
458
+ const errorClass = error.errorClass;
341
459
  counters.failures[errorClass] = (counters.failures[errorClass] ?? 0) + 1;
460
+ const info = error.info;
342
461
  const retryAt = info?.retryAt ?? 0;
343
462
  if (errorClass === "rate_limit") {
344
463
  await this.#headroom(lane, info, retryAt > now ? retryAt : now + this.#settings.rateLimitCooldownMs);
345
464
  return;
346
465
  }
466
+ if (errorClass === "quota") {
467
+ // The provider's window, else its request-limit reset (Groq's daily requests), else the pool's TTL; a later retry-after wins.
468
+ const reset = info !== null && info.remainingRequests === 0 && info.requestsResetAt > now ? info.requestsResetAt : now + this.#settings.quotaBlacklistMs;
469
+ const window = error.window === undefined ? reset : windowEnd(error.window, now);
470
+ await this.#blacklist(lane, "quota", Math.max(window, retryAt), fingerprintOf(secret), error.code);
471
+ return;
472
+ }
347
473
  if (errorClass === "auth") {
348
- const id = this.#keyId(lane.provider.id, lane.key.alias);
349
- const record = await this.#read(id);
350
474
  const until = this.#settings.authDisableMs === null ? FOREVER : now + this.#settings.authDisableMs;
351
- const reason = missingKey ? "missing" : "auth";
352
- await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason });
353
- this.#emitter.emit("key-disabled", { type: "key-disabled", provider: lane.provider.id, key: lane.key.alias, reason, until: until >= FOREVER ? null : until });
475
+ await this.#blacklist(lane, error.missingKey ? "missing" : "auth", until, fingerprintOf(error.missingKey ? undefined : secret), error.code);
354
476
  return;
355
477
  }
356
478
  if (errorClass === "transient" || errorClass === "timeout" || errorClass === "network" || errorClass === "stream") {
@@ -384,11 +506,14 @@ export class PoolState {
384
506
  failures: keyRecord.failures,
385
507
  disabled,
386
508
  disabledReason: disabled ? keyRecord.disabledReason : null,
509
+ disabledUntil: disabled && keyRecord.disabledUntil < FOREVER ? keyRecord.disabledUntil : null,
387
510
  inflight: this.#inflightKey.get(`${provider.id}/${key.alias}`) ?? 0,
388
511
  models,
389
512
  };
390
513
  }));
391
- return { id: provider.id, breaker: breakerState(record, now), failures: record.failures, inflight: this.#inflightProvider.get(provider.id) ?? 0, keys };
514
+ const { day, month } = periods(now);
515
+ const spend = provider.budget === undefined ? {} : { spend: { day: record.spendDay === day ? record.spendDayCost : 0, month: record.spendMonth === month ? record.spendMonthCost : 0 } };
516
+ return { id: provider.id, kind: presetOf(provider).kind, breaker: breakerState(record, now), failures: record.failures, inflight: this.#inflightProvider.get(provider.id) ?? 0, ...spend, keys };
392
517
  }));
393
518
  return { at: now, providers };
394
519
  }
@@ -0,0 +1,34 @@
1
+ import type { ProviderConfig, ProviderKind } from "./types.js";
2
+ /** The wire protocol a provider speaks: OpenAI chat-completions and embeddings, or Anthropic Messages. */
3
+ export type Protocol = "openai" | "anthropic";
4
+ /** When a provider's exhausted quota comes back when its answer does not say: never on its own, the next UTC month, or a calendar day in a time zone. */
5
+ export type QuotaWindow = {
6
+ readonly kind: "month-utc";
7
+ } | {
8
+ readonly kind: "day";
9
+ readonly timeZone: string;
10
+ };
11
+ /** What a provider kind fixes: its protocol, default base URL, auth header, embeddings support and env prefix. */
12
+ export interface ProviderPreset {
13
+ readonly kind: ProviderKind;
14
+ readonly protocol: Protocol;
15
+ /** Null: the base URL is required. */
16
+ readonly baseUrl: string | null;
17
+ /** `bearer` sends `Authorization: Bearer <key>`; `x-api-key` sends `x-api-key: <key>` (Anthropic). */
18
+ readonly auth: "bearer" | "x-api-key";
19
+ /** False when the provider has no embeddings API: a model there declaring `embeddings` is refused. */
20
+ readonly embeddings: boolean;
21
+ /** The prefix of its canonical env variables (`<PREFIX>_API_KEY`, `_2`, ..., `<PREFIX>_BASE_URL`); null: the app names one. */
22
+ readonly envPrefix: string | null;
23
+ /** The official docs this preset was read from (docs/design.md lists the date). */
24
+ readonly docs: string;
25
+ }
26
+ /**
27
+ * The provider matrix (docs/design.md): read from each provider's official documentation on 2026-10-31. A base URL is
28
+ * the default only; `baseUrl` (or `<PREFIX>_BASE_URL`) moves it, e.g. Qwen to another region or workspace host.
29
+ */
30
+ export declare const PROVIDER_PRESETS: Readonly<Record<ProviderKind, ProviderPreset>>;
31
+ /** The preset of a provider: its kind's, `openai-compatible` when it names none. */
32
+ export declare function presetOf(provider: ProviderConfig): ProviderPreset;
33
+ /** The base URL a provider is called at, without a trailing slash; empty when neither the provider nor its kind has one. */
34
+ export declare function baseUrlOf(provider: ProviderConfig): string;
@@ -0,0 +1,22 @@
1
+ /**
2
+ * The provider matrix (docs/design.md): read from each provider's official documentation on 2026-10-31. A base URL is
3
+ * the default only; `baseUrl` (or `<PREFIX>_BASE_URL`) moves it, e.g. Qwen to another region or workspace host.
4
+ */
5
+ export const PROVIDER_PRESETS = Object.freeze({
6
+ openai: { kind: "openai", protocol: "openai", baseUrl: "https://api.openai.com/v1", auth: "bearer", embeddings: true, envPrefix: "OPENAI", docs: "https://developers.openai.com/api/docs/guides/error-codes" },
7
+ anthropic: { kind: "anthropic", protocol: "anthropic", baseUrl: "https://api.anthropic.com/v1", auth: "x-api-key", embeddings: false, envPrefix: "ANTHROPIC", docs: "https://platform.claude.com/docs/en/api/errors" },
8
+ openrouter: { kind: "openrouter", protocol: "openai", baseUrl: "https://openrouter.ai/api/v1", auth: "bearer", embeddings: true, envPrefix: "OPENROUTER", docs: "https://openrouter.ai/docs/api/reference/errors-and-debugging" },
9
+ qwen: { kind: "qwen", protocol: "openai", baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", auth: "bearer", embeddings: true, envPrefix: "DASHSCOPE", docs: "https://www.alibabacloud.com/help/en/model-studio/error-code" },
10
+ deepseek: { kind: "deepseek", protocol: "openai", baseUrl: "https://api.deepseek.com", auth: "bearer", embeddings: false, envPrefix: "DEEPSEEK", docs: "https://api-docs.deepseek.com/quick_start/error_codes" },
11
+ gemini: { kind: "gemini", protocol: "openai", baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai", auth: "bearer", embeddings: true, envPrefix: "GEMINI", docs: "https://ai.google.dev/gemini-api/docs/api-errors" },
12
+ groq: { kind: "groq", protocol: "openai", baseUrl: "https://api.groq.com/openai/v1", auth: "bearer", embeddings: false, envPrefix: "GROQ", docs: "https://console.groq.com/docs/errors" },
13
+ "openai-compatible": { kind: "openai-compatible", protocol: "openai", baseUrl: null, auth: "bearer", embeddings: true, envPrefix: null, docs: "https://docs.vllm.ai/en/latest/serving/online_serving/" },
14
+ });
15
+ /** The preset of a provider: its kind's, `openai-compatible` when it names none. */
16
+ export function presetOf(provider) {
17
+ return PROVIDER_PRESETS[provider.kind ?? "openai-compatible"];
18
+ }
19
+ /** The base URL a provider is called at, without a trailing slash; empty when neither the provider nor its kind has one. */
20
+ export function baseUrlOf(provider) {
21
+ return (provider.baseUrl ?? presetOf(provider).baseUrl ?? "").replace(/\/+$/, "");
22
+ }