@starci/ai-router 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ /** What a response's headers say about a key's headroom; times are epoch milliseconds, 0 when absent. */
2
+ export interface RateLimitInfo {
3
+ readonly remainingRequests: number | null;
4
+ readonly requestsResetAt: number;
5
+ readonly remainingTokens: number | null;
6
+ readonly tokensResetAt: number;
7
+ /** From `retry-after-ms` or `retry-after` (seconds or an HTTP date). */
8
+ readonly retryAt: number;
9
+ }
10
+ /** Parses an OpenAI reset duration (`1s`, `6m0s`, `20ms`, `1h2m`) or a plain number of seconds into milliseconds. */
11
+ export declare function parseDuration(value: string): number | null;
12
+ /** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
13
+ export declare function retryAt(headers: Headers, now: number): number;
14
+ /** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
15
+ export declare function readRateLimit(headers: Headers, now: number): RateLimitInfo;
@@ -0,0 +1,66 @@
1
+ const DURATION = /^(?:(\d+(?:\.\d+)?)h)?(?:(\d+(?:\.\d+)?)m(?!s))?(?:(\d+(?:\.\d+)?)s)?(?:(\d+(?:\.\d+)?)ms)?$/;
2
+ /** Parses an OpenAI reset duration (`1s`, `6m0s`, `20ms`, `1h2m`) or a plain number of seconds into milliseconds. */
3
+ export function parseDuration(value) {
4
+ const text = value.trim();
5
+ if (text === "")
6
+ return null;
7
+ if (/^\d+(?:\.\d+)?$/.test(text))
8
+ return Number(text) * 1000;
9
+ const match = DURATION.exec(text);
10
+ if (match === null)
11
+ return null;
12
+ const [, h, m, s, ms] = match;
13
+ if (h === undefined && m === undefined && s === undefined && ms === undefined)
14
+ return null;
15
+ return Number(h ?? 0) * 3_600_000 + Number(m ?? 0) * 60_000 + Number(s ?? 0) * 1000 + Number(ms ?? 0);
16
+ }
17
+ function count(headers, names) {
18
+ for (const name of names) {
19
+ const raw = headers.get(name);
20
+ if (raw === null)
21
+ continue;
22
+ const value = Number(raw.trim());
23
+ if (Number.isFinite(value) && value >= 0)
24
+ return value;
25
+ }
26
+ return null;
27
+ }
28
+ function resetAt(headers, names, now) {
29
+ for (const name of names) {
30
+ const raw = headers.get(name);
31
+ if (raw === null)
32
+ continue;
33
+ const plain = Number(raw.trim());
34
+ // A plain number past 1e9 is an epoch in seconds (x-ratelimit-reset of some providers); smaller is a delta.
35
+ if (Number.isFinite(plain) && plain > 1e9)
36
+ return plain * 1000;
37
+ const ms = parseDuration(raw);
38
+ if (ms !== null)
39
+ return now + ms;
40
+ }
41
+ return 0;
42
+ }
43
+ /** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
44
+ export function retryAt(headers, now) {
45
+ const ms = headers.get("retry-after-ms");
46
+ if (ms !== null && Number.isFinite(Number(ms)))
47
+ return now + Number(ms);
48
+ const after = headers.get("retry-after");
49
+ if (after === null)
50
+ return 0;
51
+ const seconds = Number(after.trim());
52
+ if (Number.isFinite(seconds))
53
+ return now + seconds * 1000;
54
+ const date = Date.parse(after);
55
+ return Number.isNaN(date) ? 0 : date;
56
+ }
57
+ /** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
58
+ export function readRateLimit(headers, now) {
59
+ return {
60
+ remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "x-ratelimit-remaining", "ratelimit-remaining"]),
61
+ requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "x-ratelimit-reset", "ratelimit-reset"], now),
62
+ remainingTokens: count(headers, ["x-ratelimit-remaining-tokens"]),
63
+ tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens"], now),
64
+ retryAt: retryAt(headers, now),
65
+ };
66
+ }
@@ -0,0 +1,9 @@
1
+ export { AiRouterModule } .js";
2
+ export type { AiRouterModuleExtras, AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } .js";
3
+ export type { AiRouterHandlers, AiRouterModuleOptions } .js";
4
+ export { AiRouterService } .js";
5
+ export { AI_ROUTER, getAiRouterServiceToken, getAiRouterToken, InjectAiRouter, InjectAiRouterService } .js";
6
+ export { AiRouterError } .js";
7
+ export type { AttemptFailure } .js";
8
+ export type { HealthRecord, StateStore } .js";
9
+ export type * .js";
package/dist/index.js ADDED
@@ -0,0 +1,4 @@
1
+ export { AiRouterModule } from "./ai-router.module.js";
2
+ export { AiRouterService } from "./ai-router.service.js";
3
+ export { AI_ROUTER, getAiRouterServiceToken, getAiRouterToken, InjectAiRouter, InjectAiRouterService } from "./ai-router.tokens.js";
4
+ export { AiRouterError } from "./errors.js";
package/dist/pool.d.ts ADDED
@@ -0,0 +1,90 @@
1
+ import type { ResolvedPolicy, Settings } .js";
2
+ import type { Emitter } .js";
3
+ import type { RateLimitInfo } .js";
4
+ import type { StateStore } .js";
5
+ import type { Capability, ErrorClass, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } .js";
6
+ /** One routable (provider, key, model) triple and the rank of its model in the call's policy. */
7
+ export interface Lane {
8
+ readonly provider: ProviderConfig;
9
+ readonly key: KeyConfig;
10
+ readonly model: ModelConfig;
11
+ readonly rank: number;
12
+ /** `provider/key/model`, the lane's name in local counters. */
13
+ readonly id: string;
14
+ /** `provider/key`, the pair a retry moves away from. */
15
+ readonly pair: string;
16
+ }
17
+ /** What one call needs from a lane. */
18
+ export interface Need {
19
+ readonly capabilities: readonly Capability[];
20
+ /** An estimate of the prompt's tokens (four characters a token). */
21
+ readonly promptTokens: number;
22
+ /** `max_tokens` or `max_completion_tokens`, 0 when unbounded. */
23
+ readonly outputTokens: number;
24
+ }
25
+ /** What a call has tried so far: lanes, key pairs and providers. */
26
+ export interface Tried {
27
+ readonly lanes: ReadonlySet<string>;
28
+ readonly pairs: ReadonlySet<string>;
29
+ readonly providers: ReadonlySet<string>;
30
+ }
31
+ /** A lane picked for an attempt, with the probe leases it holds while a breaker is half-open. */
32
+ export interface Pick {
33
+ readonly lane: Lane;
34
+ readonly leases: readonly string[];
35
+ }
36
+ /** No lane is usable now: `waitUntil` is when the earliest one may be (Infinity: never within reach). */
37
+ export interface Blocked {
38
+ readonly waitUntil: number;
39
+ /** True when a lane waits only on in-process concurrency or a probe another attempt holds. */
40
+ readonly busy: boolean;
41
+ }
42
+ /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
43
+ export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
44
+ /**
45
+ * The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
46
+ * round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
47
+ */
48
+ export declare class PoolState {
49
+ #private;
50
+ readonly totals: {
51
+ requests: number;
52
+ succeeded: number;
53
+ failed: number;
54
+ attempts: number;
55
+ };
56
+ /** Binds the state to a pool's providers, settings, store and emitter. */
57
+ constructor(providers: readonly ProviderConfig[], settings: Settings, store: StateStore, emitter: Emitter);
58
+ /** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
59
+ lanes(policy: ResolvedPolicy, need: Need): Lane[];
60
+ /**
61
+ * Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
62
+ * call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
63
+ * to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
64
+ * Within the first non-empty tier:
65
+ * the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
66
+ * round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
67
+ * overshoot a cap.
68
+ */
69
+ pick(lanes: readonly Lane[], need: Need, tried: Tried, leaseMs: number): Promise<Pick | Blocked>;
70
+ /** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
71
+ count(lane: Lane): void;
72
+ /** Ends an in-flight attempt, frees its probe leases and wakes every call waiting for capacity. */
73
+ release(lane: Lane, leases: readonly string[]): Promise<void>;
74
+ /** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
75
+ waitForCapacity(ms: number, signal: AbortSignal): Promise<void>;
76
+ /** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
77
+ succeeded(lane: Lane, info: RateLimitInfo | null, latencyMs: number, usage: Usage | undefined): Promise<number | undefined>;
78
+ /**
79
+ * Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
80
+ * key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
81
+ * class touches no shared state.
82
+ */
83
+ failed(lane: Lane, errorClass: ErrorClass, info: RateLimitInfo | null, missingKey: boolean): Promise<void>;
84
+ /** Reads every record into a health snapshot. */
85
+ health(): Promise<HealthSnapshot>;
86
+ /** Returns this process's counters. */
87
+ stats(): StatsSnapshot;
88
+ /** Deletes the records of the target and forgets its local latency and round-robin state. */
89
+ reset(target: ResetTarget): Promise<void>;
90
+ }
package/dist/pool.js ADDED
@@ -0,0 +1,435 @@
1
+ import { DEFAULT_CAPABILITIES, modelMatches } from "./config.js";
2
+ import { EMPTY_RECORD } from "./store.js";
3
+ const FOREVER = Number.MAX_SAFE_INTEGER;
4
+ /** The latency samples a lane needs before its EWMA ranks it: one cold call (a JIT warm-up, a first TLS handshake) never decides. */
5
+ const LATENCY_SAMPLES = 3;
6
+ /** How far a lane latency leaves out drifts toward the best EWMA per pick: a 2.5x-slow lane is retried about every tenth pick. */
7
+ const LATENCY_DRIFT = 0.1;
8
+ function breakerState(record, now) {
9
+ if (record.breaker === "closed")
10
+ return "closed";
11
+ return record.openUntil > now ? "open" : "half-open";
12
+ }
13
+ /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
14
+ export function costOf(model, usage) {
15
+ if (model.cost === undefined || usage === undefined)
16
+ return undefined;
17
+ const completion = usage.completion_tokens ?? 0;
18
+ return (usage.prompt_tokens * model.cost.input + completion * model.cost.output) / 1_000_000;
19
+ }
20
+ /**
21
+ * The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
22
+ * round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
23
+ */
24
+ export class PoolState {
25
+ #providers;
26
+ #settings;
27
+ #store;
28
+ #emitter;
29
+ #inflightKey = new Map();
30
+ #inflightProvider = new Map();
31
+ #ewma = new Map();
32
+ #samples = new Map();
33
+ #roundRobin = new Map();
34
+ #counters = new Map();
35
+ #waiters = new Set();
36
+ totals = { requests: 0, succeeded: 0, failed: 0, attempts: 0 };
37
+ /** Binds the state to a pool's providers, settings, store and emitter. */
38
+ constructor(providers, settings, store, emitter) {
39
+ this.#providers = providers;
40
+ this.#settings = settings;
41
+ this.#store = store;
42
+ this.#emitter = emitter;
43
+ }
44
+ #keyId(provider, alias) {
45
+ return `${this.#settings.namespace}:key:${provider}/${alias}`;
46
+ }
47
+ #providerId(provider) {
48
+ return `${this.#settings.namespace}:provider:${provider}`;
49
+ }
50
+ #laneId(provider, alias, model) {
51
+ return `${this.#settings.namespace}:lane:${provider}/${alias}/${model}`;
52
+ }
53
+ async #read(id) {
54
+ return (await this.#store.get(id)) ?? EMPTY_RECORD;
55
+ }
56
+ /** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
57
+ lanes(policy, need) {
58
+ const lanes = [];
59
+ policy.models.forEach((name, rank) => {
60
+ for (const provider of this.#providers) {
61
+ if (policy.providers !== null && !policy.providers.has(provider.id))
62
+ continue;
63
+ for (const model of provider.models) {
64
+ if (!modelMatches(model, name))
65
+ continue;
66
+ const capabilities = model.capabilities ?? DEFAULT_CAPABILITIES;
67
+ if (!need.capabilities.every((capability) => capabilities.includes(capability)))
68
+ continue;
69
+ if (model.contextWindow !== undefined && need.promptTokens + need.outputTokens > model.contextWindow)
70
+ continue;
71
+ if (policy.maxCost !== null && model.cost !== undefined) {
72
+ const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
73
+ if (worst > policy.maxCost)
74
+ continue;
75
+ }
76
+ for (const key of provider.keys) {
77
+ const id = `${provider.id}/${key.alias}/${model.id}`;
78
+ if (lanes.some((lane) => lane.id === id))
79
+ continue;
80
+ lanes.push({ provider, key, model, rank, id, pair: `${provider.id}/${key.alias}` });
81
+ }
82
+ }
83
+ }
84
+ });
85
+ return lanes;
86
+ }
87
+ async #status(lane, need, now) {
88
+ const [key, provider, headroom] = await Promise.all([
89
+ this.#read(this.#keyId(lane.provider.id, lane.key.alias)),
90
+ this.#read(this.#providerId(lane.provider.id)),
91
+ this.#read(this.#laneId(lane.provider.id, lane.key.alias, lane.model.id)),
92
+ ]);
93
+ const blocks = [];
94
+ if (key.disabledUntil > now)
95
+ blocks.push(key.disabledUntil >= FOREVER ? Infinity : key.disabledUntil);
96
+ const probes = [];
97
+ for (const [record, id] of [[key, this.#keyId(lane.provider.id, lane.key.alias)], [provider, this.#providerId(lane.provider.id)]]) {
98
+ const state = breakerState(record, now);
99
+ if (state === "open")
100
+ blocks.push(record.openUntil);
101
+ if (state === "half-open")
102
+ probes.push(`${id}:probe`);
103
+ }
104
+ if (headroom.cooldownUntil > now)
105
+ blocks.push(headroom.cooldownUntil);
106
+ if (headroom.remainingRequests === 0 && headroom.requestsResetAt > now)
107
+ blocks.push(headroom.requestsResetAt);
108
+ if (headroom.remainingTokens !== null && headroom.remainingTokens < need.promptTokens + need.outputTokens && headroom.tokensResetAt > now) {
109
+ blocks.push(headroom.tokensResetAt);
110
+ }
111
+ if (blocks.length > 0)
112
+ return { usable: false, until: Math.max(...blocks), low: false, probes };
113
+ const low = headroom.remainingRequests !== null && headroom.remainingRequests <= this.#settings.lowHeadroom && headroom.requestsResetAt > now;
114
+ return { usable: true, until: now, low, probes };
115
+ }
116
+ #busy(lane) {
117
+ return ((this.#inflightKey.get(lane.pair) ?? 0) >= (lane.key.maxConcurrency ?? Infinity) ||
118
+ (this.#inflightProvider.get(lane.provider.id) ?? 0) >= (lane.provider.maxConcurrency ?? Infinity));
119
+ }
120
+ #hold(lane, delta) {
121
+ this.#inflightKey.set(lane.pair, Math.max(0, (this.#inflightKey.get(lane.pair) ?? 0) + delta));
122
+ this.#inflightProvider.set(lane.provider.id, Math.max(0, (this.#inflightProvider.get(lane.provider.id) ?? 0) + delta));
123
+ }
124
+ /**
125
+ * Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
126
+ * call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
127
+ * to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
128
+ * Within the first non-empty tier:
129
+ * the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
130
+ * round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
131
+ * overshoot a cap.
132
+ */
133
+ async pick(lanes, need, tried, leaseMs) {
134
+ const now = Date.now();
135
+ const statuses = await Promise.all(lanes.map((lane) => this.#status(lane, need, now)));
136
+ let usable = [];
137
+ let waitUntil = Infinity;
138
+ lanes.forEach((lane, index) => {
139
+ const status = statuses[index];
140
+ if (status === undefined)
141
+ return;
142
+ if (!status.usable) {
143
+ waitUntil = Math.min(waitUntil, status.until);
144
+ return;
145
+ }
146
+ const tier = [
147
+ tried.lanes.has(lane.id) ? 1 : 0,
148
+ lane.rank,
149
+ tried.pairs.has(lane.pair) ? 1 : 0,
150
+ tried.providers.has(lane.provider.id) ? 1 : 0,
151
+ status.low ? 1 : 0,
152
+ ];
153
+ usable.push({ lane, status, tier });
154
+ });
155
+ usable.sort((a, b) => compareTiers(a.tier, b.tier));
156
+ let busy = false;
157
+ for (;;) {
158
+ const free = usable.filter((entry) => !this.#busy(entry.lane));
159
+ if (free.length < usable.length)
160
+ busy = true;
161
+ const first = free[0];
162
+ if (first === undefined)
163
+ break;
164
+ const group = free.filter((entry) => compareTiers(entry.tier, first.tier) === 0);
165
+ const chosen = this.#choose(group.map((entry) => entry.lane));
166
+ const entry = group.find((candidate) => candidate.lane === chosen) ?? first;
167
+ this.#hold(entry.lane, 1);
168
+ const leases = [];
169
+ let held = true;
170
+ for (const lease of entry.status.probes) {
171
+ if (await this.#store.lease(lease, leaseMs))
172
+ leases.push(lease);
173
+ else
174
+ held = false;
175
+ }
176
+ if (held)
177
+ return { lane: entry.lane, leases };
178
+ for (const lease of leases)
179
+ await this.#store.release(lease);
180
+ this.#hold(entry.lane, -1);
181
+ busy = true;
182
+ usable = usable.filter((candidate) => candidate !== entry);
183
+ }
184
+ return { waitUntil: busy ? Math.min(waitUntil, now) : waitUntil, busy };
185
+ }
186
+ #choose(group) {
187
+ const load = (lane) => this.#inflightKey.get(lane.pair) ?? 0;
188
+ const least = Math.min(...group.map(load));
189
+ let pool = group.filter((lane) => load(lane) === least);
190
+ // Latency ranks only lanes with LATENCY_SAMPLES samples, so one cold call never decides; a lane it leaves out drifts
191
+ // toward the best EWMA each time, so it is tried again instead of losing its turn for good on samples it can no longer add.
192
+ const measured = (lane) => ((this.#samples.get(lane.id) ?? 0) >= LATENCY_SAMPLES ? this.#ewma.get(lane.id) : undefined);
193
+ const known = pool.map(measured).filter((value) => value !== undefined);
194
+ if (known.length > 0) {
195
+ const best = Math.min(...known);
196
+ const limit = best * this.#settings.latencyTolerance + this.#settings.latencySlackMs;
197
+ const fast = pool.filter((lane) => (measured(lane) ?? best) <= limit);
198
+ for (const lane of pool)
199
+ if (!fast.includes(lane))
200
+ this.#ewma.set(lane.id, LATENCY_DRIFT * best + (1 - LATENCY_DRIFT) * (this.#ewma.get(lane.id) ?? best));
201
+ pool = fast;
202
+ }
203
+ // Smooth weighted round-robin (nginx): each lane gains its weight, the leader is picked and pays the total.
204
+ let total = 0;
205
+ let best;
206
+ let bestWeight = -Infinity;
207
+ for (const lane of pool) {
208
+ const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1);
209
+ total += weight;
210
+ const current = (this.#roundRobin.get(lane.id) ?? 0) + weight;
211
+ this.#roundRobin.set(lane.id, current);
212
+ if (current > bestWeight) {
213
+ bestWeight = current;
214
+ best = lane;
215
+ }
216
+ }
217
+ if (best !== undefined)
218
+ this.#roundRobin.set(best.id, bestWeight - total);
219
+ return best;
220
+ }
221
+ /** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
222
+ count(lane) {
223
+ this.totals.attempts++;
224
+ this.#count(lane).attempts++;
225
+ }
226
+ /** Ends an in-flight attempt, frees its probe leases and wakes every call waiting for capacity. */
227
+ async release(lane, leases) {
228
+ this.#hold(lane, -1);
229
+ for (const lease of leases)
230
+ await this.#store.release(lease);
231
+ for (const wake of [...this.#waiters])
232
+ wake();
233
+ }
234
+ /** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
235
+ waitForCapacity(ms, signal) {
236
+ return new Promise((resolve, reject) => {
237
+ const done = () => {
238
+ clearTimeout(timer);
239
+ this.#waiters.delete(done);
240
+ signal.removeEventListener("abort", aborted);
241
+ resolve();
242
+ };
243
+ const aborted = () => {
244
+ clearTimeout(timer);
245
+ this.#waiters.delete(done);
246
+ reject(signal.reason);
247
+ };
248
+ const timer = setTimeout(done, Math.max(0, ms));
249
+ if (signal.aborted) {
250
+ aborted();
251
+ return;
252
+ }
253
+ this.#waiters.add(done);
254
+ signal.addEventListener("abort", aborted, { once: true });
255
+ });
256
+ }
257
+ #count(lane) {
258
+ let counters = this.#counters.get(lane.id);
259
+ if (counters === undefined) {
260
+ counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cost: 0 };
261
+ this.#counters.set(lane.id, counters);
262
+ }
263
+ return counters;
264
+ }
265
+ async #headroom(lane, info, cooldownUntil) {
266
+ const id = this.#laneId(lane.provider.id, lane.key.alias, lane.model.id);
267
+ const current = await this.#read(id);
268
+ const known = info !== null && (info.remainingRequests !== null || info.remainingTokens !== null);
269
+ if (!known && cooldownUntil === current.cooldownUntil)
270
+ return;
271
+ await this.#store.set(id, {
272
+ ...current,
273
+ remainingRequests: info?.remainingRequests ?? current.remainingRequests,
274
+ requestsResetAt: info !== null && info.remainingRequests !== null ? info.requestsResetAt : current.requestsResetAt,
275
+ remainingTokens: info?.remainingTokens ?? current.remainingTokens,
276
+ tokensResetAt: info !== null && info.remainingTokens !== null ? info.tokensResetAt : current.tokensResetAt,
277
+ cooldownUntil,
278
+ });
279
+ }
280
+ async #breakerSuccess(id, scope, lane, now) {
281
+ const record = await this.#read(id);
282
+ if (record.breaker === "closed" && record.failures === 0)
283
+ return;
284
+ await this.#store.set(id, { ...record, breaker: "closed", failures: 0, openUntil: 0, cooldownMs: 0 });
285
+ if (record.breaker === "open")
286
+ this.#emitBreaker(scope, lane, breakerState(record, now), "closed");
287
+ }
288
+ async #breakerFailure(id, scope, lane, settings, now, retryAt) {
289
+ const record = await this.#read(id);
290
+ const from = breakerState(record, now);
291
+ const failures = record.failures + 1;
292
+ if (from === "open") {
293
+ await this.#store.set(id, { ...record, failures });
294
+ return;
295
+ }
296
+ const opens = from === "half-open" || failures >= settings.failureThreshold;
297
+ if (!opens) {
298
+ await this.#store.set(id, { ...record, failures });
299
+ return;
300
+ }
301
+ const cooldownMs = from === "half-open" ? Math.min(settings.maxCooldownMs, Math.max(record.cooldownMs, settings.cooldownMs) * 2) : settings.cooldownMs;
302
+ await this.#store.set(id, { ...record, breaker: "open", failures, cooldownMs, openUntil: Math.max(now + cooldownMs, retryAt) });
303
+ this.#emitBreaker(scope, lane, from, "open");
304
+ }
305
+ #emitBreaker(scope, lane, from, to) {
306
+ this.#emitter.emit("breaker", {
307
+ type: "breaker",
308
+ scope,
309
+ provider: lane.provider.id,
310
+ ...(scope === "key" ? { key: lane.key.alias } : {}),
311
+ from,
312
+ to,
313
+ });
314
+ }
315
+ /** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
316
+ async succeeded(lane, info, latencyMs, usage) {
317
+ const now = Date.now();
318
+ await this.#breakerSuccess(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, now);
319
+ await this.#breakerSuccess(this.#providerId(lane.provider.id), "provider", lane, now);
320
+ await this.#headroom(lane, info, 0);
321
+ const previous = this.#ewma.get(lane.id);
322
+ const alpha = this.#settings.ewmaAlpha;
323
+ this.#ewma.set(lane.id, previous === undefined ? latencyMs : alpha * latencyMs + (1 - alpha) * previous);
324
+ this.#samples.set(lane.id, (this.#samples.get(lane.id) ?? 0) + 1);
325
+ const counters = this.#count(lane);
326
+ counters.successes++;
327
+ const cost = costOf(lane.model, usage);
328
+ counters.promptTokens += usage?.prompt_tokens ?? 0;
329
+ counters.completionTokens += usage?.completion_tokens ?? 0;
330
+ counters.cost += cost ?? 0;
331
+ return cost;
332
+ }
333
+ /**
334
+ * Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
335
+ * key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
336
+ * class touches no shared state.
337
+ */
338
+ async failed(lane, errorClass, info, missingKey) {
339
+ const now = Date.now();
340
+ const counters = this.#count(lane);
341
+ counters.failures[errorClass] = (counters.failures[errorClass] ?? 0) + 1;
342
+ const retryAt = info?.retryAt ?? 0;
343
+ if (errorClass === "rate_limit") {
344
+ await this.#headroom(lane, info, retryAt > now ? retryAt : now + this.#settings.rateLimitCooldownMs);
345
+ return;
346
+ }
347
+ if (errorClass === "auth") {
348
+ const id = this.#keyId(lane.provider.id, lane.key.alias);
349
+ const record = await this.#read(id);
350
+ const until = this.#settings.authDisableMs === null ? FOREVER : now + this.#settings.authDisableMs;
351
+ const reason = missingKey ? "missing" : "auth";
352
+ await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason });
353
+ this.#emitter.emit("key-disabled", { type: "key-disabled", provider: lane.provider.id, key: lane.key.alias, reason, until: until >= FOREVER ? null : until });
354
+ return;
355
+ }
356
+ if (errorClass === "transient" || errorClass === "timeout" || errorClass === "network" || errorClass === "stream") {
357
+ await this.#breakerFailure(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, this.#settings.keyBreaker, now, retryAt);
358
+ await this.#breakerFailure(this.#providerId(lane.provider.id), "provider", lane, this.#settings.providerBreaker, now, retryAt);
359
+ if (retryAt > now)
360
+ await this.#headroom(lane, info, retryAt);
361
+ }
362
+ }
363
+ /** Reads every record into a health snapshot. */
364
+ async health() {
365
+ const now = Date.now();
366
+ const providers = await Promise.all(this.#providers.map(async (provider) => {
367
+ const record = await this.#read(this.#providerId(provider.id));
368
+ const keys = await Promise.all(provider.keys.map(async (key) => {
369
+ const keyRecord = await this.#read(this.#keyId(provider.id, key.alias));
370
+ const models = await Promise.all(provider.models.map(async (model) => {
371
+ const lane = await this.#read(this.#laneId(provider.id, key.alias, model.id));
372
+ return {
373
+ model: model.id,
374
+ remainingRequests: lane.remainingRequests,
375
+ remainingTokens: lane.remainingTokens,
376
+ cooldownUntil: lane.cooldownUntil > now ? lane.cooldownUntil : 0,
377
+ latencyEwmaMs: this.#ewma.get(`${provider.id}/${key.alias}/${model.id}`) ?? null,
378
+ };
379
+ }));
380
+ const disabled = keyRecord.disabledUntil > now;
381
+ return {
382
+ alias: key.alias,
383
+ breaker: breakerState(keyRecord, now),
384
+ failures: keyRecord.failures,
385
+ disabled,
386
+ disabledReason: disabled ? keyRecord.disabledReason : null,
387
+ inflight: this.#inflightKey.get(`${provider.id}/${key.alias}`) ?? 0,
388
+ models,
389
+ };
390
+ }));
391
+ return { id: provider.id, breaker: breakerState(record, now), failures: record.failures, inflight: this.#inflightProvider.get(provider.id) ?? 0, keys };
392
+ }));
393
+ return { at: now, providers };
394
+ }
395
+ /** Returns this process's counters. */
396
+ stats() {
397
+ const lanes = [];
398
+ for (const [id, counters] of this.#counters) {
399
+ const [provider = "", key = "", ...model] = id.split("/");
400
+ lanes.push({ provider, key, model: model.join("/"), ...counters, failures: { ...counters.failures } });
401
+ }
402
+ return { ...this.totals, lanes };
403
+ }
404
+ /** Deletes the records of the target and forgets its local latency and round-robin state. */
405
+ async reset(target) {
406
+ const ns = this.#settings.namespace;
407
+ const prefixes = [];
408
+ if (target.provider === undefined)
409
+ prefixes.push(`${ns}:`);
410
+ else if (target.key === undefined)
411
+ prefixes.push(`${ns}:provider:${target.provider}`, `${ns}:key:${target.provider}/`, `${ns}:lane:${target.provider}/`);
412
+ else
413
+ prefixes.push(`${ns}:key:${target.provider}/${target.key}`, `${ns}:lane:${target.provider}/${target.key}/`);
414
+ for (const prefix of prefixes) {
415
+ for (const id of await this.#store.keys(prefix)) {
416
+ if (prefix.endsWith("/") || prefix.endsWith(":") || id === prefix)
417
+ await this.#store.delete(id);
418
+ }
419
+ }
420
+ const local = target.provider === undefined ? "" : target.key === undefined ? `${target.provider}/` : `${target.provider}/${target.key}/`;
421
+ for (const map of [this.#ewma, this.#samples, this.#roundRobin]) {
422
+ for (const id of [...map.keys()])
423
+ if (id.startsWith(local))
424
+ map.delete(id);
425
+ }
426
+ }
427
+ }
428
+ function compareTiers(a, b) {
429
+ for (let index = 0; index < a.length; index++) {
430
+ const diff = (a[index] ?? 0) - (b[index] ?? 0);
431
+ if (diff !== 0)
432
+ return diff;
433
+ }
434
+ return 0;
435
+ }
@@ -0,0 +1,3 @@
1
+ import type { AiRouter, PoolConfig } .js";
2
+ /** Creates a router over a pool of providers, keys and models. The pool is validated now; keys are read per attempt. */
3
+ export declare function createRouter<P extends string = string>(config: PoolConfig<P>): AiRouter<P>;