@starci/ai-router 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/pool.js ADDED
@@ -0,0 +1,421 @@
1
+ import { DEFAULT_CAPABILITIES, modelMatches } from "./config.js";
2
+ import { EMPTY_RECORD } from "./store.js";
3
+ const FOREVER = Number.MAX_SAFE_INTEGER;
4
+ function breakerState(record, now) {
5
+ if (record.breaker === "closed")
6
+ return "closed";
7
+ return record.openUntil > now ? "open" : "half-open";
8
+ }
9
+ /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
10
+ export function costOf(model, usage) {
11
+ if (model.cost === undefined || usage === undefined)
12
+ return undefined;
13
+ const completion = usage.completion_tokens ?? 0;
14
+ return (usage.prompt_tokens * model.cost.input + completion * model.cost.output) / 1_000_000;
15
+ }
16
+ /**
17
+ * The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
18
+ * round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
19
+ */
20
+ export class PoolState {
21
+ #providers;
22
+ #settings;
23
+ #store;
24
+ #emitter;
25
+ #inflightKey = new Map();
26
+ #inflightProvider = new Map();
27
+ #ewma = new Map();
28
+ #roundRobin = new Map();
29
+ #counters = new Map();
30
+ #waiters = new Set();
31
+ totals = { requests: 0, succeeded: 0, failed: 0, attempts: 0 };
32
+ /** Binds the state to a pool's providers, settings, store and emitter. */
33
+ constructor(providers, settings, store, emitter) {
34
+ this.#providers = providers;
35
+ this.#settings = settings;
36
+ this.#store = store;
37
+ this.#emitter = emitter;
38
+ }
39
+ #keyId(provider, alias) {
40
+ return `${this.#settings.namespace}:key:${provider}/${alias}`;
41
+ }
42
+ #providerId(provider) {
43
+ return `${this.#settings.namespace}:provider:${provider}`;
44
+ }
45
+ #laneId(provider, alias, model) {
46
+ return `${this.#settings.namespace}:lane:${provider}/${alias}/${model}`;
47
+ }
48
+ async #read(id) {
49
+ return (await this.#store.get(id)) ?? EMPTY_RECORD;
50
+ }
51
+ /** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
52
+ lanes(policy, need) {
53
+ const lanes = [];
54
+ policy.models.forEach((name, rank) => {
55
+ for (const provider of this.#providers) {
56
+ if (policy.providers !== null && !policy.providers.has(provider.id))
57
+ continue;
58
+ for (const model of provider.models) {
59
+ if (!modelMatches(model, name))
60
+ continue;
61
+ const capabilities = model.capabilities ?? DEFAULT_CAPABILITIES;
62
+ if (!need.capabilities.every((capability) => capabilities.includes(capability)))
63
+ continue;
64
+ if (model.contextWindow !== undefined && need.promptTokens + need.outputTokens > model.contextWindow)
65
+ continue;
66
+ if (policy.maxCost !== null && model.cost !== undefined) {
67
+ const worst = (need.promptTokens * model.cost.input + need.outputTokens * model.cost.output) / 1_000_000;
68
+ if (worst > policy.maxCost)
69
+ continue;
70
+ }
71
+ for (const key of provider.keys) {
72
+ const id = `${provider.id}/${key.alias}/${model.id}`;
73
+ if (lanes.some((lane) => lane.id === id))
74
+ continue;
75
+ lanes.push({ provider, key, model, rank, id, pair: `${provider.id}/${key.alias}` });
76
+ }
77
+ }
78
+ }
79
+ });
80
+ return lanes;
81
+ }
82
+ async #status(lane, need, now) {
83
+ const [key, provider, headroom] = await Promise.all([
84
+ this.#read(this.#keyId(lane.provider.id, lane.key.alias)),
85
+ this.#read(this.#providerId(lane.provider.id)),
86
+ this.#read(this.#laneId(lane.provider.id, lane.key.alias, lane.model.id)),
87
+ ]);
88
+ const blocks = [];
89
+ if (key.disabledUntil > now)
90
+ blocks.push(key.disabledUntil >= FOREVER ? Infinity : key.disabledUntil);
91
+ const probes = [];
92
+ for (const [record, id] of [[key, this.#keyId(lane.provider.id, lane.key.alias)], [provider, this.#providerId(lane.provider.id)]]) {
93
+ const state = breakerState(record, now);
94
+ if (state === "open")
95
+ blocks.push(record.openUntil);
96
+ if (state === "half-open")
97
+ probes.push(`${id}:probe`);
98
+ }
99
+ if (headroom.cooldownUntil > now)
100
+ blocks.push(headroom.cooldownUntil);
101
+ if (headroom.remainingRequests === 0 && headroom.requestsResetAt > now)
102
+ blocks.push(headroom.requestsResetAt);
103
+ if (headroom.remainingTokens !== null && headroom.remainingTokens < need.promptTokens + need.outputTokens && headroom.tokensResetAt > now) {
104
+ blocks.push(headroom.tokensResetAt);
105
+ }
106
+ if (blocks.length > 0)
107
+ return { usable: false, until: Math.max(...blocks), low: false, probes };
108
+ const low = headroom.remainingRequests !== null && headroom.remainingRequests <= this.#settings.lowHeadroom && headroom.requestsResetAt > now;
109
+ return { usable: true, until: now, low, probes };
110
+ }
111
+ #busy(lane) {
112
+ return ((this.#inflightKey.get(lane.pair) ?? 0) >= (lane.key.maxConcurrency ?? Infinity) ||
113
+ (this.#inflightProvider.get(lane.provider.id) ?? 0) >= (lane.provider.maxConcurrency ?? Infinity));
114
+ }
115
+ #hold(lane, delta) {
116
+ this.#inflightKey.set(lane.pair, Math.max(0, (this.#inflightKey.get(lane.pair) ?? 0) + delta));
117
+ this.#inflightProvider.set(lane.provider.id, Math.max(0, (this.#inflightProvider.get(lane.provider.id) ?? 0) + delta));
118
+ }
119
+ /**
120
+ * Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
121
+ * call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
122
+ * to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
123
+ * Within the first non-empty tier:
124
+ * the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
125
+ * round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
126
+ * overshoot a cap.
127
+ */
128
+ async pick(lanes, need, tried, leaseMs) {
129
+ const now = Date.now();
130
+ const statuses = await Promise.all(lanes.map((lane) => this.#status(lane, need, now)));
131
+ let usable = [];
132
+ let waitUntil = Infinity;
133
+ lanes.forEach((lane, index) => {
134
+ const status = statuses[index];
135
+ if (status === undefined)
136
+ return;
137
+ if (!status.usable) {
138
+ waitUntil = Math.min(waitUntil, status.until);
139
+ return;
140
+ }
141
+ const tier = [
142
+ tried.lanes.has(lane.id) ? 1 : 0,
143
+ lane.rank,
144
+ tried.pairs.has(lane.pair) ? 1 : 0,
145
+ tried.providers.has(lane.provider.id) ? 1 : 0,
146
+ status.low ? 1 : 0,
147
+ ];
148
+ usable.push({ lane, status, tier });
149
+ });
150
+ usable.sort((a, b) => compareTiers(a.tier, b.tier));
151
+ let busy = false;
152
+ for (;;) {
153
+ const free = usable.filter((entry) => !this.#busy(entry.lane));
154
+ if (free.length < usable.length)
155
+ busy = true;
156
+ const first = free[0];
157
+ if (first === undefined)
158
+ break;
159
+ const group = free.filter((entry) => compareTiers(entry.tier, first.tier) === 0);
160
+ const chosen = this.#choose(group.map((entry) => entry.lane));
161
+ const entry = group.find((candidate) => candidate.lane === chosen) ?? first;
162
+ this.#hold(entry.lane, 1);
163
+ const leases = [];
164
+ let held = true;
165
+ for (const lease of entry.status.probes) {
166
+ if (await this.#store.lease(lease, leaseMs))
167
+ leases.push(lease);
168
+ else
169
+ held = false;
170
+ }
171
+ if (held)
172
+ return { lane: entry.lane, leases };
173
+ for (const lease of leases)
174
+ await this.#store.release(lease);
175
+ this.#hold(entry.lane, -1);
176
+ busy = true;
177
+ usable = usable.filter((candidate) => candidate !== entry);
178
+ }
179
+ return { waitUntil: busy ? Math.min(waitUntil, now) : waitUntil, busy };
180
+ }
181
+ #choose(group) {
182
+ const load = (lane) => this.#inflightKey.get(lane.pair) ?? 0;
183
+ const least = Math.min(...group.map(load));
184
+ let pool = group.filter((lane) => load(lane) === least);
185
+ const known = pool.map((lane) => this.#ewma.get(lane.id)).filter((value) => value !== undefined);
186
+ if (known.length > 0) {
187
+ const limit = Math.min(...known) * this.#settings.latencyTolerance + this.#settings.latencySlackMs;
188
+ pool = pool.filter((lane) => (this.#ewma.get(lane.id) ?? 0) <= limit);
189
+ }
190
+ // Smooth weighted round-robin (nginx): each lane gains its weight, the leader is picked and pays the total.
191
+ let total = 0;
192
+ let best;
193
+ let bestWeight = -Infinity;
194
+ for (const lane of pool) {
195
+ const weight = (lane.provider.weight ?? 1) * (lane.key.weight ?? 1) * (lane.model.weight ?? 1);
196
+ total += weight;
197
+ const current = (this.#roundRobin.get(lane.id) ?? 0) + weight;
198
+ this.#roundRobin.set(lane.id, current);
199
+ if (current > bestWeight) {
200
+ bestWeight = current;
201
+ best = lane;
202
+ }
203
+ }
204
+ if (best !== undefined)
205
+ this.#roundRobin.set(best.id, bestWeight - total);
206
+ return best;
207
+ }
208
+ /** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
209
+ count(lane) {
210
+ this.totals.attempts++;
211
+ this.#count(lane).attempts++;
212
+ }
213
+ /** Ends an in-flight attempt, frees its probe leases and wakes every call waiting for capacity. */
214
+ async release(lane, leases) {
215
+ this.#hold(lane, -1);
216
+ for (const lease of leases)
217
+ await this.#store.release(lease);
218
+ for (const wake of [...this.#waiters])
219
+ wake();
220
+ }
221
+ /** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
222
+ waitForCapacity(ms, signal) {
223
+ return new Promise((resolve, reject) => {
224
+ const done = () => {
225
+ clearTimeout(timer);
226
+ this.#waiters.delete(done);
227
+ signal.removeEventListener("abort", aborted);
228
+ resolve();
229
+ };
230
+ const aborted = () => {
231
+ clearTimeout(timer);
232
+ this.#waiters.delete(done);
233
+ reject(signal.reason);
234
+ };
235
+ const timer = setTimeout(done, Math.max(0, ms));
236
+ if (signal.aborted) {
237
+ aborted();
238
+ return;
239
+ }
240
+ this.#waiters.add(done);
241
+ signal.addEventListener("abort", aborted, { once: true });
242
+ });
243
+ }
244
+ #count(lane) {
245
+ let counters = this.#counters.get(lane.id);
246
+ if (counters === undefined) {
247
+ counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cost: 0 };
248
+ this.#counters.set(lane.id, counters);
249
+ }
250
+ return counters;
251
+ }
252
+ async #headroom(lane, info, cooldownUntil) {
253
+ const id = this.#laneId(lane.provider.id, lane.key.alias, lane.model.id);
254
+ const current = await this.#read(id);
255
+ const known = info !== null && (info.remainingRequests !== null || info.remainingTokens !== null);
256
+ if (!known && cooldownUntil === current.cooldownUntil)
257
+ return;
258
+ await this.#store.set(id, {
259
+ ...current,
260
+ remainingRequests: info?.remainingRequests ?? current.remainingRequests,
261
+ requestsResetAt: info !== null && info.remainingRequests !== null ? info.requestsResetAt : current.requestsResetAt,
262
+ remainingTokens: info?.remainingTokens ?? current.remainingTokens,
263
+ tokensResetAt: info !== null && info.remainingTokens !== null ? info.tokensResetAt : current.tokensResetAt,
264
+ cooldownUntil,
265
+ });
266
+ }
267
+ async #breakerSuccess(id, scope, lane, now) {
268
+ const record = await this.#read(id);
269
+ if (record.breaker === "closed" && record.failures === 0)
270
+ return;
271
+ await this.#store.set(id, { ...record, breaker: "closed", failures: 0, openUntil: 0, cooldownMs: 0 });
272
+ if (record.breaker === "open")
273
+ this.#emitBreaker(scope, lane, breakerState(record, now), "closed");
274
+ }
275
+ async #breakerFailure(id, scope, lane, settings, now, retryAt) {
276
+ const record = await this.#read(id);
277
+ const from = breakerState(record, now);
278
+ const failures = record.failures + 1;
279
+ if (from === "open") {
280
+ await this.#store.set(id, { ...record, failures });
281
+ return;
282
+ }
283
+ const opens = from === "half-open" || failures >= settings.failureThreshold;
284
+ if (!opens) {
285
+ await this.#store.set(id, { ...record, failures });
286
+ return;
287
+ }
288
+ const cooldownMs = from === "half-open" ? Math.min(settings.maxCooldownMs, Math.max(record.cooldownMs, settings.cooldownMs) * 2) : settings.cooldownMs;
289
+ await this.#store.set(id, { ...record, breaker: "open", failures, cooldownMs, openUntil: Math.max(now + cooldownMs, retryAt) });
290
+ this.#emitBreaker(scope, lane, from, "open");
291
+ }
292
+ #emitBreaker(scope, lane, from, to) {
293
+ this.#emitter.emit("breaker", {
294
+ type: "breaker",
295
+ scope,
296
+ provider: lane.provider.id,
297
+ ...(scope === "key" ? { key: lane.key.alias } : {}),
298
+ from,
299
+ to,
300
+ });
301
+ }
302
+ /** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
303
+ async succeeded(lane, info, latencyMs, usage) {
304
+ const now = Date.now();
305
+ await this.#breakerSuccess(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, now);
306
+ await this.#breakerSuccess(this.#providerId(lane.provider.id), "provider", lane, now);
307
+ await this.#headroom(lane, info, 0);
308
+ const previous = this.#ewma.get(lane.id);
309
+ const alpha = this.#settings.ewmaAlpha;
310
+ this.#ewma.set(lane.id, previous === undefined ? latencyMs : alpha * latencyMs + (1 - alpha) * previous);
311
+ const counters = this.#count(lane);
312
+ counters.successes++;
313
+ const cost = costOf(lane.model, usage);
314
+ counters.promptTokens += usage?.prompt_tokens ?? 0;
315
+ counters.completionTokens += usage?.completion_tokens ?? 0;
316
+ counters.cost += cost ?? 0;
317
+ return cost;
318
+ }
319
+ /**
320
+ * Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
321
+ * key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
322
+ * class touches no shared state.
323
+ */
324
+ async failed(lane, errorClass, info, missingKey) {
325
+ const now = Date.now();
326
+ const counters = this.#count(lane);
327
+ counters.failures[errorClass] = (counters.failures[errorClass] ?? 0) + 1;
328
+ const retryAt = info?.retryAt ?? 0;
329
+ if (errorClass === "rate_limit") {
330
+ await this.#headroom(lane, info, retryAt > now ? retryAt : now + this.#settings.rateLimitCooldownMs);
331
+ return;
332
+ }
333
+ if (errorClass === "auth") {
334
+ const id = this.#keyId(lane.provider.id, lane.key.alias);
335
+ const record = await this.#read(id);
336
+ const until = this.#settings.authDisableMs === null ? FOREVER : now + this.#settings.authDisableMs;
337
+ const reason = missingKey ? "missing" : "auth";
338
+ await this.#store.set(id, { ...record, disabledUntil: until, disabledReason: reason });
339
+ this.#emitter.emit("key-disabled", { type: "key-disabled", provider: lane.provider.id, key: lane.key.alias, reason, until: until >= FOREVER ? null : until });
340
+ return;
341
+ }
342
+ if (errorClass === "transient" || errorClass === "timeout" || errorClass === "network" || errorClass === "stream") {
343
+ await this.#breakerFailure(this.#keyId(lane.provider.id, lane.key.alias), "key", lane, this.#settings.keyBreaker, now, retryAt);
344
+ await this.#breakerFailure(this.#providerId(lane.provider.id), "provider", lane, this.#settings.providerBreaker, now, retryAt);
345
+ if (retryAt > now)
346
+ await this.#headroom(lane, info, retryAt);
347
+ }
348
+ }
349
+ /** Reads every record into a health snapshot. */
350
+ async health() {
351
+ const now = Date.now();
352
+ const providers = await Promise.all(this.#providers.map(async (provider) => {
353
+ const record = await this.#read(this.#providerId(provider.id));
354
+ const keys = await Promise.all(provider.keys.map(async (key) => {
355
+ const keyRecord = await this.#read(this.#keyId(provider.id, key.alias));
356
+ const models = await Promise.all(provider.models.map(async (model) => {
357
+ const lane = await this.#read(this.#laneId(provider.id, key.alias, model.id));
358
+ return {
359
+ model: model.id,
360
+ remainingRequests: lane.remainingRequests,
361
+ remainingTokens: lane.remainingTokens,
362
+ cooldownUntil: lane.cooldownUntil > now ? lane.cooldownUntil : 0,
363
+ latencyEwmaMs: this.#ewma.get(`${provider.id}/${key.alias}/${model.id}`) ?? null,
364
+ };
365
+ }));
366
+ const disabled = keyRecord.disabledUntil > now;
367
+ return {
368
+ alias: key.alias,
369
+ breaker: breakerState(keyRecord, now),
370
+ failures: keyRecord.failures,
371
+ disabled,
372
+ disabledReason: disabled ? keyRecord.disabledReason : null,
373
+ inflight: this.#inflightKey.get(`${provider.id}/${key.alias}`) ?? 0,
374
+ models,
375
+ };
376
+ }));
377
+ return { id: provider.id, breaker: breakerState(record, now), failures: record.failures, inflight: this.#inflightProvider.get(provider.id) ?? 0, keys };
378
+ }));
379
+ return { at: now, providers };
380
+ }
381
+ /** Returns this process's counters. */
382
+ stats() {
383
+ const lanes = [];
384
+ for (const [id, counters] of this.#counters) {
385
+ const [provider = "", key = "", ...model] = id.split("/");
386
+ lanes.push({ provider, key, model: model.join("/"), ...counters, failures: { ...counters.failures } });
387
+ }
388
+ return { ...this.totals, lanes };
389
+ }
390
+ /** Deletes the records of the target and forgets its local latency and round-robin state. */
391
+ async reset(target) {
392
+ const ns = this.#settings.namespace;
393
+ const prefixes = [];
394
+ if (target.provider === undefined)
395
+ prefixes.push(`${ns}:`);
396
+ else if (target.key === undefined)
397
+ prefixes.push(`${ns}:provider:${target.provider}`, `${ns}:key:${target.provider}/`, `${ns}:lane:${target.provider}/`);
398
+ else
399
+ prefixes.push(`${ns}:key:${target.provider}/${target.key}`, `${ns}:lane:${target.provider}/${target.key}/`);
400
+ for (const prefix of prefixes) {
401
+ for (const id of await this.#store.keys(prefix)) {
402
+ if (prefix.endsWith("/") || prefix.endsWith(":") || id === prefix)
403
+ await this.#store.delete(id);
404
+ }
405
+ }
406
+ const local = target.provider === undefined ? "" : target.key === undefined ? `${target.provider}/` : `${target.provider}/${target.key}/`;
407
+ for (const map of [this.#ewma, this.#roundRobin]) {
408
+ for (const id of [...map.keys()])
409
+ if (id.startsWith(local))
410
+ map.delete(id);
411
+ }
412
+ }
413
+ }
414
+ function compareTiers(a, b) {
415
+ for (let index = 0; index < a.length; index++) {
416
+ const diff = (a[index] ?? 0) - (b[index] ?? 0);
417
+ if (diff !== 0)
418
+ return diff;
419
+ }
420
+ return 0;
421
+ }
@@ -0,0 +1,3 @@
1
+ import type { AiRouter, PoolConfig } .js";
2
+ /** Creates a router over a pool of providers, keys and models. The pool is validated now; keys are read per attempt. */
3
+ export declare function createRouter<P extends string = string>(config: PoolConfig<P>): AiRouter<P>;