@starci/ai-router 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,21 @@
1
+ # Changelog
2
+
3
+ ## 0.2.0 - 2026-10-27
4
+
5
+ - Breaking: the package is a NestJS module. The root entry is the Nest API (`AiRouterModule`, `AiRouterService`, the tokens and decorators, the options, config, policy and event types, `AiRouterError`, the `StateStore` type); `createRouter`, `definePool`, `validateConfig`, `MemoryStateStore`, `parseSse`, `parseDuration`, `readRateLimit` and `classifyStatus` are no longer exported, and the router engine is reached only through the service.
6
+ - Breaking: `@nestjs/common` and `@nestjs/core` (11 or 12) and `reflect-metadata` are required peer dependencies.
7
+ - Changed: `createFakeProvider` moved to `@starci/ai-router/testing`, beside `AiRouterTestingModule.register({ fake, ...options })`, which registers the real module with every request sent to the fake and a fixed jitter.
8
+ - Added: `AiRouterModule.register(options)` and `AiRouterModule.registerAsync({ imports, inject, useFactory | useClass | useExisting })`, built with `ConfigurableModuleBuilder`, taking the pool config plus `on` handlers and the `isGlobal` and `name` extras.
9
+ - Added: the `AiRouterService` (`chat`, `stream`, `embeddings`, `health`, `stats`, `on`, `reset`, typed profiles), the `AI_ROUTER` raw-router token, `@InjectAiRouter(name?)`, `@InjectAiRouterService(name?)`, `getAiRouterToken(name?)` and `getAiRouterServiceToken(name?)`; named registrations hold separate pools in one app.
10
+ - Added: on application shutdown the service aborts its calls in flight and releases its handlers.
11
+ - Fixed: latency ranks a lane only after three samples, so one cold call no longer takes a key out of rotation, and a lane latency leaves out drifts back toward the best EWMA and is retried instead of being starved.
12
+
13
+ ## 0.1.0 - 2026-10-27
14
+
15
+ - Added: `createRouter(config)` over a pool of OpenAI-compatible providers, keys (by env var name or resolver, named by alias) and models (aliases, context window, cost, capabilities), with `chat`, `stream`, `embeddings`, `health`, `stats`, `on` and `reset`.
16
+ - Added: model policies per call or per named profile (preferred model, ordered fallbacks, `allowFallback`, `maxAttempts`, attempt timeout, total deadline, budget, provider restriction, required capabilities), and the typed builder `definePool()`.
17
+ - Added: selection by health and headroom (untried lane, model order, untried key and provider, headroom tier), then least in-flight, latency EWMA within a tolerance, and smooth weighted round-robin; per-key and per-provider concurrency caps that queue instead of failing.
18
+ - Added: circuit breakers per key and per provider (closed, open, half-open with one leased probe, doubling cooldown, `retry-after` honoured), error classification, jittered exponential backoff onto a different lane, auth-disabled keys, rate-limit cooldowns from `retry-after` and `x-ratelimit-*` headers, idempotent request ids, cancellation and deadlines.
19
+ - Added: SSE streaming with backpressure; fail-over before the first token, an `afterFirstToken` error after it, an idle timeout between chunks.
20
+ - Added: `attempt`, `request`, `breaker` and `key-disabled` events with no key values and no content unless `telemetry.includeContent`; key values scrubbed from error messages.
21
+ - Added: the pluggable `StateStore` (in-memory `MemoryStateStore` by default) and the deterministic test double `createFakeProvider()`.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 starci183
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,400 @@
1
- # Temporary Holding Version
1
+ # @starci/ai-router
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ A NestJS module that routes OpenAI-compatible calls across a pool. Register many keys, many models and many providers
4
+ once at the app root, inject `AiRouterService`, and call `chat`, `stream` and `embeddings`: the router picks a healthy
5
+ key for each attempt, backs off what is failing, honours rate limits, retries on a different key or provider, falls
6
+ back to another model when you allow it, and tells you what it did without ever exposing a key or a prompt.
7
+
8
+ - A Nest package: `@nestjs/common` and `@nestjs/core` 11 or 12, and `reflect-metadata`, are peer dependencies your app
9
+ already has. No other runtime dependency: `fetch`, `AbortController` and `ReadableStream`. ESM, typed, Node 20 or
10
+ newer.
11
+ - OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
12
+ behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
13
+ - Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
14
+ everywhere else.
15
+
16
+ ## Why
17
+
18
+ One key hits its rate limit, another is revoked, a provider has a bad five minutes, and a model is briefly overloaded.
19
+ Hand-written retry loops get each of those slightly wrong: they retry a 400, hammer a key that answered 401, ignore
20
+ `retry-after`, or retry the same overloaded key. This package owns that loop once, as a Nest capability, with a testing
21
+ module to prove your use of it.
22
+
23
+ ## Install
24
+
25
+ ```sh
26
+ npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata
27
+ ```
28
+
29
+ ## Register
30
+
31
+ The module follows the Nest configurable-module shape: `AiRouterModule.register(options)` or
32
+ `AiRouterModule.registerAsync(...)`, registered once at the app root. The options are the pool
33
+ ([Configuration reference](#configuration-reference)) plus `on` (one handler per [event](#observability)), and the
34
+ extras `isGlobal` (default `false`) and `name` (for [several pools](#named-pools)).
35
+
36
+ ```ts
37
+ import { Module } from "@nestjs/common"
38
+ import { AiRouterModule } from "@starci/ai-router"
39
+
40
+ @Module({
41
+ imports: [
42
+ AiRouterModule.register({
43
+ isGlobal: true,
44
+ providers: [
45
+ {
46
+ id: "openai",
47
+ baseUrl: "https://api.openai.com/v1",
48
+ keys: [
49
+ { alias: "team-a", env: "OPENAI_KEY_A" },
50
+ { alias: "team-b", env: "OPENAI_KEY_B", maxConcurrency: 8 },
51
+ ],
52
+ models: [
53
+ { id: "gpt-4o-mini", aliases: ["fast"], contextWindow: 128_000, cost: { input: 0.15, output: 0.6 } },
54
+ { id: "gpt-4o", aliases: ["smart"], cost: { input: 2.5, output: 10 } },
55
+ { id: "text-embedding-3-small", capabilities: ["embeddings"] },
56
+ ],
57
+ },
58
+ {
59
+ id: "openrouter",
60
+ baseUrl: "https://openrouter.ai/api/v1",
61
+ keys: [{ alias: "or-1", env: "OPENROUTER_KEY" }],
62
+ models: [{ id: "openai/gpt-4o-mini", aliases: ["fast"] }],
63
+ },
64
+ ],
65
+ defaults: { model: "fast", fallbacks: ["smart"], deadlineMs: 60_000 },
66
+ profiles: { strict: { model: "smart", allowFallback: false, maxAttempts: 3 } },
67
+ }),
68
+ ],
69
+ })
70
+ export class AppModule {}
71
+ ```
72
+
73
+ When the options come from another provider, use `registerAsync` with `imports`, `inject` and `useFactory`, or
74
+ `useClass` / `useExisting` with a class whose `createAiRouterOptions()` returns them. With `@nestjs/config`:
75
+
76
+ ```ts
77
+ import { ConfigModule, ConfigService } from "@nestjs/config"
78
+ import { AiRouterModule } from "@starci/ai-router"
79
+
80
+ AiRouterModule.registerAsync({
81
+ isGlobal: true,
82
+ imports: [ConfigModule],
83
+ inject: [ConfigService],
84
+ useFactory: (config: ConfigService) => ({
85
+ providers: [
86
+ {
87
+ id: "openai",
88
+ baseUrl: config.getOrThrow<string>("OPENAI_BASE_URL"),
89
+ keys: [{ alias: "main", resolve: () => config.get<string>("OPENAI_API_KEY") }],
90
+ models: [{ id: "gpt-4o-mini", aliases: ["fast"] }],
91
+ },
92
+ ],
93
+ defaults: { model: "fast" },
94
+ }),
95
+ })
96
+ ```
97
+
98
+ The router is created, and its pool validated, when Nest instantiates the module, so a bad pool fails the bootstrap
99
+ with an `AiRouterError` of class `config` naming the problem.
100
+
101
+ ## Inject and call
102
+
103
+ Each registration provides:
104
+
105
+ | Token | Value | Inject with |
106
+ | --- | --- | --- |
107
+ | `AiRouterService` | the service: `chat`, `stream`, `embeddings`, `health`, `stats`, `on`, `reset`, and `router` | the constructor type, or `@InjectAiRouterService()` |
108
+ | `AI_ROUTER` | the raw `AiRouter` the service delegates to | `@InjectAiRouter()` |
109
+
110
+ ```ts
111
+ import { Injectable } from "@nestjs/common"
112
+ import { AiRouterService } from "@starci/ai-router"
113
+
114
+ @Injectable()
115
+ export class SummaryService {
116
+ constructor(private readonly ai: AiRouterService<"strict">) {}
117
+
118
+ async summarise(text: string): Promise<string> {
119
+ const reply = await this.ai.chat({ messages: [{ role: "user", content: `Summarise: ${text}` }] }, { profile: "strict" })
120
+ return reply.choices[0]?.message.content ?? "" // reply.routing: provider, key alias, model, attempts, cost
121
+ }
122
+
123
+ embed(input: string[]) {
124
+ return this.ai.embeddings({ model: "text-embedding-3-small", input })
125
+ }
126
+ }
127
+ ```
128
+
129
+ The service's type parameter names the profiles a call may select, so `{ profile: "strcit" }` does not compile; type
130
+ the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in step.
131
+
132
+ | member | does |
133
+ |---|---|
134
+ | `chat(request, options?)` | sends a chat-completions request; resolves to the OpenAI completion plus `routing` |
135
+ | `stream(request, options?)` | returns an async iterable of OpenAI chunks, with `requestId`; see [Streaming](#streaming) |
136
+ | `embeddings(request, options?)` | sends an embeddings request; resolves to the OpenAI response plus `routing` |
137
+ | `health()` | reads every breaker, disabled key, headroom and latency EWMA |
138
+ | `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
139
+ | `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled` until shutdown; returns the unsubscribe function |
140
+ | `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
141
+
142
+ `options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
143
+ `x-request-id` and `idempotency-key` on every attempt of the call, so a provider that deduplicates sees one request.
144
+ `headers` adds headers to this call; it can never replace `authorization`.
145
+
146
+ Every failure is an `AiRouterError` with `errorClass`, `retryable`, `status`, `provider`, `key` (alias), `model`,
147
+ `requestId`, `attempts` (every failed attempt: where and why), `afterFirstToken` and `retryAt` (when a 429 said to
148
+ come back).
149
+
150
+ On application shutdown (`app.close()`, or a signal after `enableShutdownHooks()`), the service aborts every call still
151
+ in flight through it (a stream ends with `cancelled`), rejects later calls as `cancelled`, and releases the `on`
152
+ handlers and every handler it subscribed. Calls made on the raw `AI_ROUTER` are the caller's to cancel.
153
+
154
+ ## Named pools
155
+
156
+ Give each registration a `name` to hold several pools in one app; its tokens become `AI_ROUTER:<name>` and
157
+ `AiRouterService:<name>` (`getAiRouterToken(name)`, `getAiRouterServiceToken(name)`), injected by name:
158
+
159
+ ```ts
160
+ imports: [
161
+ AiRouterModule.register({ name: "chat", isGlobal: true, ...chatPool }),
162
+ AiRouterModule.registerAsync({ name: "embed", isGlobal: true, imports: [ConfigModule], inject: [ConfigService], useFactory: embedPool }),
163
+ ]
164
+
165
+ constructor(
166
+ @InjectAiRouterService("chat") private readonly chat: AiRouterService,
167
+ @InjectAiRouterService("embed") private readonly embed: AiRouterService,
168
+ ) {}
169
+ ```
170
+
171
+ A named registration provides no default instance; `@InjectAiRouterService()` without a name needs an unnamed one.
172
+
173
+ ## Configuration reference
174
+
175
+ | field | meaning | default |
176
+ |---|---|---|
177
+ | `providers[].id` | a name, unique, without `/` | required |
178
+ | `providers[].baseUrl` | the URL the OpenAI paths hang from (`.../v1`) | required |
179
+ | `providers[].headers` | extra headers for every request (an `authorization` here is ignored) | none |
180
+ | `providers[].weight` | multiplies the round-robin share of every lane of the provider | 1 |
181
+ | `providers[].maxConcurrency` | in-flight requests across the provider's keys, in this process | unbounded |
182
+ | `providers[].keys[]` | `{ alias, env }` or `{ alias, resolve }`, plus `weight`, `maxConcurrency` | at least one |
183
+ | `providers[].models[]` | `{ id, aliases?, contextWindow?, cost?, capabilities?, weight? }` | at least one |
184
+ | `defaults` | the policy every call starts from | none |
185
+ | `profiles` | named policies a call selects with `{ profile }` | none |
186
+ | `resilience.keyBreaker` | `{ failureThreshold, cooldownMs, maxCooldownMs }` per key | 3, 15000, 300000 |
187
+ | `resilience.providerBreaker` | the same per provider, fed by all its keys | 8, 30000, 300000 |
188
+ | `resilience.backoff` | `{ baseMs, factor, maxMs }` between attempts after a transient failure | 100, 2, 5000 |
189
+ | `resilience.authDisableMs` | how long a 401/403 key stays out; `null` until `reset()` | `null` |
190
+ | `resilience.rateLimitCooldownMs` | a 429's cooldown when it has no `retry-after` | 10000 |
191
+ | `selection.latencyTolerance`, `latencySlackMs` | how far above the best latency EWMA a lane still ties | 1.5, 50 |
192
+ | `selection.ewmaAlpha` | the weight of the newest latency sample | 0.3 |
193
+ | `selection.lowHeadroom` | remaining requests at or below which a lane is used only when nothing else is | 1 |
194
+ | `telemetry.includeContent` | adds request and response bodies to `attempt` events | `false` |
195
+ | `store`, `storeNamespace` | where shared state lives, and its key prefix | in memory, `ai-router` |
196
+ | `fetch` | the `fetch` to use (a proxy agent, a test double) | global `fetch` |
197
+ | `random` | the jitter source | `Math.random` |
198
+
199
+ A model's `id` is the name sent to the provider; a caller may name it by `id` or any alias, and an alias shared by
200
+ several providers makes them interchangeable for that call. `cost` is per one million tokens, `input` and `output`, in
201
+ whatever unit you budget in. `capabilities` defaults to `chat`, `streaming`, `tools` and `json`; an embeddings model
202
+ declares `["embeddings"]`.
203
+
204
+ ## Policies
205
+
206
+ A call's policy is `defaults`, then the selected profile, then `options.policy`, merged field by field (later wins).
207
+ The preferred model is the call policy's `model`, else the profile's, else the request's own `model`, else the
208
+ defaults'.
209
+
210
+ | field | meaning | default |
211
+ |---|---|---|
212
+ | `model` | the preferred model | the request's |
213
+ | `fallbacks` | models tried in order once the preferred one has no usable lane | none |
214
+ | `allowFallback` | `false` pins the call to the preferred model | `true` |
215
+ | `maxAttempts` | attempts across keys, providers and models | 4 |
216
+ | `timeoutMs` | one attempt (a stream: until the first token, then between chunks) | 60000 |
217
+ | `deadlineMs` | the whole call, waits and backoff included | 120000 |
218
+ | `budget.maxCost` | a model whose worst-case cost for this request is higher is no candidate | none |
219
+ | `providers` | restrict the call to these provider ids | all |
220
+ | `require` | capabilities needed beyond those the request implies | none |
221
+
222
+ The request implies its own needs: `tools` needs `tools`, a non-text `response_format` needs `json`, `stream()` needs
223
+ `streaming`, `embeddings()` needs `embeddings`. A model is also no candidate when its `contextWindow` is smaller than
224
+ the estimated prompt (four characters a token) plus `max_tokens`. When nothing is left, the call fails at once with
225
+ class `no_candidate`.
226
+
227
+ ## Selection
228
+
229
+ A candidate ("lane") is one provider, key and model. A lane is out while its key is disabled, its key's or provider's
230
+ breaker is open, a 429 cooldown holds it, its last reported remaining requests are 0 (or remaining tokens are below
231
+ the estimate) until the reported reset, or its key or provider is at its concurrency cap. Among the lanes left, tiers
232
+ decide, in order:
233
+
234
+ 1. a lane this call has not tried yet;
235
+ 2. the policy's model order (preferred, then each fallback);
236
+ 3. a key this call has not tried yet, then a provider it has not tried yet;
237
+ 4. enough headroom (lanes at `lowHeadroom` or below come last).
238
+
239
+ Within the first non-empty tier: the fewest in-flight requests on the key, then latency (lanes whose EWMA is within
240
+ `best * latencyTolerance + latencySlackMs`; a lane never measured stays in), then smooth weighted round-robin by
241
+ `provider.weight * key.weight * model.weight`. Equal keys therefore alternate deterministically rather than randomly.
242
+
243
+ When no lane is usable now but one will be before the deadline (a cooldown, a breaker, a concurrency slot), the call
244
+ waits for it rather than failing. When none will be, it fails at once with the class of the last failure (or
245
+ `no_candidate`) and `retryAt`.
246
+
247
+ ## Resilience
248
+
249
+ | what happened | class | effect | retried |
250
+ |---|---|---|---|
251
+ | 429, or a rate/quota error object | `rate_limit` | the lane (key and model) cools down until `retry-after` / `retry-after-ms`, else `rateLimitCooldownMs` | yes, at once, on another lane |
252
+ | 401, 403, an auth error object | `auth` | the key is disabled (`authDisableMs`, default until `reset()`); `key-disabled` event | yes, at once, on another key |
253
+ | a resolver that returns nothing or throws, an unset env var | `auth` | the key is disabled with reason `missing`; nothing is sent | yes |
254
+ | 408, 409, 425, 5xx | `transient` | counts toward the key's and the provider's breakers; a `retry-after` also cools the lane | yes, after backoff |
255
+ | connection failure | `network` | as transient | yes, after backoff |
256
+ | attempt timeout | `timeout` | as transient | yes, after backoff |
257
+ | broken or malformed stream before the first token | `stream` (or the streamed error's class) | as transient | yes, after backoff |
258
+ | 400, 404, 422, other 4xx, an invalid-request error object | `invalid_request` | none | no |
259
+ | the caller's signal aborted | `cancelled` | the attempt in flight is aborted | no |
260
+ | the call's deadline passed | `deadline` | the attempt in flight is aborted | no |
261
+
262
+ Breakers are closed, open or half-open. A closed breaker opens after `failureThreshold` consecutive transient
263
+ failures (a success resets the count), for `cooldownMs` or until a `retry-after`, whichever is later. Past its
264
+ cooldown it is half-open: exactly one attempt, the probe, holds a lease and goes through while every other call waits.
265
+ A successful probe closes it; a failed one reopens it with the cooldown doubled, up to `maxCooldownMs`. Rate limits,
266
+ invalid requests and cancellations never count against a breaker.
267
+
268
+ Backoff after a transient failure is `min(maxMs, baseMs * factor^(n-1))` scaled by a random factor in `[0.5, 1)`, and
269
+ the next attempt goes to a different lane by the tiers above. Rate and auth failures belong to one key, so the next key
270
+ is tried without waiting.
271
+
272
+ ## Streaming
273
+
274
+ `stream()` sends `stream: true` with `accept: text/event-stream` and parses the server-sent events itself. The body is
275
+ read only when you ask for the next chunk, so a slow consumer holds the connection back instead of buffering the whole
276
+ reply; breaking out of the loop cancels the body and frees the key's slot.
277
+
278
+ - **Before the first token** (the first chunk whose delta carries anything besides `role`), chunks are held back.
279
+ If the stream fails then, by HTTP status, a broken connection, malformed data or a streamed `error` object, the
280
+ attempt fails over like any other and the held chunks are dropped: you never see two role chunks or half an answer.
281
+ - **After the first token**, chunks are yielded as they arrive and a failure is thrown as an `AiRouterError` with
282
+ `afterFirstToken: true`. The router does not retry it, since a second answer cannot be spliced onto the first; retry
283
+ the whole call if your use allows it.
284
+ - `timeoutMs` bounds the wait for the first token and then each gap between chunks; `deadlineMs` bounds the whole
285
+ stream.
286
+ - Stopping early counts as a success of the lane. Usage is read from a final usage chunk when the provider sends one
287
+ (OpenAI does when you pass `stream_options: { include_usage: true }`).
288
+ - Policy errors (an unknown model or profile) throw from `stream()` itself; everything else is thrown from iteration.
289
+
290
+ ## Observability
291
+
292
+ Handlers given in the options as `on` are subscribed when the module is instantiated; `service.on(event, handler)`
293
+ subscribes later. Both are released at application shutdown.
294
+
295
+ ```ts
296
+ AiRouterModule.register({
297
+ ...pool,
298
+ on: {
299
+ attempt: (event) => metrics.histogram("llm.attempt", event.latencyMs, { provider: event.provider, key: event.key, model: event.model, outcome: event.outcome, class: event.errorClass }),
300
+ breaker: (event) => log.warn("breaker", event),
301
+ "key-disabled": (event) => alert(`key ${event.provider}/${event.key} disabled: ${event.reason}`),
302
+ request: (event) => log.info("llm call", event),
303
+ },
304
+ })
305
+ ```
306
+
307
+ | event | when | carries |
308
+ |---|---|---|
309
+ | `attempt` | every attempt | `requestId`, `operation`, `attempt`, `provider`, `key` alias, `model`, `latencyMs`, `outcome`, `errorClass`, `status`, `usage`, `cost` |
310
+ | `request` | every call, once | `requestId`, `operation`, `outcome`, `attempts`, `latencyMs`, the answering lane or the `errorClass` |
311
+ | `breaker` | a breaker changes state | `scope` (`key` or `provider`), `provider`, `key`, `from`, `to` |
312
+ | `key-disabled` | a key is disabled | `provider`, `key`, `reason` (`auth` or `missing`), `until` (null: until `reset()`) |
313
+
314
+ Events never carry message content unless `telemetry.includeContent` is `true`; then `attempt` events add `content`
315
+ with the request body and, for `chat` and `embeddings`, the response. A handler that throws is isolated from the call.
316
+ The module itself logs nothing.
317
+
318
+ ## Sharing state between processes
319
+
320
+ Breakers, disabled keys and headroom live in the `store` option; the default keeps them in memory. Several processes
321
+ (or several registrations) share them through one store: an adapter implements the exported `StateStore` type (`get`,
322
+ `set`, `delete`, `keys(prefix)`, and an atomic `lease(key, ttlMs)` / `release(key)` pair for the half-open probe).
323
+ Methods may return values or promises. Records are plain JSON (`HealthRecord`); a Redis adapter is a `GET`/`SET`, a
324
+ `SCAN` and a `SET NX PX` lease. In-flight counts, latency EWMAs and round-robin positions stay per process. Updates
325
+ are read-modify-write, so two processes failing at the same instant may count one failure between them: breakers are a
326
+ health signal, not a ledger.
327
+
328
+ ## Testing
329
+
330
+ `@starci/ai-router/testing` holds an in-process OpenAI-compatible fake provider and `AiRouterTestingModule`, which
331
+ registers the real module (the same options, `isGlobal` and `name`) with every request sent to the fake and a fixed
332
+ jitter. Script replies per base URL, key, model or path; replies are used in order and the last one repeats.
333
+
334
+ ```ts
335
+ import { Test } from "@nestjs/testing"
336
+ import { AiRouterService } from "@starci/ai-router"
337
+ import { AiRouterTestingModule, createFakeProvider } from "@starci/ai-router/testing"
338
+
339
+ const fake = createFakeProvider()
340
+ fake.on({ key: "sk-test-1" }, { status: 429, headers: { "retry-after": "30" } })
341
+ fake.on({ key: "sk-test-2" }, { latencyMs: 20, text: "hello" })
342
+ fake.on({ model: "m-stream" }, { stream: { chunks: ["a", "b"], failAfter: 1 } })
343
+
344
+ const app = await Test.createTestingModule({
345
+ imports: [
346
+ AiRouterTestingModule.register({
347
+ fake,
348
+ isGlobal: true,
349
+ providers: [{ id: "p", baseUrl: "https://p.test/v1", keys: [{ alias: "one", resolve: () => "sk-test-1" }, { alias: "two", resolve: () => "sk-test-2" }], models: [{ id: "m" }, { id: "m-stream" }] }],
350
+ }),
351
+ ],
352
+ providers: [SummaryService],
353
+ }).compile()
354
+
355
+ const reply = await app.get(AiRouterService).chat({ model: "m", messages: [{ role: "user", content: "hi" }] })
356
+ // reply.routing.key === "two"; fake.calls lists both requests with their key, model, headers and body
357
+ await app.close()
358
+ ```
359
+
360
+ A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `usage`, `embedding`, a raw `body`, and
361
+ `stream: { chunks, delayMs, failAfter, failWith: "network" | "error-event", stallAfter, stallMs, includeUsage }`.
362
+ Everything is deterministic except the latencies you script.
363
+
364
+ ## Security notes
365
+
366
+ - A key is given as an environment variable name or a resolver; the package never needs a literal in configuration.
367
+ It is read at each attempt (so rotation needs no restart) and is sent only in the `authorization` header to its own
368
+ provider's `baseUrl`.
369
+ - Keys are never logged, never put in events, errors, `health()` or `stats()`; every surface names them by alias. An
370
+ error body that echoes the key has it replaced by `[redacted]`.
371
+ - Prompts and responses are not in events unless you set `telemetry.includeContent`.
372
+ - `headers` in a provider or a call cannot override `authorization`.
373
+ - The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
374
+
375
+ ## FAQ
376
+
377
+ **Is this a proxy server?** No, it is a module inside your Nest process. Expose it through a controller if you need one.
378
+
379
+ **Can I use it outside Nest?** No. The package is the Nest module; the engine behind the service is not exported.
380
+
381
+ **Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
382
+ the OpenAI chat-completions and embeddings API.
383
+
384
+ **Why did my call wait instead of failing?** A lane was going to be usable before the deadline (a `retry-after`, a
385
+ breaker cooldown, a concurrency slot). Lower `deadlineMs` to fail sooner.
386
+
387
+ **Why is a key never used again?** It answered 401/403, or its resolver returned nothing, and `authDisableMs` is `null`
388
+ by default. Fix the key and call `service.reset({ provider, key })`, or set `authDisableMs`.
389
+
390
+ **How do I keep calls on one model?** `allowFallback: false` in the policy or a profile. `providers: [...]` keeps them
391
+ on chosen providers.
392
+
393
+ **Why does a stream throw instead of failing over?** It had already yielded tokens; see [Streaming](#streaming).
394
+
395
+ **Are costs exact?** They are usage times your configured price; the budget check before a call uses an estimate (four
396
+ characters a token, plus `max_tokens`).
397
+
398
+ ## License
399
+
400
+ MIT
@@ -0,0 +1,13 @@
1
+ import type { DynamicModule } from "@nestjs/common";
2
+ import type { AiRouterRegisterOptions } .js";
3
+ import type { FakeProvider } .js";
4
+ /** What `AiRouterTestingModule.register` takes: a registration whose requests a fake provider answers. */
5
+ export type AiRouterTestingOptions = Omit<AiRouterRegisterOptions, "fetch"> & {
6
+ /** The provider every request goes to, in process; script it with `fake.on` and read `fake.calls`. */
7
+ readonly fake: FakeProvider;
8
+ };
9
+ /** `AiRouterModule` wired to a fake provider, for a consumer's tests. */
10
+ export declare const AiRouterTestingModule: {
11
+ /** Registers the router like `AiRouterModule.register`, sending through `fake`, with a fixed jitter unless `random` is given. */
12
+ register({ fake, ...options }: AiRouterTestingOptions): DynamicModule;
13
+ };
@@ -0,0 +1,8 @@
1
+ import { AiRouterModule } from "./ai-router.module.js";
2
+ /** `AiRouterModule` wired to a fake provider, for a consumer's tests. */
3
+ export const AiRouterTestingModule = {
4
+ /** Registers the router like `AiRouterModule.register`, sending through `fake`, with a fixed jitter unless `random` is given. */
5
+ register({ fake, ...options }) {
6
+ return AiRouterModule.register({ ...options, fetch: fake.fetch, random: options.random ?? (() => 0.5) });
7
+ },
8
+ };
@@ -0,0 +1,13 @@
1
+ import type { AiRouterModuleOptions } .js";
2
+ /** The extras every registration may set: global visibility, and the name of one of several pools. */
3
+ export interface AiRouterModuleExtras {
4
+ readonly isGlobal: boolean;
5
+ readonly name: string | undefined;
6
+ }
7
+ export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterModuleOptions, "register", "createAiRouterOptions", AiRouterModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: .js").PoolConfig<string> & {
8
+ readonly on?: .js").AiRouterHandlers;
9
+ } & Partial<AiRouterModuleExtras>, ASYNC_OPTIONS_TYPE: import("@nestjs/common").ConfigurableModuleAsyncOptions<AiRouterModuleOptions, "createAiRouterOptions"> & Partial<AiRouterModuleExtras>;
10
+ /** What `AiRouterModule.register` takes: the options, plus `isGlobal` and `name`. */
11
+ export type AiRouterRegisterOptions = typeof OPTIONS_TYPE;
12
+ /** What `AiRouterModule.registerAsync` takes: `imports`, `inject` and one of `useFactory`, `useClass`, `useExisting`, plus `isGlobal` and `name`. */
13
+ export type AiRouterRegisterAsyncOptions = typeof ASYNC_OPTIONS_TYPE;
@@ -0,0 +1,6 @@
1
+ import { ConfigurableModuleBuilder } from "@nestjs/common";
2
+ export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = new ConfigurableModuleBuilder({ moduleName: "AiRouter" })
3
+ .setClassMethodName("register")
4
+ .setFactoryMethodName("createAiRouterOptions")
5
+ .setExtras({ isGlobal: false, name: undefined }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
6
+ .build();
@@ -0,0 +1,10 @@
1
+ import type { DynamicModule } from "@nestjs/common";
2
+ import { ConfigurableModuleClass } .js";
3
+ import type { AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } .js";
4
+ /** A pool router as a Nest capability: registered once at the app root, or once per name for several pools. */
5
+ export declare class AiRouterModule extends ConfigurableModuleClass {
6
+ /** Registers a router from options known now. */
7
+ static register(options: AiRouterRegisterOptions): DynamicModule;
8
+ /** Registers a router from options a factory, class or existing provider supplies. */
9
+ static registerAsync(options: AiRouterRegisterAsyncOptions): DynamicModule;
10
+ }
@@ -0,0 +1,36 @@
1
+ import { Module } from "@nestjs/common";
2
+ import { createRouter } from "./router.js";
3
+ import { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN } from "./ai-router.module-definition.js";
4
+ import { AiRouterService } from "./ai-router.service.js";
5
+ import { getAiRouterServiceToken, getAiRouterToken } from "./ai-router.tokens.js";
6
+ /** Adds the router and its service, built from this registration's options when the module is instantiated. */
7
+ function withRouter(base, name) {
8
+ const routerToken = getAiRouterToken(name);
9
+ const serviceToken = getAiRouterServiceToken(name);
10
+ const providers = [
11
+ {
12
+ provide: routerToken,
13
+ inject: [MODULE_OPTIONS_TOKEN],
14
+ // The router reads only its pool config; the handlers are the service's to subscribe and release.
15
+ useFactory: (options) => createRouter(options),
16
+ },
17
+ {
18
+ provide: serviceToken,
19
+ inject: [routerToken, MODULE_OPTIONS_TOKEN],
20
+ useFactory: (router, options) => new AiRouterService(router, options.on),
21
+ },
22
+ ];
23
+ return { ...base, providers: [...(base.providers ?? []), ...providers], exports: [...(base.exports ?? []), routerToken, serviceToken] };
24
+ }
25
+ /** A pool router as a Nest capability: registered once at the app root, or once per name for several pools. */
26
+ export class AiRouterModule extends ConfigurableModuleClass {
27
+ /** Registers a router from options known now. */
28
+ static register(options) {
29
+ return withRouter(super.register(options), options.name);
30
+ }
31
+ /** Registers a router from options a factory, class or existing provider supplies. */
32
+ static registerAsync(options) {
33
+ return withRouter(super.registerAsync(options), options.name);
34
+ }
35
+ }
36
+ Module({})(AiRouterModule);
@@ -0,0 +1,10 @@
1
+ import type { PoolConfig, RouterEvents } .js";
2
+ /** One handler per router event, subscribed when the module starts and released when the application shuts down. */
3
+ export type AiRouterHandlers = {
4
+ readonly [E in keyof RouterEvents]?: (payload: RouterEvents[E]) => void;
5
+ };
6
+ /** What `AiRouterModule.register` takes: the router's pool config plus optional event handlers. */
7
+ export type AiRouterModuleOptions<Profiles extends string = string> = PoolConfig<Profiles> & {
8
+ /** Event handlers; like the events themselves, they never see a key value. */
9
+ readonly on?: AiRouterHandlers;
10
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,29 @@
1
+ import type { OnApplicationShutdown } from "@nestjs/common";
2
+ import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterEvents, StatsSnapshot } .js";
3
+ import type { AiRouterHandlers } .js";
4
+ /**
5
+ * The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
6
+ * handler it subscribed is released then; calls made on the raw router (`AI_ROUTER`) are the caller's to cancel.
7
+ */
8
+ export declare class AiRouterService<Profiles extends string = string> implements AiRouter<Profiles>, OnApplicationShutdown {
9
+ #private;
10
+ /** The raw router this service delegates to. */
11
+ readonly router: AiRouter<Profiles>;
12
+ constructor(router: AiRouter<Profiles>, handlers?: AiRouterHandlers);
13
+ /** Sends a chat-completions request through the pool. */
14
+ chat(request: ChatCompletionRequest, options?: CallOptions<Profiles>): Promise<ChatCompletion>;
15
+ /** Streams a chat completion; the stream ends `cancelled` when the application shuts down. */
16
+ stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
17
+ /** Sends an embeddings request through the pool. */
18
+ embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
19
+ /** Reads every breaker, disabled key and headroom from the store. */
20
+ health(): Promise<HealthSnapshot>;
21
+ /** Returns this process's counters. */
22
+ stats(): StatsSnapshot;
23
+ /** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
24
+ on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
25
+ /** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
26
+ reset(target?: ResetTarget): Promise<void>;
27
+ /** Aborts every call still in flight through this service and releases its handlers. */
28
+ onApplicationShutdown(): void;
29
+ }