@starci/ai-router 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +168 -81
- package/dist/ai-router-testing.module.d.ts +13 -0
- package/dist/ai-router-testing.module.js +8 -0
- package/dist/ai-router.module-definition.d.ts +13 -0
- package/dist/ai-router.module-definition.js +6 -0
- package/dist/ai-router.module.d.ts +10 -0
- package/dist/ai-router.module.js +36 -0
- package/dist/ai-router.options.d.ts +10 -0
- package/dist/ai-router.options.js +1 -0
- package/dist/ai-router.service.d.ts +29 -0
- package/dist/ai-router.service.js +65 -0
- package/dist/ai-router.tokens.d.ts +11 -0
- package/dist/ai-router.tokens.js +20 -0
- package/dist/config.d.ts +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/events.d.ts +1 -1
- package/dist/fake.d.ts +1 -1
- package/dist/index.d.ts +9 -15
- package/dist/index.js +4 -8
- package/dist/pool.d.ts +5 -5
- package/dist/pool.js +18 -4
- package/dist/router.d.ts +1 -1
- package/dist/store.d.ts +1 -1
- package/dist/testing.d.ts +4 -0
- package/dist/testing.js +2 -0
- package/dist/types.d.ts +4 -3
- package/package.json +19 -4
- package/dist/builder.d.ts +0 -33
- package/dist/builder.js +0 -49
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.1 - 2026-10-28
|
|
4
|
+
|
|
5
|
+
- Fixed: 0.2.0 shipped corrupt declarations: the build rewrote `.ts` specifiers with a `$1` that sh expanded to nothing, so every `dist/*.d.ts` read `export { X } .js";` and a consumer tsc failed. The declaration step now names `.js` with a replacer function and `"`, and reads alike under sh and cmd.
|
|
6
|
+
|
|
7
|
+
## 0.2.0 - 2026-10-27
|
|
8
|
+
|
|
9
|
+
- Breaking: the package is a NestJS module. The root entry is the Nest API (`AiRouterModule`, `AiRouterService`, the tokens and decorators, the options, config, policy and event types, `AiRouterError`, the `StateStore` type); `createRouter`, `definePool`, `validateConfig`, `MemoryStateStore`, `parseSse`, `parseDuration`, `readRateLimit` and `classifyStatus` are no longer exported, and the router engine is reached only through the service.
|
|
10
|
+
- Breaking: `@nestjs/common` and `@nestjs/core` (11 or 12) and `reflect-metadata` are required peer dependencies.
|
|
11
|
+
- Changed: `createFakeProvider` moved to `@starci/ai-router/testing`, beside `AiRouterTestingModule.register({ fake, ...options })`, which registers the real module with every request sent to the fake and a fixed jitter.
|
|
12
|
+
- Added: `AiRouterModule.register(options)` and `AiRouterModule.registerAsync({ imports, inject, useFactory | useClass | useExisting })`, built with `ConfigurableModuleBuilder`, taking the pool config plus `on` handlers and the `isGlobal` and `name` extras.
|
|
13
|
+
- Added: the `AiRouterService` (`chat`, `stream`, `embeddings`, `health`, `stats`, `on`, `reset`, typed profiles), the `AI_ROUTER` raw-router token, `@InjectAiRouter(name?)`, `@InjectAiRouterService(name?)`, `getAiRouterToken(name?)` and `getAiRouterServiceToken(name?)`; named registrations hold separate pools in one app.
|
|
14
|
+
- Added: on application shutdown the service aborts its calls in flight and releases its handlers.
|
|
15
|
+
- Fixed: latency ranks a lane only after three samples, so one cold call no longer takes a key out of rotation, and a lane latency leaves out drifts back toward the best EWMA and is retried instead of being starved.
|
|
16
|
+
|
|
3
17
|
## 0.1.0 - 2026-10-27
|
|
4
18
|
|
|
5
19
|
- Added: `createRouter(config)` over a pool of OpenAI-compatible providers, keys (by env var name or resolver, named by alias) and models (aliases, context window, cost, capabilities), with `chat`, `stream`, `embeddings`, `health`, `stats`, `on` and `reset`.
|
package/README.md
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
# @starci/ai-router
|
|
2
2
|
|
|
3
|
-
A
|
|
4
|
-
|
|
5
|
-
limits, retries on a different key or provider, falls
|
|
6
|
-
did without ever exposing a key or a prompt.
|
|
7
|
-
|
|
8
|
-
-
|
|
3
|
+
A NestJS module that routes OpenAI-compatible calls across a pool. Register many keys, many models and many providers
|
|
4
|
+
once at the app root, inject `AiRouterService`, and call `chat`, `stream` and `embeddings`: the router picks a healthy
|
|
5
|
+
key for each attempt, backs off what is failing, honours rate limits, retries on a different key or provider, falls
|
|
6
|
+
back to another model when you allow it, and tells you what it did without ever exposing a key or a prompt.
|
|
7
|
+
|
|
8
|
+
- A Nest package: `@nestjs/common` and `@nestjs/core` 11 or 12, and `reflect-metadata`, are peer dependencies your app
|
|
9
|
+
already has. No other runtime dependency: `fetch`, `AbortController` and `ReadableStream`. ESM, typed, Node 20 or
|
|
10
|
+
newer.
|
|
9
11
|
- OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
|
|
10
12
|
behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
|
|
11
13
|
- Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
|
|
@@ -15,78 +17,117 @@ did without ever exposing a key or a prompt.
|
|
|
15
17
|
|
|
16
18
|
One key hits its rate limit, another is revoked, a provider has a bad five minutes, and a model is briefly overloaded.
|
|
17
19
|
Hand-written retry loops get each of those slightly wrong: they retry a 400, hammer a key that answered 401, ignore
|
|
18
|
-
`retry-after`, or retry the same overloaded key. This package owns that loop once,
|
|
19
|
-
of it.
|
|
20
|
+
`retry-after`, or retry the same overloaded key. This package owns that loop once, as a Nest capability, with a testing
|
|
21
|
+
module to prove your use of it.
|
|
20
22
|
|
|
21
23
|
## Install
|
|
22
24
|
|
|
23
25
|
```sh
|
|
24
|
-
npm install @starci/ai-router
|
|
26
|
+
npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata
|
|
25
27
|
```
|
|
26
28
|
|
|
27
|
-
##
|
|
29
|
+
## Register
|
|
30
|
+
|
|
31
|
+
The module follows the Nest configurable-module shape: `AiRouterModule.register(options)` or
|
|
32
|
+
`AiRouterModule.registerAsync(...)`, registered once at the app root. The options are the pool
|
|
33
|
+
([Configuration reference](#configuration-reference)) plus `on` (one handler per [event](#observability)), and the
|
|
34
|
+
extras `isGlobal` (default `false`) and `name` (for [several pools](#named-pools)).
|
|
28
35
|
|
|
29
36
|
```ts
|
|
30
|
-
import {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
{
|
|
39
|
-
|
|
37
|
+
import { Module } from "@nestjs/common"
|
|
38
|
+
import { AiRouterModule } from "@starci/ai-router"
|
|
39
|
+
|
|
40
|
+
@Module({
|
|
41
|
+
imports: [
|
|
42
|
+
AiRouterModule.register({
|
|
43
|
+
isGlobal: true,
|
|
44
|
+
providers: [
|
|
45
|
+
{
|
|
46
|
+
id: "openai",
|
|
47
|
+
baseUrl: "https://api.openai.com/v1",
|
|
48
|
+
keys: [
|
|
49
|
+
{ alias: "team-a", env: "OPENAI_KEY_A" },
|
|
50
|
+
{ alias: "team-b", env: "OPENAI_KEY_B", maxConcurrency: 8 },
|
|
51
|
+
],
|
|
52
|
+
models: [
|
|
53
|
+
{ id: "gpt-4o-mini", aliases: ["fast"], contextWindow: 128_000, cost: { input: 0.15, output: 0.6 } },
|
|
54
|
+
{ id: "gpt-4o", aliases: ["smart"], cost: { input: 2.5, output: 10 } },
|
|
55
|
+
{ id: "text-embedding-3-small", capabilities: ["embeddings"] },
|
|
56
|
+
],
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: "openrouter",
|
|
60
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
61
|
+
keys: [{ alias: "or-1", env: "OPENROUTER_KEY" }],
|
|
62
|
+
models: [{ id: "openai/gpt-4o-mini", aliases: ["fast"] }],
|
|
63
|
+
},
|
|
40
64
|
],
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
{ id: "text-embedding-3-small", capabilities: ["embeddings"] },
|
|
45
|
-
],
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
id: "openrouter",
|
|
49
|
-
baseUrl: "https://openrouter.ai/api/v1",
|
|
50
|
-
keys: [{ alias: "or-1", resolve: () => vault.read("openrouter") }],
|
|
51
|
-
models: [{ id: "openai/gpt-4o-mini", aliases: ["fast"] }],
|
|
52
|
-
},
|
|
65
|
+
defaults: { model: "fast", fallbacks: ["smart"], deadlineMs: 60_000 },
|
|
66
|
+
profiles: { strict: { model: "smart", allowFallback: false, maxAttempts: 3 } },
|
|
67
|
+
}),
|
|
53
68
|
],
|
|
54
|
-
defaults: { model: "fast", fallbacks: ["smart"], deadlineMs: 60_000 },
|
|
55
|
-
profiles: { strict: { model: "smart", allowFallback: false, maxAttempts: 3 } },
|
|
56
69
|
})
|
|
70
|
+
export class AppModule {}
|
|
71
|
+
```
|
|
57
72
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
for await (const chunk of router.stream({ model: "fast", messages: [{ role: "user", content: "Count to 5" }] })) {
|
|
62
|
-
process.stdout.write(chunk.choices[0]?.delta.content ?? "")
|
|
63
|
-
}
|
|
73
|
+
When the options come from another provider, use `registerAsync` with `imports`, `inject` and `useFactory`, or
|
|
74
|
+
`useClass` / `useExisting` with a class whose `createAiRouterOptions()` returns them. With `@nestjs/config`:
|
|
64
75
|
|
|
65
|
-
|
|
76
|
+
```ts
|
|
77
|
+
import { ConfigModule, ConfigService } from "@nestjs/config"
|
|
78
|
+
import { AiRouterModule } from "@starci/ai-router"
|
|
79
|
+
|
|
80
|
+
AiRouterModule.registerAsync({
|
|
81
|
+
isGlobal: true,
|
|
82
|
+
imports: [ConfigModule],
|
|
83
|
+
inject: [ConfigService],
|
|
84
|
+
useFactory: (config: ConfigService) => ({
|
|
85
|
+
providers: [
|
|
86
|
+
{
|
|
87
|
+
id: "openai",
|
|
88
|
+
baseUrl: config.getOrThrow<string>("OPENAI_BASE_URL"),
|
|
89
|
+
keys: [{ alias: "main", resolve: () => config.get<string>("OPENAI_API_KEY") }],
|
|
90
|
+
models: [{ id: "gpt-4o-mini", aliases: ["fast"] }],
|
|
91
|
+
},
|
|
92
|
+
],
|
|
93
|
+
defaults: { model: "fast" },
|
|
94
|
+
}),
|
|
95
|
+
})
|
|
66
96
|
```
|
|
67
97
|
|
|
68
|
-
The
|
|
98
|
+
The router is created, and its pool validated, when Nest instantiates the module, so a bad pool fails the bootstrap
|
|
99
|
+
with an `AiRouterError` of class `config` naming the problem.
|
|
100
|
+
|
|
101
|
+
## Inject and call
|
|
102
|
+
|
|
103
|
+
Each registration provides:
|
|
104
|
+
|
|
105
|
+
| Token | Value | Inject with |
|
|
106
|
+
| --- | --- | --- |
|
|
107
|
+
| `AiRouterService` | the service: `chat`, `stream`, `embeddings`, `health`, `stats`, `on`, `reset`, and `router` | the constructor type, or `@InjectAiRouterService()` |
|
|
108
|
+
| `AI_ROUTER` | the raw `AiRouter` the service delegates to | `@InjectAiRouter()` |
|
|
69
109
|
|
|
70
110
|
```ts
|
|
71
|
-
import {
|
|
72
|
-
|
|
73
|
-
const config = definePool()
|
|
74
|
-
.provider("openai", { baseUrl: "https://api.openai.com/v1" })
|
|
75
|
-
.key("team-a", { env: "OPENAI_KEY_A" })
|
|
76
|
-
.model("gpt-4o-mini", { aliases: ["fast"] })
|
|
77
|
-
.model("gpt-4o", { aliases: ["smart"] })
|
|
78
|
-
.profile("strict", { model: "smart", allowFallback: false })
|
|
79
|
-
.options({ defaults: { model: "fast", fallbacks: ["smart"] } })
|
|
80
|
-
.build()
|
|
81
|
-
|
|
82
|
-
const router = createRouter(config)
|
|
83
|
-
await router.chat({ messages: [{ role: "user", content: "Hi" }] }, { profile: "strict" })
|
|
84
|
-
```
|
|
111
|
+
import { Injectable } from "@nestjs/common"
|
|
112
|
+
import { AiRouterService } from "@starci/ai-router"
|
|
85
113
|
|
|
86
|
-
|
|
114
|
+
@Injectable()
|
|
115
|
+
export class SummaryService {
|
|
116
|
+
constructor(private readonly ai: AiRouterService<"strict">) {}
|
|
87
117
|
|
|
88
|
-
|
|
89
|
-
|
|
118
|
+
async summarise(text: string): Promise<string> {
|
|
119
|
+
const reply = await this.ai.chat({ messages: [{ role: "user", content: `Summarise: ${text}` }] }, { profile: "strict" })
|
|
120
|
+
return reply.choices[0]?.message.content ?? "" // reply.routing: provider, key alias, model, attempts, cost
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
embed(input: string[]) {
|
|
124
|
+
return this.ai.embeddings({ model: "text-embedding-3-small", input })
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The service's type parameter names the profiles a call may select, so `{ profile: "strcit" }` does not compile; type
|
|
130
|
+
the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in step.
|
|
90
131
|
|
|
91
132
|
| member | does |
|
|
92
133
|
|---|---|
|
|
@@ -95,7 +136,7 @@ returns:
|
|
|
95
136
|
| `embeddings(request, options?)` | sends an embeddings request; resolves to the OpenAI response plus `routing` |
|
|
96
137
|
| `health()` | reads every breaker, disabled key, headroom and latency EWMA |
|
|
97
138
|
| `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
|
|
98
|
-
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled
|
|
139
|
+
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled` until shutdown; returns the unsubscribe function |
|
|
99
140
|
| `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
|
|
100
141
|
|
|
101
142
|
`options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
|
|
@@ -106,6 +147,29 @@ Every failure is an `AiRouterError` with `errorClass`, `retryable`, `status`, `p
|
|
|
106
147
|
`requestId`, `attempts` (every failed attempt: where and why), `afterFirstToken` and `retryAt` (when a 429 said to
|
|
107
148
|
come back).
|
|
108
149
|
|
|
150
|
+
On application shutdown (`app.close()`, or a signal after `enableShutdownHooks()`), the service aborts every call still
|
|
151
|
+
in flight through it (a stream ends with `cancelled`), rejects later calls as `cancelled`, and releases the `on`
|
|
152
|
+
handlers and every handler it subscribed. Calls made on the raw `AI_ROUTER` are the caller's to cancel.
|
|
153
|
+
|
|
154
|
+
## Named pools
|
|
155
|
+
|
|
156
|
+
Give each registration a `name` to hold several pools in one app; its tokens become `AI_ROUTER:<name>` and
|
|
157
|
+
`AiRouterService:<name>` (`getAiRouterToken(name)`, `getAiRouterServiceToken(name)`), injected by name:
|
|
158
|
+
|
|
159
|
+
```ts
|
|
160
|
+
imports: [
|
|
161
|
+
AiRouterModule.register({ name: "chat", isGlobal: true, ...chatPool }),
|
|
162
|
+
AiRouterModule.registerAsync({ name: "embed", isGlobal: true, imports: [ConfigModule], inject: [ConfigService], useFactory: embedPool }),
|
|
163
|
+
]
|
|
164
|
+
|
|
165
|
+
constructor(
|
|
166
|
+
@InjectAiRouterService("chat") private readonly chat: AiRouterService,
|
|
167
|
+
@InjectAiRouterService("embed") private readonly embed: AiRouterService,
|
|
168
|
+
) {}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
A named registration provides no default instance; `@InjectAiRouterService()` without a name needs an unnamed one.
|
|
172
|
+
|
|
109
173
|
## Configuration reference
|
|
110
174
|
|
|
111
175
|
| field | meaning | default |
|
|
@@ -225,11 +289,19 @@ reply; breaking out of the loop cancels the body and frees the key's slot.
|
|
|
225
289
|
|
|
226
290
|
## Observability
|
|
227
291
|
|
|
292
|
+
Handlers given in the options as `on` are subscribed when the module is instantiated; `service.on(event, handler)`
|
|
293
|
+
subscribes later. Both are released at application shutdown.
|
|
294
|
+
|
|
228
295
|
```ts
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
296
|
+
AiRouterModule.register({
|
|
297
|
+
...pool,
|
|
298
|
+
on: {
|
|
299
|
+
attempt: (event) => metrics.histogram("llm.attempt", event.latencyMs, { provider: event.provider, key: event.key, model: event.model, outcome: event.outcome, class: event.errorClass }),
|
|
300
|
+
breaker: (event) => log.warn("breaker", event),
|
|
301
|
+
"key-disabled": (event) => alert(`key ${event.provider}/${event.key} disabled: ${event.reason}`),
|
|
302
|
+
request: (event) => log.info("llm call", event),
|
|
303
|
+
},
|
|
304
|
+
})
|
|
233
305
|
```
|
|
234
306
|
|
|
235
307
|
| event | when | carries |
|
|
@@ -241,35 +313,48 @@ router.on("request", (event) => log.info("llm call", event))
|
|
|
241
313
|
|
|
242
314
|
Events never carry message content unless `telemetry.includeContent` is `true`; then `attempt` events add `content`
|
|
243
315
|
with the request body and, for `chat` and `embeddings`, the response. A handler that throws is isolated from the call.
|
|
316
|
+
The module itself logs nothing.
|
|
244
317
|
|
|
245
318
|
## Sharing state between processes
|
|
246
319
|
|
|
247
|
-
Breakers, disabled keys and headroom live in
|
|
248
|
-
several
|
|
249
|
-
atomic `lease(key, ttlMs)` / `release(key)` pair for the half-open probe.
|
|
250
|
-
Records are plain JSON (`HealthRecord`); a Redis adapter is a `GET`/`SET`, a
|
|
251
|
-
counts, latency EWMAs and round-robin positions stay per process. Updates
|
|
252
|
-
failing at the same instant may count one failure between them: breakers are a
|
|
320
|
+
Breakers, disabled keys and headroom live in the `store` option; the default keeps them in memory. Several processes
|
|
321
|
+
(or several registrations) share them through one store: an adapter implements the exported `StateStore` type (`get`,
|
|
322
|
+
`set`, `delete`, `keys(prefix)`, and an atomic `lease(key, ttlMs)` / `release(key)` pair for the half-open probe).
|
|
323
|
+
Methods may return values or promises. Records are plain JSON (`HealthRecord`); a Redis adapter is a `GET`/`SET`, a
|
|
324
|
+
`SCAN` and a `SET NX PX` lease. In-flight counts, latency EWMAs and round-robin positions stay per process. Updates
|
|
325
|
+
are read-modify-write, so two processes failing at the same instant may count one failure between them: breakers are a
|
|
326
|
+
health signal, not a ledger.
|
|
253
327
|
|
|
254
|
-
## Testing
|
|
328
|
+
## Testing
|
|
255
329
|
|
|
256
|
-
`
|
|
257
|
-
|
|
330
|
+
`@starci/ai-router/testing` holds an in-process OpenAI-compatible fake provider and `AiRouterTestingModule`, which
|
|
331
|
+
registers the real module (the same options, `isGlobal` and `name`) with every request sent to the fake and a fixed
|
|
332
|
+
jitter. Script replies per base URL, key, model or path; replies are used in order and the last one repeats.
|
|
258
333
|
|
|
259
334
|
```ts
|
|
260
|
-
import {
|
|
335
|
+
import { Test } from "@nestjs/testing"
|
|
336
|
+
import { AiRouterService } from "@starci/ai-router"
|
|
337
|
+
import { AiRouterTestingModule, createFakeProvider } from "@starci/ai-router/testing"
|
|
261
338
|
|
|
262
339
|
const fake = createFakeProvider()
|
|
263
340
|
fake.on({ key: "sk-test-1" }, { status: 429, headers: { "retry-after": "30" } })
|
|
264
341
|
fake.on({ key: "sk-test-2" }, { latencyMs: 20, text: "hello" })
|
|
265
342
|
fake.on({ model: "m-stream" }, { stream: { chunks: ["a", "b"], failAfter: 1 } })
|
|
266
343
|
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
344
|
+
const app = await Test.createTestingModule({
|
|
345
|
+
imports: [
|
|
346
|
+
AiRouterTestingModule.register({
|
|
347
|
+
fake,
|
|
348
|
+
isGlobal: true,
|
|
349
|
+
providers: [{ id: "p", baseUrl: "https://p.test/v1", keys: [{ alias: "one", resolve: () => "sk-test-1" }, { alias: "two", resolve: () => "sk-test-2" }], models: [{ id: "m" }, { id: "m-stream" }] }],
|
|
350
|
+
}),
|
|
351
|
+
],
|
|
352
|
+
providers: [SummaryService],
|
|
353
|
+
}).compile()
|
|
354
|
+
|
|
355
|
+
const reply = await app.get(AiRouterService).chat({ model: "m", messages: [{ role: "user", content: "hi" }] })
|
|
272
356
|
// reply.routing.key === "two"; fake.calls lists both requests with their key, model, headers and body
|
|
357
|
+
await app.close()
|
|
273
358
|
```
|
|
274
359
|
|
|
275
360
|
A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `usage`, `embedding`, a raw `body`, and
|
|
@@ -289,7 +374,9 @@ Everything is deterministic except the latencies you script.
|
|
|
289
374
|
|
|
290
375
|
## FAQ
|
|
291
376
|
|
|
292
|
-
**Is this a proxy server?** No, it is a
|
|
377
|
+
**Is this a proxy server?** No, it is a module inside your Nest process. Expose it through a controller if you need one.
|
|
378
|
+
|
|
379
|
+
**Can I use it outside Nest?** No. The package is the Nest module; the engine behind the service is not exported.
|
|
293
380
|
|
|
294
381
|
**Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
|
|
295
382
|
the OpenAI chat-completions and embeddings API.
|
|
@@ -298,7 +385,7 @@ the OpenAI chat-completions and embeddings API.
|
|
|
298
385
|
breaker cooldown, a concurrency slot). Lower `deadlineMs` to fail sooner.
|
|
299
386
|
|
|
300
387
|
**Why is a key never used again?** It answered 401/403, or its resolver returned nothing, and `authDisableMs` is `null`
|
|
301
|
-
by default. Fix the key and call `
|
|
388
|
+
by default. Fix the key and call `service.reset({ provider, key })`, or set `authDisableMs`.
|
|
302
389
|
|
|
303
390
|
**How do I keep calls on one model?** `allowFallback: false` in the policy or a profile. `providers: [...]` keeps them
|
|
304
391
|
on chosen providers.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { DynamicModule } from "@nestjs/common";
|
|
2
|
+
import type { AiRouterRegisterOptions } from "./ai-router.module-definition.js";
|
|
3
|
+
import type { FakeProvider } from "./fake.js";
|
|
4
|
+
/** What `AiRouterTestingModule.register` takes: a registration whose requests a fake provider answers. */
|
|
5
|
+
export type AiRouterTestingOptions = Omit<AiRouterRegisterOptions, "fetch"> & {
|
|
6
|
+
/** The provider every request goes to, in process; script it with `fake.on` and read `fake.calls`. */
|
|
7
|
+
readonly fake: FakeProvider;
|
|
8
|
+
};
|
|
9
|
+
/** `AiRouterModule` wired to a fake provider, for a consumer's tests. */
|
|
10
|
+
export declare const AiRouterTestingModule: {
|
|
11
|
+
/** Registers the router like `AiRouterModule.register`, sending through `fake`, with a fixed jitter unless `random` is given. */
|
|
12
|
+
register({ fake, ...options }: AiRouterTestingOptions): DynamicModule;
|
|
13
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { AiRouterModule } from "./ai-router.module.js";
|
|
2
|
+
/** `AiRouterModule` wired to a fake provider, for a consumer's tests. */
|
|
3
|
+
export const AiRouterTestingModule = {
|
|
4
|
+
/** Registers the router like `AiRouterModule.register`, sending through `fake`, with a fixed jitter unless `random` is given. */
|
|
5
|
+
register({ fake, ...options }) {
|
|
6
|
+
return AiRouterModule.register({ ...options, fetch: fake.fetch, random: options.random ?? (() => 0.5) });
|
|
7
|
+
},
|
|
8
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AiRouterModuleOptions } from "./ai-router.options.js";
|
|
2
|
+
/** The extras every registration may set: global visibility, and the name of one of several pools. */
|
|
3
|
+
export interface AiRouterModuleExtras {
|
|
4
|
+
readonly isGlobal: boolean;
|
|
5
|
+
readonly name: string | undefined;
|
|
6
|
+
}
|
|
7
|
+
export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterModuleOptions, "register", "createAiRouterOptions", AiRouterModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: import("./types.js").PoolConfig<string> & {
|
|
8
|
+
readonly on?: import("./ai-router.options.js").AiRouterHandlers;
|
|
9
|
+
} & Partial<AiRouterModuleExtras>, ASYNC_OPTIONS_TYPE: import("@nestjs/common").ConfigurableModuleAsyncOptions<AiRouterModuleOptions, "createAiRouterOptions"> & Partial<AiRouterModuleExtras>;
|
|
10
|
+
/** What `AiRouterModule.register` takes: the options, plus `isGlobal` and `name`. */
|
|
11
|
+
export type AiRouterRegisterOptions = typeof OPTIONS_TYPE;
|
|
12
|
+
/** What `AiRouterModule.registerAsync` takes: `imports`, `inject` and one of `useFactory`, `useClass`, `useExisting`, plus `isGlobal` and `name`. */
|
|
13
|
+
export type AiRouterRegisterAsyncOptions = typeof ASYNC_OPTIONS_TYPE;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { ConfigurableModuleBuilder } from "@nestjs/common";
|
|
2
|
+
export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = new ConfigurableModuleBuilder({ moduleName: "AiRouter" })
|
|
3
|
+
.setClassMethodName("register")
|
|
4
|
+
.setFactoryMethodName("createAiRouterOptions")
|
|
5
|
+
.setExtras({ isGlobal: false, name: undefined }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
|
|
6
|
+
.build();
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DynamicModule } from "@nestjs/common";
|
|
2
|
+
import { ConfigurableModuleClass } from "./ai-router.module-definition.js";
|
|
3
|
+
import type { AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } from "./ai-router.module-definition.js";
|
|
4
|
+
/** A pool router as a Nest capability: registered once at the app root, or once per name for several pools. */
|
|
5
|
+
export declare class AiRouterModule extends ConfigurableModuleClass {
|
|
6
|
+
/** Registers a router from options known now. */
|
|
7
|
+
static register(options: AiRouterRegisterOptions): DynamicModule;
|
|
8
|
+
/** Registers a router from options a factory, class or existing provider supplies. */
|
|
9
|
+
static registerAsync(options: AiRouterRegisterAsyncOptions): DynamicModule;
|
|
10
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Module } from "@nestjs/common";
|
|
2
|
+
import { createRouter } from "./router.js";
|
|
3
|
+
import { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN } from "./ai-router.module-definition.js";
|
|
4
|
+
import { AiRouterService } from "./ai-router.service.js";
|
|
5
|
+
import { getAiRouterServiceToken, getAiRouterToken } from "./ai-router.tokens.js";
|
|
6
|
+
/** Adds the router and its service, built from this registration's options when the module is instantiated. */
|
|
7
|
+
function withRouter(base, name) {
|
|
8
|
+
const routerToken = getAiRouterToken(name);
|
|
9
|
+
const serviceToken = getAiRouterServiceToken(name);
|
|
10
|
+
const providers = [
|
|
11
|
+
{
|
|
12
|
+
provide: routerToken,
|
|
13
|
+
inject: [MODULE_OPTIONS_TOKEN],
|
|
14
|
+
// The router reads only its pool config; the handlers are the service's to subscribe and release.
|
|
15
|
+
useFactory: (options) => createRouter(options),
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
provide: serviceToken,
|
|
19
|
+
inject: [routerToken, MODULE_OPTIONS_TOKEN],
|
|
20
|
+
useFactory: (router, options) => new AiRouterService(router, options.on),
|
|
21
|
+
},
|
|
22
|
+
];
|
|
23
|
+
return { ...base, providers: [...(base.providers ?? []), ...providers], exports: [...(base.exports ?? []), routerToken, serviceToken] };
|
|
24
|
+
}
|
|
25
|
+
/** A pool router as a Nest capability: registered once at the app root, or once per name for several pools. */
|
|
26
|
+
export class AiRouterModule extends ConfigurableModuleClass {
|
|
27
|
+
/** Registers a router from options known now. */
|
|
28
|
+
static register(options) {
|
|
29
|
+
return withRouter(super.register(options), options.name);
|
|
30
|
+
}
|
|
31
|
+
/** Registers a router from options a factory, class or existing provider supplies. */
|
|
32
|
+
static registerAsync(options) {
|
|
33
|
+
return withRouter(super.registerAsync(options), options.name);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
Module({})(AiRouterModule);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { PoolConfig, RouterEvents } from "./types.js";
|
|
2
|
+
/** One handler per router event, subscribed when the module starts and released when the application shuts down. */
|
|
3
|
+
export type AiRouterHandlers = {
|
|
4
|
+
readonly [E in keyof RouterEvents]?: (payload: RouterEvents[E]) => void;
|
|
5
|
+
};
|
|
6
|
+
/** What `AiRouterModule.register` takes: the router's pool config plus optional event handlers. */
|
|
7
|
+
export type AiRouterModuleOptions<Profiles extends string = string> = PoolConfig<Profiles> & {
|
|
8
|
+
/** Event handlers; like the events themselves, they never see a key value. */
|
|
9
|
+
readonly on?: AiRouterHandlers;
|
|
10
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { OnApplicationShutdown } from "@nestjs/common";
|
|
2
|
+
import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterEvents, StatsSnapshot } from "./types.js";
|
|
3
|
+
import type { AiRouterHandlers } from "./ai-router.options.js";
|
|
4
|
+
/**
|
|
5
|
+
* The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
|
|
6
|
+
* handler it subscribed is released then; calls made on the raw router (`AI_ROUTER`) are the caller's to cancel.
|
|
7
|
+
*/
|
|
8
|
+
export declare class AiRouterService<Profiles extends string = string> implements AiRouter<Profiles>, OnApplicationShutdown {
|
|
9
|
+
#private;
|
|
10
|
+
/** The raw router this service delegates to. */
|
|
11
|
+
readonly router: AiRouter<Profiles>;
|
|
12
|
+
constructor(router: AiRouter<Profiles>, handlers?: AiRouterHandlers);
|
|
13
|
+
/** Sends a chat-completions request through the pool. */
|
|
14
|
+
chat(request: ChatCompletionRequest, options?: CallOptions<Profiles>): Promise<ChatCompletion>;
|
|
15
|
+
/** Streams a chat completion; the stream ends `cancelled` when the application shuts down. */
|
|
16
|
+
stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
|
|
17
|
+
/** Sends an embeddings request through the pool. */
|
|
18
|
+
embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
|
|
19
|
+
/** Reads every breaker, disabled key and headroom from the store. */
|
|
20
|
+
health(): Promise<HealthSnapshot>;
|
|
21
|
+
/** Returns this process's counters. */
|
|
22
|
+
stats(): StatsSnapshot;
|
|
23
|
+
/** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
|
|
24
|
+
on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
|
|
25
|
+
/** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
|
|
26
|
+
reset(target?: ResetTarget): Promise<void>;
|
|
27
|
+
/** Aborts every call still in flight through this service and releases its handlers. */
|
|
28
|
+
onApplicationShutdown(): void;
|
|
29
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
|
|
3
|
+
* handler it subscribed is released then; calls made on the raw router (`AI_ROUTER`) are the caller's to cancel.
|
|
4
|
+
*/
|
|
5
|
+
export class AiRouterService {
|
|
6
|
+
/** The raw router this service delegates to. */
|
|
7
|
+
router;
|
|
8
|
+
#shutdown = new AbortController();
|
|
9
|
+
#releases = new Set();
|
|
10
|
+
constructor(router, handlers = {}) {
|
|
11
|
+
this.router = router;
|
|
12
|
+
if (handlers.attempt !== undefined)
|
|
13
|
+
this.on("attempt", handlers.attempt);
|
|
14
|
+
if (handlers.breaker !== undefined)
|
|
15
|
+
this.on("breaker", handlers.breaker);
|
|
16
|
+
if (handlers["key-disabled"] !== undefined)
|
|
17
|
+
this.on("key-disabled", handlers["key-disabled"]);
|
|
18
|
+
if (handlers.request !== undefined)
|
|
19
|
+
this.on("request", handlers.request);
|
|
20
|
+
}
|
|
21
|
+
#options(options) {
|
|
22
|
+
const signal = options.signal === undefined ? this.#shutdown.signal : AbortSignal.any([this.#shutdown.signal, options.signal]);
|
|
23
|
+
return { ...options, signal };
|
|
24
|
+
}
|
|
25
|
+
/** Sends a chat-completions request through the pool. */
|
|
26
|
+
chat(request, options = {}) {
|
|
27
|
+
return this.router.chat(request, this.#options(options));
|
|
28
|
+
}
|
|
29
|
+
/** Streams a chat completion; the stream ends `cancelled` when the application shuts down. */
|
|
30
|
+
stream(request, options = {}) {
|
|
31
|
+
return this.router.stream(request, this.#options(options));
|
|
32
|
+
}
|
|
33
|
+
/** Sends an embeddings request through the pool. */
|
|
34
|
+
embeddings(request, options = {}) {
|
|
35
|
+
return this.router.embeddings(request, this.#options(options));
|
|
36
|
+
}
|
|
37
|
+
/** Reads every breaker, disabled key and headroom from the store. */
|
|
38
|
+
health() {
|
|
39
|
+
return this.router.health();
|
|
40
|
+
}
|
|
41
|
+
/** Returns this process's counters. */
|
|
42
|
+
stats() {
|
|
43
|
+
return this.router.stats();
|
|
44
|
+
}
|
|
45
|
+
/** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
|
|
46
|
+
on(event, handler) {
|
|
47
|
+
const unsubscribe = this.router.on(event, handler);
|
|
48
|
+
const release = () => {
|
|
49
|
+
unsubscribe();
|
|
50
|
+
this.#releases.delete(release);
|
|
51
|
+
};
|
|
52
|
+
this.#releases.add(release);
|
|
53
|
+
return release;
|
|
54
|
+
}
|
|
55
|
+
/** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
|
|
56
|
+
reset(target) {
|
|
57
|
+
return this.router.reset(target);
|
|
58
|
+
}
|
|
59
|
+
/** Aborts every call still in flight through this service and releases its handlers. */
|
|
60
|
+
onApplicationShutdown() {
|
|
61
|
+
this.#shutdown.abort();
|
|
62
|
+
for (const release of [...this.#releases])
|
|
63
|
+
release();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { AiRouterService } from "./ai-router.service.js";
|
|
2
|
+
/** The injection token of the default instance's raw router (`AiRouter`). */
|
|
3
|
+
export declare const AI_ROUTER = "AI_ROUTER";
|
|
4
|
+
/** The token of a raw router: `AI_ROUTER` for the default instance, `AI_ROUTER:<name>` for a named one. */
|
|
5
|
+
export declare function getAiRouterToken(name?: string): string;
|
|
6
|
+
/** The token of a service: the `AiRouterService` class for the default instance, `AiRouterService:<name>` for a named one. */
|
|
7
|
+
export declare function getAiRouterServiceToken(name?: string): typeof AiRouterService | string;
|
|
8
|
+
/** Injects the raw router of the default instance, or of the instance registered under `name`. */
|
|
9
|
+
export declare function InjectAiRouter(name?: string): PropertyDecorator & ParameterDecorator;
|
|
10
|
+
/** Injects the `AiRouterService` of the default instance, or of the instance registered under `name`. */
|
|
11
|
+
export declare function InjectAiRouterService(name?: string): PropertyDecorator & ParameterDecorator;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Inject } from "@nestjs/common";
|
|
2
|
+
import { AiRouterService } from "./ai-router.service.js";
|
|
3
|
+
/** The injection token of the default instance's raw router (`AiRouter`). */
|
|
4
|
+
export const AI_ROUTER = "AI_ROUTER";
|
|
5
|
+
/** The token of a raw router: `AI_ROUTER` for the default instance, `AI_ROUTER:<name>` for a named one. */
|
|
6
|
+
export function getAiRouterToken(name) {
|
|
7
|
+
return name === undefined ? AI_ROUTER : `${AI_ROUTER}:${name}`;
|
|
8
|
+
}
|
|
9
|
+
/** The token of a service: the `AiRouterService` class for the default instance, `AiRouterService:<name>` for a named one. */
|
|
10
|
+
export function getAiRouterServiceToken(name) {
|
|
11
|
+
return name === undefined ? AiRouterService : `AiRouterService:${name}`;
|
|
12
|
+
}
|
|
13
|
+
/** Injects the raw router of the default instance, or of the instance registered under `name`. */
|
|
14
|
+
export function InjectAiRouter(name) {
|
|
15
|
+
return Inject(getAiRouterToken(name));
|
|
16
|
+
}
|
|
17
|
+
/** Injects the `AiRouterService` of the default instance, or of the instance registered under `name`. */
|
|
18
|
+
export function InjectAiRouterService(name) {
|
|
19
|
+
return Inject(getAiRouterServiceToken(name));
|
|
20
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Capability, CallOptions, ModelConfig, PoolConfig } .js";
|
|
1
|
+
import type { Capability, CallOptions, ModelConfig, PoolConfig } from "./types.js";
|
|
2
2
|
/** The capabilities of a model that declares none. */
|
|
3
3
|
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
4
|
/** A breaker's thresholds with every default applied. */
|
package/dist/errors.d.ts
CHANGED
package/dist/events.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RouterEvents } .js";
|
|
1
|
+
import type { RouterEvents } from "./types.js";
|
|
2
2
|
type Handler<E extends keyof RouterEvents> = (payload: RouterEvents[E]) => void;
|
|
3
3
|
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
4
4
|
export declare class Emitter {
|
package/dist/fake.d.ts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export type {
|
|
4
|
-
export {
|
|
5
|
-
export {
|
|
6
|
-
export
|
|
7
|
-
export {
|
|
8
|
-
export type {
|
|
9
|
-
export
|
|
10
|
-
export type { SseEvent } .js";
|
|
11
|
-
export { EMPTY_RECORD, MemoryStateStore } .js";
|
|
12
|
-
export type { HealthRecord, StateStore } .js";
|
|
13
|
-
export { createFakeProvider } .js";
|
|
14
|
-
export type { FakeCall, FakeMatch, FakeProvider, FakeReply, FakeStream } .js";
|
|
15
|
-
export type * .js";
|
|
1
|
+
export { AiRouterModule } from "./ai-router.module.js";
|
|
2
|
+
export type { AiRouterModuleExtras, AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } from "./ai-router.module-definition.js";
|
|
3
|
+
export type { AiRouterHandlers, AiRouterModuleOptions } from "./ai-router.options.js";
|
|
4
|
+
export { AiRouterService } from "./ai-router.service.js";
|
|
5
|
+
export { AI_ROUTER, getAiRouterServiceToken, getAiRouterToken, InjectAiRouter, InjectAiRouterService } from "./ai-router.tokens.js";
|
|
6
|
+
export { AiRouterError } from "./errors.js";
|
|
7
|
+
export type { AttemptFailure } from "./errors.js";
|
|
8
|
+
export type { HealthRecord, StateStore } from "./store.js";
|
|
9
|
+
export type * from "./types.js";
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
4
|
-
export { AiRouterError
|
|
5
|
-
export { parseDuration, readRateLimit } from "./headers.js";
|
|
6
|
-
export { parseSse } from "./sse.js";
|
|
7
|
-
export { EMPTY_RECORD, MemoryStateStore } from "./store.js";
|
|
8
|
-
export { createFakeProvider } from "./fake.js";
|
|
1
|
+
export { AiRouterModule } from "./ai-router.module.js";
|
|
2
|
+
export { AiRouterService } from "./ai-router.service.js";
|
|
3
|
+
export { AI_ROUTER, getAiRouterServiceToken, getAiRouterToken, InjectAiRouter, InjectAiRouterService } from "./ai-router.tokens.js";
|
|
4
|
+
export { AiRouterError } from "./errors.js";
|
package/dist/pool.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { ResolvedPolicy, Settings } .js";
|
|
2
|
-
import type { Emitter } .js";
|
|
3
|
-
import type { RateLimitInfo } .js";
|
|
4
|
-
import type { StateStore } .js";
|
|
5
|
-
import type { Capability, ErrorClass, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } .js";
|
|
1
|
+
import type { ResolvedPolicy, Settings } from "./config.js";
|
|
2
|
+
import type { Emitter } from "./events.js";
|
|
3
|
+
import type { RateLimitInfo } from "./headers.js";
|
|
4
|
+
import type { StateStore } from "./store.js";
|
|
5
|
+
import type { Capability, ErrorClass, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } from "./types.js";
|
|
6
6
|
/** One routable (provider, key, model) triple and the rank of its model in the call's policy. */
|
|
7
7
|
export interface Lane {
|
|
8
8
|
readonly provider: ProviderConfig;
|
package/dist/pool.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { DEFAULT_CAPABILITIES, modelMatches } from "./config.js";
|
|
2
2
|
import { EMPTY_RECORD } from "./store.js";
|
|
3
3
|
const FOREVER = Number.MAX_SAFE_INTEGER;
|
|
4
|
+
/** The latency samples a lane needs before its EWMA ranks it: one cold call (a JIT warm-up, a first TLS handshake) never decides. */
|
|
5
|
+
const LATENCY_SAMPLES = 3;
|
|
6
|
+
/** How far a lane latency leaves out drifts toward the best EWMA per pick: a 2.5x-slow lane is retried about every tenth pick. */
|
|
7
|
+
const LATENCY_DRIFT = 0.1;
|
|
4
8
|
function breakerState(record, now) {
|
|
5
9
|
if (record.breaker === "closed")
|
|
6
10
|
return "closed";
|
|
@@ -25,6 +29,7 @@ export class PoolState {
|
|
|
25
29
|
#inflightKey = new Map();
|
|
26
30
|
#inflightProvider = new Map();
|
|
27
31
|
#ewma = new Map();
|
|
32
|
+
#samples = new Map();
|
|
28
33
|
#roundRobin = new Map();
|
|
29
34
|
#counters = new Map();
|
|
30
35
|
#waiters = new Set();
|
|
@@ -182,10 +187,18 @@ export class PoolState {
|
|
|
182
187
|
const load = (lane) => this.#inflightKey.get(lane.pair) ?? 0;
|
|
183
188
|
const least = Math.min(...group.map(load));
|
|
184
189
|
let pool = group.filter((lane) => load(lane) === least);
|
|
185
|
-
|
|
190
|
+
// Latency ranks only lanes with LATENCY_SAMPLES samples, so one cold call never decides; a lane it leaves out drifts
|
|
191
|
+
// toward the best EWMA each time, so it is tried again instead of losing its turn for good on samples it can no longer add.
|
|
192
|
+
const measured = (lane) => ((this.#samples.get(lane.id) ?? 0) >= LATENCY_SAMPLES ? this.#ewma.get(lane.id) : undefined);
|
|
193
|
+
const known = pool.map(measured).filter((value) => value !== undefined);
|
|
186
194
|
if (known.length > 0) {
|
|
187
|
-
const
|
|
188
|
-
|
|
195
|
+
const best = Math.min(...known);
|
|
196
|
+
const limit = best * this.#settings.latencyTolerance + this.#settings.latencySlackMs;
|
|
197
|
+
const fast = pool.filter((lane) => (measured(lane) ?? best) <= limit);
|
|
198
|
+
for (const lane of pool)
|
|
199
|
+
if (!fast.includes(lane))
|
|
200
|
+
this.#ewma.set(lane.id, LATENCY_DRIFT * best + (1 - LATENCY_DRIFT) * (this.#ewma.get(lane.id) ?? best));
|
|
201
|
+
pool = fast;
|
|
189
202
|
}
|
|
190
203
|
// Smooth weighted round-robin (nginx): each lane gains its weight, the leader is picked and pays the total.
|
|
191
204
|
let total = 0;
|
|
@@ -308,6 +321,7 @@ export class PoolState {
|
|
|
308
321
|
const previous = this.#ewma.get(lane.id);
|
|
309
322
|
const alpha = this.#settings.ewmaAlpha;
|
|
310
323
|
this.#ewma.set(lane.id, previous === undefined ? latencyMs : alpha * latencyMs + (1 - alpha) * previous);
|
|
324
|
+
this.#samples.set(lane.id, (this.#samples.get(lane.id) ?? 0) + 1);
|
|
311
325
|
const counters = this.#count(lane);
|
|
312
326
|
counters.successes++;
|
|
313
327
|
const cost = costOf(lane.model, usage);
|
|
@@ -404,7 +418,7 @@ export class PoolState {
|
|
|
404
418
|
}
|
|
405
419
|
}
|
|
406
420
|
const local = target.provider === undefined ? "" : target.key === undefined ? `${target.provider}/` : `${target.provider}/${target.key}/`;
|
|
407
|
-
for (const map of [this.#ewma, this.#roundRobin]) {
|
|
421
|
+
for (const map of [this.#ewma, this.#samples, this.#roundRobin]) {
|
|
408
422
|
for (const id of [...map.keys()])
|
|
409
423
|
if (id.startsWith(local))
|
|
410
424
|
map.delete(id);
|
package/dist/router.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type { AiRouter, PoolConfig } .js";
|
|
1
|
+
import type { AiRouter, PoolConfig } from "./types.js";
|
|
2
2
|
/** Creates a router over a pool of providers, keys and models. The pool is validated now; keys are read per attempt. */
|
|
3
3
|
export declare function createRouter<P extends string = string>(config: PoolConfig<P>): AiRouter<P>;
|
package/dist/store.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { MaybePromise } .js";
|
|
1
|
+
import type { MaybePromise } from "./types.js";
|
|
2
2
|
/**
|
|
3
3
|
* The shared health of one key, one provider or one key and model ("lane"). Every field is a JSON number, string or
|
|
4
4
|
* null, so an adapter can store it anywhere. Times are epoch milliseconds; 0 means "not set".
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { AiRouterTestingModule } from "./ai-router-testing.module.js";
|
|
2
|
+
export type { AiRouterTestingOptions } from "./ai-router-testing.module.js";
|
|
3
|
+
export { createFakeProvider } from "./fake.js";
|
|
4
|
+
export type { FakeCall, FakeMatch, FakeProvider, FakeReply, FakeStream } from "./fake.js";
|
package/dist/testing.js
ADDED
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { StateStore } .js";
|
|
1
|
+
import type { StateStore } from "./store.js";
|
|
2
2
|
/** A value or a promise of it: store adapters and key resolvers may answer either way. */
|
|
3
3
|
export type MaybePromise<T> = T | Promise<T>;
|
|
4
4
|
/** What a model can do; a request needs every capability its shape implies. */
|
|
@@ -102,7 +102,8 @@ export interface ResilienceConfig {
|
|
|
102
102
|
export interface SelectionConfig {
|
|
103
103
|
/**
|
|
104
104
|
* A candidate whose latency EWMA exceeds `best * latencyTolerance + latencySlackMs` drops out of the tie.
|
|
105
|
-
* Defaults 1.5 and 50 ms; the slack keeps millisecond jitter from deciding between equally fast keys.
|
|
105
|
+
* Defaults 1.5 and 50 ms; the slack keeps millisecond jitter from deciding between equally fast keys. A lane is ranked
|
|
106
|
+
* by latency once it has three samples, and one left out drifts toward the best EWMA until it is tried again.
|
|
106
107
|
*/
|
|
107
108
|
readonly latencyTolerance?: number;
|
|
108
109
|
readonly latencySlackMs?: number;
|
|
@@ -406,7 +407,7 @@ export interface ResetTarget {
|
|
|
406
407
|
readonly provider?: string;
|
|
407
408
|
readonly key?: string;
|
|
408
409
|
}
|
|
409
|
-
/** The router `
|
|
410
|
+
/** The raw router behind `AiRouterService`, injected with `@InjectAiRouter()` (token `AI_ROUTER`). */
|
|
410
411
|
export interface AiRouter<Profiles extends string = string> {
|
|
411
412
|
/** Sends a chat-completions request through the pool. */
|
|
412
413
|
chat(request: ChatCompletionRequest, options?: CallOptions<Profiles>): Promise<ChatCompletion>;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@starci/ai-router",
|
|
3
|
-
"version": "0.1
|
|
4
|
-
"description": "A
|
|
3
|
+
"version": "0.2.1",
|
|
4
|
+
"description": "A NestJS module that routes OpenAI-compatible calls across a pool of keys, models and providers: AiRouterModule.register and an injectable AiRouterService for chat, stream and embeddings, with health-aware selection, circuit breakers, rate-limit awareness, retries onto another key, model fallback and redacted telemetry.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai",
|
|
7
7
|
"llm",
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
"failover",
|
|
14
14
|
"circuit-breaker",
|
|
15
15
|
"rate-limit",
|
|
16
|
-
"streaming"
|
|
16
|
+
"streaming",
|
|
17
|
+
"nestjs"
|
|
17
18
|
],
|
|
18
19
|
"license": "MIT",
|
|
19
20
|
"author": "starci183",
|
|
@@ -35,6 +36,10 @@
|
|
|
35
36
|
"types": "./dist/index.d.ts",
|
|
36
37
|
"default": "./dist/index.js"
|
|
37
38
|
},
|
|
39
|
+
"./testing": {
|
|
40
|
+
"types": "./dist/testing.d.ts",
|
|
41
|
+
"default": "./dist/testing.js"
|
|
42
|
+
},
|
|
38
43
|
"./package.json": "./package.json"
|
|
39
44
|
},
|
|
40
45
|
"files": [
|
|
@@ -47,12 +52,17 @@
|
|
|
47
52
|
"engines": {
|
|
48
53
|
"node": ">=20"
|
|
49
54
|
},
|
|
55
|
+
"peerDependencies": {
|
|
56
|
+
"@nestjs/common": "^11.0.0 || ^12.0.0",
|
|
57
|
+
"@nestjs/core": "^11.0.0 || ^12.0.0",
|
|
58
|
+
"reflect-metadata": "^0.1.12 || ^0.2.0"
|
|
59
|
+
},
|
|
50
60
|
"publishConfig": {
|
|
51
61
|
"access": "public",
|
|
52
62
|
"provenance": true
|
|
53
63
|
},
|
|
54
64
|
"scripts": {
|
|
55
|
-
"build": "node -e \"require('node:fs').rmSync('dist', {recursive: true, force: true})\" && tsc -p tsconfig.build.json && node -e \"const fs = require('node:fs'); for (const f of fs.readdirSync('dist').filter((n) => n.endsWith('.d.ts'))) fs.writeFileSync('dist/' + f, fs.readFileSync('dist/' + f, 'utf8').replace(/(from
|
|
65
|
+
"build": "node -e \"require('node:fs').rmSync('dist', {recursive: true, force: true})\" && tsc -p tsconfig.build.json && node -e \"const fs = require('node:fs'); for (const f of fs.readdirSync('dist', {recursive: true}).filter((n) => n.endsWith('.d.ts'))) fs.writeFileSync('dist/' + f, fs.readFileSync('dist/' + f, 'utf8').replace(/((?:from |import\\()['\\x22]\\.\\.?\\/[^'\\x22]+)\\.ts(['\\x22])/g, (all, head, quote) => head + '.js' + quote))\"",
|
|
56
66
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
57
67
|
"lint": "eslint .",
|
|
58
68
|
"test": "npm run typecheck && npm run lint && node --test --experimental-test-coverage --test-coverage-include=\"src/**/*.ts\" --test-coverage-lines=90 \"test/**/*.test.ts\"",
|
|
@@ -60,8 +70,13 @@
|
|
|
60
70
|
},
|
|
61
71
|
"devDependencies": {
|
|
62
72
|
"@eslint/js": "10.0.1",
|
|
73
|
+
"@nestjs/common": "12.1.3",
|
|
74
|
+
"@nestjs/core": "12.1.3",
|
|
75
|
+
"@nestjs/testing": "12.1.3",
|
|
63
76
|
"@types/node": "24.19.2",
|
|
64
77
|
"eslint": "10.13.0",
|
|
78
|
+
"reflect-metadata": "0.2.2",
|
|
79
|
+
"rxjs": "7.8.2",
|
|
65
80
|
"typescript": "5.9.3",
|
|
66
81
|
"typescript-eslint": "8.72.0"
|
|
67
82
|
}
|
package/dist/builder.d.ts
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import type { KeyResolver, ModelConfig, ModelPolicy, PoolConfig, ProviderConfig } .js";
|
|
2
|
-
/** The fields of a provider the builder takes up front; keys and models follow by method. */
|
|
3
|
-
export type ProviderOptions = Omit<ProviderConfig, "id" | "keys" | "models">;
|
|
4
|
-
/** The fields of a key besides its alias, with exactly one source: `env` or `resolve`. */
|
|
5
|
-
export type KeyOptions = {
|
|
6
|
-
readonly env: string;
|
|
7
|
-
readonly weight?: number;
|
|
8
|
-
readonly maxConcurrency?: number;
|
|
9
|
-
} | {
|
|
10
|
-
readonly resolve: KeyResolver;
|
|
11
|
-
readonly weight?: number;
|
|
12
|
-
readonly maxConcurrency?: number;
|
|
13
|
-
};
|
|
14
|
-
/** The fields of a model besides its id. */
|
|
15
|
-
export type ModelOptions = Omit<ModelConfig, "id">;
|
|
16
|
-
/** A fluent, typed way to write a `PoolConfig`; profile names accumulate in the type so calls can only name real ones. */
|
|
17
|
-
export declare class PoolBuilder<Profiles extends string = never> {
|
|
18
|
-
#private;
|
|
19
|
-
/** Starts a provider; the `key` and `model` calls after it belong to it. */
|
|
20
|
-
provider(id: string, options: ProviderOptions): this;
|
|
21
|
-
/** Adds a key to the current provider, by environment variable name or resolver. */
|
|
22
|
-
key(alias: string, options: KeyOptions): this;
|
|
23
|
-
/** Adds a model to the current provider. */
|
|
24
|
-
model(id: string, options?: ModelOptions): this;
|
|
25
|
-
/** Names a policy profile. */
|
|
26
|
-
profile<Name extends string>(name: Name, policy: ModelPolicy): PoolBuilder<Profiles | Name>;
|
|
27
|
-
/** Sets the pool-wide options: defaults, resilience, selection, telemetry, store, fetch. */
|
|
28
|
-
options(options: Omit<PoolConfig, "providers" | "profiles">): this;
|
|
29
|
-
/** Returns the validated `PoolConfig`. */
|
|
30
|
-
build(): PoolConfig<Profiles>;
|
|
31
|
-
}
|
|
32
|
-
/** Starts a typed pool builder. */
|
|
33
|
-
export declare function definePool(): PoolBuilder;
|
package/dist/builder.js
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
import { validateConfig } from "./config.js";
|
|
2
|
-
/** A fluent, typed way to write a `PoolConfig`; profile names accumulate in the type so calls can only name real ones. */
|
|
3
|
-
export class PoolBuilder {
|
|
4
|
-
#providers = [];
|
|
5
|
-
#profiles = {};
|
|
6
|
-
#rest = {};
|
|
7
|
-
/** Starts a provider; the `key` and `model` calls after it belong to it. */
|
|
8
|
-
provider(id, options) {
|
|
9
|
-
this.#providers.push({ id, options, keys: [], models: [] });
|
|
10
|
-
return this;
|
|
11
|
-
}
|
|
12
|
-
#current(what) {
|
|
13
|
-
const draft = this.#providers.at(-1);
|
|
14
|
-
if (draft === undefined)
|
|
15
|
-
throw new TypeError(`${what} needs a provider() before it`);
|
|
16
|
-
return draft;
|
|
17
|
-
}
|
|
18
|
-
/** Adds a key to the current provider, by environment variable name or resolver. */
|
|
19
|
-
key(alias, options) {
|
|
20
|
-
this.#current("key()").keys.push({ alias, ...options });
|
|
21
|
-
return this;
|
|
22
|
-
}
|
|
23
|
-
/** Adds a model to the current provider. */
|
|
24
|
-
model(id, options = {}) {
|
|
25
|
-
this.#current("model()").models.push({ id, ...options });
|
|
26
|
-
return this;
|
|
27
|
-
}
|
|
28
|
-
/** Names a policy profile. */
|
|
29
|
-
profile(name, policy) {
|
|
30
|
-
this.#profiles = { ...this.#profiles, [name]: policy };
|
|
31
|
-
return this;
|
|
32
|
-
}
|
|
33
|
-
/** Sets the pool-wide options: defaults, resilience, selection, telemetry, store, fetch. */
|
|
34
|
-
options(options) {
|
|
35
|
-
this.#rest = { ...this.#rest, ...options };
|
|
36
|
-
return this;
|
|
37
|
-
}
|
|
38
|
-
/** Returns the validated `PoolConfig`. */
|
|
39
|
-
build() {
|
|
40
|
-
const providers = this.#providers.map((draft) => ({ ...draft.options, id: draft.id, keys: [...draft.keys], models: [...draft.models] }));
|
|
41
|
-
const config = { ...this.#rest, providers, profiles: { ...this.#profiles } };
|
|
42
|
-
validateConfig(config);
|
|
43
|
-
return config;
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
/** Starts a typed pool builder. */
|
|
47
|
-
export function definePool() {
|
|
48
|
-
return new PoolBuilder();
|
|
49
|
-
}
|