@starci/ai-router 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +21 -0
- package/README.md +312 -2
- package/dist/builder.d.ts +33 -0
- package/dist/builder.js +49 -0
- package/dist/config.d.ts +46 -0
- package/dist/config.js +148 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +66 -0
- package/dist/events.d.ts +11 -0
- package/dist/events.js +31 -0
- package/dist/fake.d.ts +67 -0
- package/dist/fake.js +153 -0
- package/dist/headers.d.ts +15 -0
- package/dist/headers.js +66 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +8 -0
- package/dist/pool.d.ts +90 -0
- package/dist/pool.js +421 -0
- package/dist/router.d.ts +3 -0
- package/dist/router.js +588 -0
- package/dist/sse.d.ts +10 -0
- package/dist/sse.js +48 -0
- package/dist/store.d.ts +56 -0
- package/dist/store.js +48 -0
- package/dist/types.d.ts +426 -0
- package/dist/types.js +1 -0
- package/package.json +66 -4
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 - 2026-10-27
|
|
4
|
+
|
|
5
|
+
- Added: `createRouter(config)` over a pool of OpenAI-compatible providers, keys (by env var name or resolver, named by alias) and models (aliases, context window, cost, capabilities), with `chat`, `stream`, `embeddings`, `health`, `stats`, `on` and `reset`.
|
|
6
|
+
- Added: model policies per call or per named profile (preferred model, ordered fallbacks, `allowFallback`, `maxAttempts`, attempt timeout, total deadline, budget, provider restriction, required capabilities), and the typed builder `definePool()`.
|
|
7
|
+
- Added: selection by health and headroom (untried lane, model order, untried key and provider, headroom tier), then least in-flight, latency EWMA within a tolerance, and smooth weighted round-robin; per-key and per-provider concurrency caps that queue instead of failing.
|
|
8
|
+
- Added: circuit breakers per key and per provider (closed, open, half-open with one leased probe, doubling cooldown, `retry-after` honoured), error classification, jittered exponential backoff onto a different lane, auth-disabled keys, rate-limit cooldowns from `retry-after` and `x-ratelimit-*` headers, idempotent request ids, cancellation and deadlines.
|
|
9
|
+
- Added: SSE streaming with backpressure; fail-over before the first token, an `afterFirstToken` error after it, an idle timeout between chunks.
|
|
10
|
+
- Added: `attempt`, `request`, `breaker` and `key-disabled` events with no key values and no content unless `telemetry.includeContent`; key values scrubbed from error messages.
|
|
11
|
+
- Added: the pluggable `StateStore` (in-memory `MemoryStateStore` by default) and the deterministic test double `createFakeProvider()`.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 starci183
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,313 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @starci/ai-router
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A pool router for OpenAI-compatible APIs. Put many keys, many models and many providers behind one `chat`, `stream`
|
|
4
|
+
and `embeddings` call, and the router picks a healthy key for each attempt, backs off what is failing, honours rate
|
|
5
|
+
limits, retries on a different key or provider, falls back to another model when you allow it, and tells you what it
|
|
6
|
+
did without ever exposing a key or a prompt.
|
|
7
|
+
|
|
8
|
+
- Zero runtime dependencies: `fetch`, `AbortController` and `ReadableStream`. ESM, typed, Node 20 or newer.
|
|
9
|
+
- OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
|
|
10
|
+
behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
|
|
11
|
+
- Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
|
|
12
|
+
everywhere else.
|
|
13
|
+
|
|
14
|
+
## Why
|
|
15
|
+
|
|
16
|
+
One key hits its rate limit, another is revoked, a provider has a bad five minutes, and a model is briefly overloaded.
|
|
17
|
+
Hand-written retry loops get each of those slightly wrong: they retry a 400, hammer a key that answered 401, ignore
|
|
18
|
+
`retry-after`, or retry the same overloaded key. This package owns that loop once, with a test double to prove your use
|
|
19
|
+
of it.
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
npm install @starci/ai-router
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
import { createRouter } from "@starci/ai-router"
|
|
31
|
+
|
|
32
|
+
const router = createRouter({
|
|
33
|
+
providers: [
|
|
34
|
+
{
|
|
35
|
+
id: "openai",
|
|
36
|
+
baseUrl: "https://api.openai.com/v1",
|
|
37
|
+
keys: [
|
|
38
|
+
{ alias: "team-a", env: "OPENAI_KEY_A" },
|
|
39
|
+
{ alias: "team-b", env: "OPENAI_KEY_B", maxConcurrency: 8 },
|
|
40
|
+
],
|
|
41
|
+
models: [
|
|
42
|
+
{ id: "gpt-4o-mini", aliases: ["fast"], contextWindow: 128_000, cost: { input: 0.15, output: 0.6 } },
|
|
43
|
+
{ id: "gpt-4o", aliases: ["smart"], cost: { input: 2.5, output: 10 } },
|
|
44
|
+
{ id: "text-embedding-3-small", capabilities: ["embeddings"] },
|
|
45
|
+
],
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: "openrouter",
|
|
49
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
50
|
+
keys: [{ alias: "or-1", resolve: () => vault.read("openrouter") }],
|
|
51
|
+
models: [{ id: "openai/gpt-4o-mini", aliases: ["fast"] }],
|
|
52
|
+
},
|
|
53
|
+
],
|
|
54
|
+
defaults: { model: "fast", fallbacks: ["smart"], deadlineMs: 60_000 },
|
|
55
|
+
profiles: { strict: { model: "smart", allowFallback: false, maxAttempts: 3 } },
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
const reply = await router.chat({ messages: [{ role: "user", content: "Hello" }] })
|
|
59
|
+
console.log(reply.choices[0]?.message.content, reply.routing) // routing: provider, key alias, model, attempts, cost
|
|
60
|
+
|
|
61
|
+
for await (const chunk of router.stream({ model: "fast", messages: [{ role: "user", content: "Count to 5" }] })) {
|
|
62
|
+
process.stdout.write(chunk.choices[0]?.delta.content ?? "")
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const vectors = await router.embeddings({ model: "text-embedding-3-small", input: ["a", "b"] })
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The same pool with the typed builder; profile names become a type, so `{ profile: "strcit" }` does not compile:
|
|
69
|
+
|
|
70
|
+
```ts
|
|
71
|
+
import { createRouter, definePool } from "@starci/ai-router"
|
|
72
|
+
|
|
73
|
+
const config = definePool()
|
|
74
|
+
.provider("openai", { baseUrl: "https://api.openai.com/v1" })
|
|
75
|
+
.key("team-a", { env: "OPENAI_KEY_A" })
|
|
76
|
+
.model("gpt-4o-mini", { aliases: ["fast"] })
|
|
77
|
+
.model("gpt-4o", { aliases: ["smart"] })
|
|
78
|
+
.profile("strict", { model: "smart", allowFallback: false })
|
|
79
|
+
.options({ defaults: { model: "fast", fallbacks: ["smart"] } })
|
|
80
|
+
.build()
|
|
81
|
+
|
|
82
|
+
const router = createRouter(config)
|
|
83
|
+
await router.chat({ messages: [{ role: "user", content: "Hi" }] }, { profile: "strict" })
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## API
|
|
87
|
+
|
|
88
|
+
`createRouter(config)` validates the pool (it throws an `AiRouterError` of class `config` naming the problem) and
|
|
89
|
+
returns:
|
|
90
|
+
|
|
91
|
+
| member | does |
|
|
92
|
+
|---|---|
|
|
93
|
+
| `chat(request, options?)` | sends a chat-completions request; resolves to the OpenAI completion plus `routing` |
|
|
94
|
+
| `stream(request, options?)` | returns an async iterable of OpenAI chunks, with `requestId`; see [Streaming](#streaming) |
|
|
95
|
+
| `embeddings(request, options?)` | sends an embeddings request; resolves to the OpenAI response plus `routing` |
|
|
96
|
+
| `health()` | reads every breaker, disabled key, headroom and latency EWMA |
|
|
97
|
+
| `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
|
|
98
|
+
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled`; returns the unsubscribe function |
|
|
99
|
+
| `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
|
|
100
|
+
|
|
101
|
+
`options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
|
|
102
|
+
`x-request-id` and `idempotency-key` on every attempt of the call, so a provider that deduplicates sees one request.
|
|
103
|
+
`headers` adds headers to this call; it can never replace `authorization`.
|
|
104
|
+
|
|
105
|
+
Every failure is an `AiRouterError` with `errorClass`, `retryable`, `status`, `provider`, `key` (alias), `model`,
|
|
106
|
+
`requestId`, `attempts` (every failed attempt: where and why), `afterFirstToken` and `retryAt` (when a 429 said to
|
|
107
|
+
come back).
|
|
108
|
+
|
|
109
|
+
## Configuration reference
|
|
110
|
+
|
|
111
|
+
| field | meaning | default |
|
|
112
|
+
|---|---|---|
|
|
113
|
+
| `providers[].id` | a name, unique, without `/` | required |
|
|
114
|
+
| `providers[].baseUrl` | the URL the OpenAI paths hang from (`.../v1`) | required |
|
|
115
|
+
| `providers[].headers` | extra headers for every request (an `authorization` here is ignored) | none |
|
|
116
|
+
| `providers[].weight` | multiplies the round-robin share of every lane of the provider | 1 |
|
|
117
|
+
| `providers[].maxConcurrency` | in-flight requests across the provider's keys, in this process | unbounded |
|
|
118
|
+
| `providers[].keys[]` | `{ alias, env }` or `{ alias, resolve }`, plus `weight`, `maxConcurrency` | at least one |
|
|
119
|
+
| `providers[].models[]` | `{ id, aliases?, contextWindow?, cost?, capabilities?, weight? }` | at least one |
|
|
120
|
+
| `defaults` | the policy every call starts from | none |
|
|
121
|
+
| `profiles` | named policies a call selects with `{ profile }` | none |
|
|
122
|
+
| `resilience.keyBreaker` | `{ failureThreshold, cooldownMs, maxCooldownMs }` per key | 3, 15000, 300000 |
|
|
123
|
+
| `resilience.providerBreaker` | the same per provider, fed by all its keys | 8, 30000, 300000 |
|
|
124
|
+
| `resilience.backoff` | `{ baseMs, factor, maxMs }` between attempts after a transient failure | 100, 2, 5000 |
|
|
125
|
+
| `resilience.authDisableMs` | how long a 401/403 key stays out; `null` until `reset()` | `null` |
|
|
126
|
+
| `resilience.rateLimitCooldownMs` | a 429's cooldown when it has no `retry-after` | 10000 |
|
|
127
|
+
| `selection.latencyTolerance`, `latencySlackMs` | how far above the best latency EWMA a lane still ties | 1.5, 50 |
|
|
128
|
+
| `selection.ewmaAlpha` | the weight of the newest latency sample | 0.3 |
|
|
129
|
+
| `selection.lowHeadroom` | remaining requests at or below which a lane is used only when nothing else is | 1 |
|
|
130
|
+
| `telemetry.includeContent` | adds request and response bodies to `attempt` events | `false` |
|
|
131
|
+
| `store`, `storeNamespace` | where shared state lives, and its key prefix | in memory, `ai-router` |
|
|
132
|
+
| `fetch` | the `fetch` to use (a proxy agent, a test double) | global `fetch` |
|
|
133
|
+
| `random` | the jitter source | `Math.random` |
|
|
134
|
+
|
|
135
|
+
A model's `id` is the name sent to the provider; a caller may name it by `id` or any alias, and an alias shared by
|
|
136
|
+
several providers makes them interchangeable for that call. `cost` is per one million tokens, `input` and `output`, in
|
|
137
|
+
whatever unit you budget in. `capabilities` defaults to `chat`, `streaming`, `tools` and `json`; an embeddings model
|
|
138
|
+
declares `["embeddings"]`.
|
|
139
|
+
|
|
140
|
+
## Policies
|
|
141
|
+
|
|
142
|
+
A call's policy is `defaults`, then the selected profile, then `options.policy`, merged field by field (later wins).
|
|
143
|
+
The preferred model is the call policy's `model`, else the profile's, else the request's own `model`, else the
|
|
144
|
+
defaults'.
|
|
145
|
+
|
|
146
|
+
| field | meaning | default |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| `model` | the preferred model | the request's |
|
|
149
|
+
| `fallbacks` | models tried in order once the preferred one has no usable lane | none |
|
|
150
|
+
| `allowFallback` | `false` pins the call to the preferred model | `true` |
|
|
151
|
+
| `maxAttempts` | attempts across keys, providers and models | 4 |
|
|
152
|
+
| `timeoutMs` | one attempt (a stream: until the first token, then between chunks) | 60000 |
|
|
153
|
+
| `deadlineMs` | the whole call, waits and backoff included | 120000 |
|
|
154
|
+
| `budget.maxCost` | a model whose worst-case cost for this request is higher is no candidate | none |
|
|
155
|
+
| `providers` | restrict the call to these provider ids | all |
|
|
156
|
+
| `require` | capabilities needed beyond those the request implies | none |
|
|
157
|
+
|
|
158
|
+
The request implies its own needs: `tools` needs `tools`, a non-text `response_format` needs `json`, `stream()` needs
|
|
159
|
+
`streaming`, `embeddings()` needs `embeddings`. A model is also no candidate when its `contextWindow` is smaller than
|
|
160
|
+
the estimated prompt (four characters a token) plus `max_tokens`. When nothing is left, the call fails at once with
|
|
161
|
+
class `no_candidate`.
|
|
162
|
+
|
|
163
|
+
## Selection
|
|
164
|
+
|
|
165
|
+
A candidate ("lane") is one provider, key and model. A lane is out while its key is disabled, its key's or provider's
|
|
166
|
+
breaker is open, a 429 cooldown holds it, its last reported remaining requests are 0 (or remaining tokens are below
|
|
167
|
+
the estimate) until the reported reset, or its key or provider is at its concurrency cap. Among the lanes left, tiers
|
|
168
|
+
decide, in order:
|
|
169
|
+
|
|
170
|
+
1. a lane this call has not tried yet;
|
|
171
|
+
2. the policy's model order (preferred, then each fallback);
|
|
172
|
+
3. a key this call has not tried yet, then a provider it has not tried yet;
|
|
173
|
+
4. enough headroom (lanes at `lowHeadroom` or below come last).
|
|
174
|
+
|
|
175
|
+
Within the first non-empty tier: the fewest in-flight requests on the key, then latency (lanes whose EWMA is within
|
|
176
|
+
`best * latencyTolerance + latencySlackMs`; a lane never measured stays in), then smooth weighted round-robin by
|
|
177
|
+
`provider.weight * key.weight * model.weight`. Equal keys therefore alternate deterministically rather than randomly.
|
|
178
|
+
|
|
179
|
+
When no lane is usable now but one will be before the deadline (a cooldown, a breaker, a concurrency slot), the call
|
|
180
|
+
waits for it rather than failing. When none will be, it fails at once with the class of the last failure (or
|
|
181
|
+
`no_candidate`) and `retryAt`.
|
|
182
|
+
|
|
183
|
+
## Resilience
|
|
184
|
+
|
|
185
|
+
| what happened | class | effect | retried |
|
|
186
|
+
|---|---|---|---|
|
|
187
|
+
| 429, or a rate/quota error object | `rate_limit` | the lane (key and model) cools down until `retry-after` / `retry-after-ms`, else `rateLimitCooldownMs` | yes, at once, on another lane |
|
|
188
|
+
| 401, 403, an auth error object | `auth` | the key is disabled (`authDisableMs`, default until `reset()`); `key-disabled` event | yes, at once, on another key |
|
|
189
|
+
| a resolver that returns nothing or throws, an unset env var | `auth` | the key is disabled with reason `missing`; nothing is sent | yes |
|
|
190
|
+
| 408, 409, 425, 5xx | `transient` | counts toward the key's and the provider's breakers; a `retry-after` also cools the lane | yes, after backoff |
|
|
191
|
+
| connection failure | `network` | as transient | yes, after backoff |
|
|
192
|
+
| attempt timeout | `timeout` | as transient | yes, after backoff |
|
|
193
|
+
| broken or malformed stream before the first token | `stream` (or the streamed error's class) | as transient | yes, after backoff |
|
|
194
|
+
| 400, 404, 422, other 4xx, an invalid-request error object | `invalid_request` | none | no |
|
|
195
|
+
| the caller's signal aborted | `cancelled` | the attempt in flight is aborted | no |
|
|
196
|
+
| the call's deadline passed | `deadline` | the attempt in flight is aborted | no |
|
|
197
|
+
|
|
198
|
+
Breakers are closed, open or half-open. A closed breaker opens after `failureThreshold` consecutive transient
|
|
199
|
+
failures (a success resets the count), for `cooldownMs` or until a `retry-after`, whichever is later. Past its
|
|
200
|
+
cooldown it is half-open: exactly one attempt, the probe, holds a lease and goes through while every other call waits.
|
|
201
|
+
A successful probe closes it; a failed one reopens it with the cooldown doubled, up to `maxCooldownMs`. Rate limits,
|
|
202
|
+
invalid requests and cancellations never count against a breaker.
|
|
203
|
+
|
|
204
|
+
Backoff after a transient failure is `min(maxMs, baseMs * factor^(n-1))` scaled by a random factor in `[0.5, 1)`, and
|
|
205
|
+
the next attempt goes to a different lane by the tiers above. Rate and auth failures belong to one key, so the next key
|
|
206
|
+
is tried without waiting.
|
|
207
|
+
|
|
208
|
+
## Streaming
|
|
209
|
+
|
|
210
|
+
`stream()` sends `stream: true` with `accept: text/event-stream` and parses the server-sent events itself. The body is
|
|
211
|
+
read only when you ask for the next chunk, so a slow consumer holds the connection back instead of buffering the whole
|
|
212
|
+
reply; breaking out of the loop cancels the body and frees the key's slot.
|
|
213
|
+
|
|
214
|
+
- **Before the first token** (the first chunk whose delta carries anything besides `role`), chunks are held back.
|
|
215
|
+
If the stream fails then, by HTTP status, a broken connection, malformed data or a streamed `error` object, the
|
|
216
|
+
attempt fails over like any other and the held chunks are dropped: you never see two role chunks or half an answer.
|
|
217
|
+
- **After the first token**, chunks are yielded as they arrive and a failure is thrown as an `AiRouterError` with
|
|
218
|
+
`afterFirstToken: true`. The router does not retry it, since a second answer cannot be spliced onto the first; retry
|
|
219
|
+
the whole call if your use allows it.
|
|
220
|
+
- `timeoutMs` bounds the wait for the first token and then each gap between chunks; `deadlineMs` bounds the whole
|
|
221
|
+
stream.
|
|
222
|
+
- Stopping early counts as a success of the lane. Usage is read from a final usage chunk when the provider sends one
|
|
223
|
+
(OpenAI does when you pass `stream_options: { include_usage: true }`).
|
|
224
|
+
- Policy errors (an unknown model or profile) throw from `stream()` itself; everything else is thrown from iteration.
|
|
225
|
+
|
|
226
|
+
## Observability
|
|
227
|
+
|
|
228
|
+
```ts
|
|
229
|
+
router.on("attempt", (event) => metrics.histogram("llm.attempt", event.latencyMs, { provider: event.provider, key: event.key, model: event.model, outcome: event.outcome, class: event.errorClass }))
|
|
230
|
+
router.on("breaker", (event) => log.warn("breaker", event))
|
|
231
|
+
router.on("key-disabled", (event) => alert(`key ${event.provider}/${event.key} disabled: ${event.reason}`))
|
|
232
|
+
router.on("request", (event) => log.info("llm call", event))
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
| event | when | carries |
|
|
236
|
+
|---|---|---|
|
|
237
|
+
| `attempt` | every attempt | `requestId`, `operation`, `attempt`, `provider`, `key` alias, `model`, `latencyMs`, `outcome`, `errorClass`, `status`, `usage`, `cost` |
|
|
238
|
+
| `request` | every call, once | `requestId`, `operation`, `outcome`, `attempts`, `latencyMs`, the answering lane or the `errorClass` |
|
|
239
|
+
| `breaker` | a breaker changes state | `scope` (`key` or `provider`), `provider`, `key`, `from`, `to` |
|
|
240
|
+
| `key-disabled` | a key is disabled | `provider`, `key`, `reason` (`auth` or `missing`), `until` (null: until `reset()`) |
|
|
241
|
+
|
|
242
|
+
Events never carry message content unless `telemetry.includeContent` is `true`; then `attempt` events add `content`
|
|
243
|
+
with the request body and, for `chat` and `embeddings`, the response. A handler that throws is isolated from the call.
|
|
244
|
+
|
|
245
|
+
## Sharing state between processes
|
|
246
|
+
|
|
247
|
+
Breakers, disabled keys and headroom live in a `StateStore`; the default keeps them in memory. Several processes (or
|
|
248
|
+
several routers) share them through one store: an adapter implements `get`, `set`, `delete`, `keys(prefix)`, and an
|
|
249
|
+
atomic `lease(key, ttlMs)` / `release(key)` pair for the half-open probe. Methods may return values or promises.
|
|
250
|
+
Records are plain JSON (`HealthRecord`); a Redis adapter is a `GET`/`SET`, a `SCAN` and a `SET NX PX` lease. In-flight
|
|
251
|
+
counts, latency EWMAs and round-robin positions stay per process. Updates are read-modify-write, so two processes
|
|
252
|
+
failing at the same instant may count one failure between them: breakers are a health signal, not a ledger.
|
|
253
|
+
|
|
254
|
+
## Testing with the fake provider
|
|
255
|
+
|
|
256
|
+
`createFakeProvider()` is an in-process OpenAI-compatible provider for your own tests. Pass its `fetch` to the router
|
|
257
|
+
and script replies per base URL, key, model or path; replies are used in order and the last one repeats.
|
|
258
|
+
|
|
259
|
+
```ts
|
|
260
|
+
import { createFakeProvider, createRouter } from "@starci/ai-router"
|
|
261
|
+
|
|
262
|
+
const fake = createFakeProvider()
|
|
263
|
+
fake.on({ key: "sk-test-1" }, { status: 429, headers: { "retry-after": "30" } })
|
|
264
|
+
fake.on({ key: "sk-test-2" }, { latencyMs: 20, text: "hello" })
|
|
265
|
+
fake.on({ model: "m-stream" }, { stream: { chunks: ["a", "b"], failAfter: 1 } })
|
|
266
|
+
|
|
267
|
+
const router = createRouter({
|
|
268
|
+
providers: [{ id: "p", baseUrl: "https://p.test/v1", keys: [{ alias: "one", resolve: () => "sk-test-1" }, { alias: "two", resolve: () => "sk-test-2" }], models: [{ id: "m" }, { id: "m-stream" }] }],
|
|
269
|
+
fetch: fake.fetch,
|
|
270
|
+
})
|
|
271
|
+
const reply = await router.chat({ model: "m", messages: [{ role: "user", content: "hi" }] })
|
|
272
|
+
// reply.routing.key === "two"; fake.calls lists both requests with their key, model, headers and body
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `usage`, `embedding`, a raw `body`, and
|
|
276
|
+
`stream: { chunks, delayMs, failAfter, failWith: "network" | "error-event", stallAfter, stallMs, includeUsage }`.
|
|
277
|
+
Everything is deterministic except the latencies you script.
|
|
278
|
+
|
|
279
|
+
## Security notes
|
|
280
|
+
|
|
281
|
+
- A key is given as an environment variable name or a resolver; the package never needs a literal in configuration.
|
|
282
|
+
It is read at each attempt (so rotation needs no restart) and is sent only in the `authorization` header to its own
|
|
283
|
+
provider's `baseUrl`.
|
|
284
|
+
- Keys are never logged, never put in events, errors, `health()` or `stats()`; every surface names them by alias. An
|
|
285
|
+
error body that echoes the key has it replaced by `[redacted]`.
|
|
286
|
+
- Prompts and responses are not in events unless you set `telemetry.includeContent`.
|
|
287
|
+
- `headers` in a provider or a call cannot override `authorization`.
|
|
288
|
+
- The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
|
|
289
|
+
|
|
290
|
+
## FAQ
|
|
291
|
+
|
|
292
|
+
**Is this a proxy server?** No, it is a library inside your process. Wrap it in a server if you need one.
|
|
293
|
+
|
|
294
|
+
**Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
|
|
295
|
+
the OpenAI chat-completions and embeddings API.
|
|
296
|
+
|
|
297
|
+
**Why did my call wait instead of failing?** A lane was going to be usable before the deadline (a `retry-after`, a
|
|
298
|
+
breaker cooldown, a concurrency slot). Lower `deadlineMs` to fail sooner.
|
|
299
|
+
|
|
300
|
+
**Why is a key never used again?** It answered 401/403, or its resolver returned nothing, and `authDisableMs` is `null`
|
|
301
|
+
by default. Fix the key and call `router.reset({ provider, key })`, or set `authDisableMs`.
|
|
302
|
+
|
|
303
|
+
**How do I keep calls on one model?** `allowFallback: false` in the policy or a profile. `providers: [...]` keeps them
|
|
304
|
+
on chosen providers.
|
|
305
|
+
|
|
306
|
+
**Why does a stream throw instead of failing over?** It had already yielded tokens; see [Streaming](#streaming).
|
|
307
|
+
|
|
308
|
+
**Are costs exact?** They are usage times your configured price; the budget check before a call uses an estimate (four
|
|
309
|
+
characters a token, plus `max_tokens`).
|
|
310
|
+
|
|
311
|
+
## License
|
|
312
|
+
|
|
313
|
+
MIT
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { KeyResolver, ModelConfig, ModelPolicy, PoolConfig, ProviderConfig } .js";
|
|
2
|
+
/** The fields of a provider the builder takes up front; keys and models follow by method. */
|
|
3
|
+
export type ProviderOptions = Omit<ProviderConfig, "id" | "keys" | "models">;
|
|
4
|
+
/** The fields of a key besides its alias, with exactly one source: `env` or `resolve`. */
|
|
5
|
+
export type KeyOptions = {
|
|
6
|
+
readonly env: string;
|
|
7
|
+
readonly weight?: number;
|
|
8
|
+
readonly maxConcurrency?: number;
|
|
9
|
+
} | {
|
|
10
|
+
readonly resolve: KeyResolver;
|
|
11
|
+
readonly weight?: number;
|
|
12
|
+
readonly maxConcurrency?: number;
|
|
13
|
+
};
|
|
14
|
+
/** The fields of a model besides its id. */
|
|
15
|
+
export type ModelOptions = Omit<ModelConfig, "id">;
|
|
16
|
+
/** A fluent, typed way to write a `PoolConfig`; profile names accumulate in the type so calls can only name real ones. */
|
|
17
|
+
export declare class PoolBuilder<Profiles extends string = never> {
|
|
18
|
+
#private;
|
|
19
|
+
/** Starts a provider; the `key` and `model` calls after it belong to it. */
|
|
20
|
+
provider(id: string, options: ProviderOptions): this;
|
|
21
|
+
/** Adds a key to the current provider, by environment variable name or resolver. */
|
|
22
|
+
key(alias: string, options: KeyOptions): this;
|
|
23
|
+
/** Adds a model to the current provider. */
|
|
24
|
+
model(id: string, options?: ModelOptions): this;
|
|
25
|
+
/** Names a policy profile. */
|
|
26
|
+
profile<Name extends string>(name: Name, policy: ModelPolicy): PoolBuilder<Profiles | Name>;
|
|
27
|
+
/** Sets the pool-wide options: defaults, resilience, selection, telemetry, store, fetch. */
|
|
28
|
+
options(options: Omit<PoolConfig, "providers" | "profiles">): this;
|
|
29
|
+
/** Returns the validated `PoolConfig`. */
|
|
30
|
+
build(): PoolConfig<Profiles>;
|
|
31
|
+
}
|
|
32
|
+
/** Starts a typed pool builder. */
|
|
33
|
+
export declare function definePool(): PoolBuilder;
|
package/dist/builder.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { validateConfig } from "./config.js";
|
|
2
|
+
/** A fluent, typed way to write a `PoolConfig`; profile names accumulate in the type so calls can only name real ones. */
|
|
3
|
+
export class PoolBuilder {
|
|
4
|
+
#providers = [];
|
|
5
|
+
#profiles = {};
|
|
6
|
+
#rest = {};
|
|
7
|
+
/** Starts a provider; the `key` and `model` calls after it belong to it. */
|
|
8
|
+
provider(id, options) {
|
|
9
|
+
this.#providers.push({ id, options, keys: [], models: [] });
|
|
10
|
+
return this;
|
|
11
|
+
}
|
|
12
|
+
#current(what) {
|
|
13
|
+
const draft = this.#providers.at(-1);
|
|
14
|
+
if (draft === undefined)
|
|
15
|
+
throw new TypeError(`${what} needs a provider() before it`);
|
|
16
|
+
return draft;
|
|
17
|
+
}
|
|
18
|
+
/** Adds a key to the current provider, by environment variable name or resolver. */
|
|
19
|
+
key(alias, options) {
|
|
20
|
+
this.#current("key()").keys.push({ alias, ...options });
|
|
21
|
+
return this;
|
|
22
|
+
}
|
|
23
|
+
/** Adds a model to the current provider. */
|
|
24
|
+
model(id, options = {}) {
|
|
25
|
+
this.#current("model()").models.push({ id, ...options });
|
|
26
|
+
return this;
|
|
27
|
+
}
|
|
28
|
+
/** Names a policy profile. */
|
|
29
|
+
profile(name, policy) {
|
|
30
|
+
this.#profiles = { ...this.#profiles, [name]: policy };
|
|
31
|
+
return this;
|
|
32
|
+
}
|
|
33
|
+
/** Sets the pool-wide options: defaults, resilience, selection, telemetry, store, fetch. */
|
|
34
|
+
options(options) {
|
|
35
|
+
this.#rest = { ...this.#rest, ...options };
|
|
36
|
+
return this;
|
|
37
|
+
}
|
|
38
|
+
/** Returns the validated `PoolConfig`. */
|
|
39
|
+
build() {
|
|
40
|
+
const providers = this.#providers.map((draft) => ({ ...draft.options, id: draft.id, keys: [...draft.keys], models: [...draft.models] }));
|
|
41
|
+
const config = { ...this.#rest, providers, profiles: { ...this.#profiles } };
|
|
42
|
+
validateConfig(config);
|
|
43
|
+
return config;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/** Starts a typed pool builder. */
|
|
47
|
+
export function definePool() {
|
|
48
|
+
return new PoolBuilder();
|
|
49
|
+
}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { Capability, CallOptions, ModelConfig, PoolConfig } .js";
|
|
2
|
+
/** The capabilities of a model that declares none. */
|
|
3
|
+
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
|
+
/** A breaker's thresholds with every default applied. */
|
|
5
|
+
export interface BreakerSettings {
|
|
6
|
+
readonly failureThreshold: number;
|
|
7
|
+
readonly cooldownMs: number;
|
|
8
|
+
readonly maxCooldownMs: number;
|
|
9
|
+
}
|
|
10
|
+
/** The pool-wide settings with every default applied. */
|
|
11
|
+
export interface Settings {
|
|
12
|
+
readonly keyBreaker: BreakerSettings;
|
|
13
|
+
readonly providerBreaker: BreakerSettings;
|
|
14
|
+
readonly backoff: {
|
|
15
|
+
readonly baseMs: number;
|
|
16
|
+
readonly factor: number;
|
|
17
|
+
readonly maxMs: number;
|
|
18
|
+
};
|
|
19
|
+
readonly authDisableMs: number | null;
|
|
20
|
+
readonly rateLimitCooldownMs: number;
|
|
21
|
+
readonly latencyTolerance: number;
|
|
22
|
+
readonly latencySlackMs: number;
|
|
23
|
+
readonly ewmaAlpha: number;
|
|
24
|
+
readonly lowHeadroom: number;
|
|
25
|
+
readonly includeContent: boolean;
|
|
26
|
+
readonly namespace: string;
|
|
27
|
+
}
|
|
28
|
+
/** One call's policy with every default applied and its model list expanded. */
|
|
29
|
+
export interface ResolvedPolicy {
|
|
30
|
+
/** The preferred model first, then the fallbacks when fallback is allowed, without repeats. */
|
|
31
|
+
readonly models: readonly string[];
|
|
32
|
+
readonly maxAttempts: number;
|
|
33
|
+
readonly timeoutMs: number;
|
|
34
|
+
readonly deadlineMs: number;
|
|
35
|
+
readonly maxCost: number | null;
|
|
36
|
+
readonly providers: ReadonlySet<string> | null;
|
|
37
|
+
readonly require: readonly Capability[];
|
|
38
|
+
}
|
|
39
|
+
/** True when the name is the model's id or one of its aliases. */
|
|
40
|
+
export declare function modelMatches(model: ModelConfig, name: string): boolean;
|
|
41
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
|
|
42
|
+
export declare function validateConfig(config: PoolConfig): void;
|
|
43
|
+
/** Applies the defaults of every pool-wide setting. */
|
|
44
|
+
export declare function resolveSettings(config: PoolConfig): Settings;
|
|
45
|
+
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
46
|
+
export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { AiRouterError } from "./errors.js";
|
|
2
|
+
/** The capabilities of a model that declares none. */
|
|
3
|
+
export const DEFAULT_CAPABILITIES = ["chat", "streaming", "tools", "json"];
|
|
4
|
+
function configError(message) {
|
|
5
|
+
return new AiRouterError({ errorClass: "config", message });
|
|
6
|
+
}
|
|
7
|
+
function positive(value, name, allowZero = false) {
|
|
8
|
+
if (value === undefined)
|
|
9
|
+
return;
|
|
10
|
+
if (!Number.isFinite(value) || value < 0 || (!allowZero && value === 0))
|
|
11
|
+
throw configError(`${name} must be a positive number`);
|
|
12
|
+
}
|
|
13
|
+
/** True when the name is the model's id or one of its aliases. */
|
|
14
|
+
export function modelMatches(model, name) {
|
|
15
|
+
return model.id === name || (model.aliases?.includes(name) ?? false);
|
|
16
|
+
}
|
|
17
|
+
function knownModel(config, name) {
|
|
18
|
+
return config.providers.some((provider) => provider.models.some((model) => modelMatches(model, name)));
|
|
19
|
+
}
|
|
20
|
+
function validatePolicy(config, policy, where) {
|
|
21
|
+
for (const name of [policy.model, ...(policy.fallbacks ?? [])]) {
|
|
22
|
+
if (name !== undefined && !knownModel(config, name))
|
|
23
|
+
throw configError(`${where} names model "${name}", which no provider serves`);
|
|
24
|
+
}
|
|
25
|
+
positive(policy.maxAttempts, `${where}.maxAttempts`);
|
|
26
|
+
positive(policy.timeoutMs, `${where}.timeoutMs`);
|
|
27
|
+
positive(policy.deadlineMs, `${where}.deadlineMs`);
|
|
28
|
+
positive(policy.budget?.maxCost, `${where}.budget.maxCost`, true);
|
|
29
|
+
for (const id of policy.providers ?? []) {
|
|
30
|
+
if (!config.providers.some((provider) => provider.id === id))
|
|
31
|
+
throw configError(`${where} names provider "${id}", which is not configured`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
|
|
35
|
+
export function validateConfig(config) {
|
|
36
|
+
if (config.providers.length === 0)
|
|
37
|
+
throw configError("the pool has no providers");
|
|
38
|
+
const ids = new Set();
|
|
39
|
+
for (const provider of config.providers) {
|
|
40
|
+
if (provider.id === "" || provider.id.includes("/"))
|
|
41
|
+
throw configError(`provider id "${provider.id}" must be non-empty and without "/"`);
|
|
42
|
+
if (ids.has(provider.id))
|
|
43
|
+
throw configError(`provider "${provider.id}" is declared twice`);
|
|
44
|
+
ids.add(provider.id);
|
|
45
|
+
let url;
|
|
46
|
+
try {
|
|
47
|
+
url = new URL(provider.baseUrl);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
throw configError(`provider "${provider.id}" has an invalid baseUrl`);
|
|
51
|
+
}
|
|
52
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
53
|
+
throw configError(`provider "${provider.id}" baseUrl must be http or https`);
|
|
54
|
+
positive(provider.weight, `provider "${provider.id}" weight`);
|
|
55
|
+
positive(provider.maxConcurrency, `provider "${provider.id}" maxConcurrency`);
|
|
56
|
+
if (provider.keys.length === 0)
|
|
57
|
+
throw configError(`provider "${provider.id}" has no keys`);
|
|
58
|
+
if (provider.models.length === 0)
|
|
59
|
+
throw configError(`provider "${provider.id}" has no models`);
|
|
60
|
+
const aliases = new Set();
|
|
61
|
+
for (const key of provider.keys) {
|
|
62
|
+
if (key.alias === "" || key.alias.includes("/"))
|
|
63
|
+
throw configError(`key alias "${key.alias}" of provider "${provider.id}" must be non-empty and without "/"`);
|
|
64
|
+
if (aliases.has(key.alias))
|
|
65
|
+
throw configError(`provider "${provider.id}" declares key "${key.alias}" twice`);
|
|
66
|
+
aliases.add(key.alias);
|
|
67
|
+
const sources = ("env" in key ? 1 : 0) + ("resolve" in key ? 1 : 0);
|
|
68
|
+
if (sources !== 1)
|
|
69
|
+
throw configError(`key "${key.alias}" of provider "${provider.id}" needs exactly one of env or resolve`);
|
|
70
|
+
positive(key.weight, `key "${key.alias}" weight`);
|
|
71
|
+
positive(key.maxConcurrency, `key "${key.alias}" maxConcurrency`);
|
|
72
|
+
}
|
|
73
|
+
const models = new Set();
|
|
74
|
+
for (const model of provider.models) {
|
|
75
|
+
if (model.id === "")
|
|
76
|
+
throw configError(`provider "${provider.id}" has a model with an empty id`);
|
|
77
|
+
if (models.has(model.id))
|
|
78
|
+
throw configError(`provider "${provider.id}" declares model "${model.id}" twice`);
|
|
79
|
+
models.add(model.id);
|
|
80
|
+
positive(model.weight, `model "${model.id}" weight`);
|
|
81
|
+
positive(model.contextWindow, `model "${model.id}" contextWindow`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (config.defaults !== undefined)
|
|
85
|
+
validatePolicy(config, config.defaults, "defaults");
|
|
86
|
+
for (const [name, policy] of Object.entries(config.profiles ?? {}))
|
|
87
|
+
validatePolicy(config, policy, `profile "${name}"`);
|
|
88
|
+
}
|
|
89
|
+
/** Applies the defaults of every pool-wide setting. */
|
|
90
|
+
export function resolveSettings(config) {
|
|
91
|
+
const r = config.resilience ?? {};
|
|
92
|
+
const s = config.selection ?? {};
|
|
93
|
+
return {
|
|
94
|
+
keyBreaker: {
|
|
95
|
+
failureThreshold: r.keyBreaker?.failureThreshold ?? 3,
|
|
96
|
+
cooldownMs: r.keyBreaker?.cooldownMs ?? 15_000,
|
|
97
|
+
maxCooldownMs: r.keyBreaker?.maxCooldownMs ?? 300_000,
|
|
98
|
+
},
|
|
99
|
+
providerBreaker: {
|
|
100
|
+
failureThreshold: r.providerBreaker?.failureThreshold ?? 8,
|
|
101
|
+
cooldownMs: r.providerBreaker?.cooldownMs ?? 30_000,
|
|
102
|
+
maxCooldownMs: r.providerBreaker?.maxCooldownMs ?? 300_000,
|
|
103
|
+
},
|
|
104
|
+
backoff: { baseMs: r.backoff?.baseMs ?? 100, factor: r.backoff?.factor ?? 2, maxMs: r.backoff?.maxMs ?? 5_000 },
|
|
105
|
+
authDisableMs: r.authDisableMs === undefined ? null : r.authDisableMs,
|
|
106
|
+
rateLimitCooldownMs: r.rateLimitCooldownMs ?? 10_000,
|
|
107
|
+
latencyTolerance: s.latencyTolerance ?? 1.5,
|
|
108
|
+
latencySlackMs: s.latencySlackMs ?? 50,
|
|
109
|
+
ewmaAlpha: s.ewmaAlpha ?? 0.3,
|
|
110
|
+
lowHeadroom: s.lowHeadroom ?? 1,
|
|
111
|
+
includeContent: config.telemetry?.includeContent ?? false,
|
|
112
|
+
namespace: config.storeNamespace ?? "ai-router",
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
116
|
+
export function resolvePolicy(config, options, requestModel) {
|
|
117
|
+
let profile = {};
|
|
118
|
+
if (options.profile !== undefined) {
|
|
119
|
+
const found = config.profiles?.[options.profile];
|
|
120
|
+
if (found === undefined)
|
|
121
|
+
throw configError(`profile "${options.profile}" is not configured`);
|
|
122
|
+
profile = found;
|
|
123
|
+
}
|
|
124
|
+
const merged = { ...config.defaults, ...profile, ...options.policy };
|
|
125
|
+
if (options.policy !== undefined)
|
|
126
|
+
validatePolicy(config, options.policy, "policy");
|
|
127
|
+
// A call's policy, then its chosen profile, then the request's own model, then the defaults.
|
|
128
|
+
const preferred = options.policy?.model ?? profile.model ?? requestModel ?? config.defaults?.model;
|
|
129
|
+
if (preferred === undefined)
|
|
130
|
+
throw configError("no model: set request.model, a policy model or a profile model");
|
|
131
|
+
if (!knownModel(config, preferred))
|
|
132
|
+
throw configError(`model "${preferred}" is not served by any provider`);
|
|
133
|
+
const models = [preferred];
|
|
134
|
+
if (merged.allowFallback !== false) {
|
|
135
|
+
for (const name of merged.fallbacks ?? [])
|
|
136
|
+
if (!models.includes(name))
|
|
137
|
+
models.push(name);
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
models,
|
|
141
|
+
maxAttempts: merged.maxAttempts ?? 4,
|
|
142
|
+
timeoutMs: merged.timeoutMs ?? 60_000,
|
|
143
|
+
deadlineMs: merged.deadlineMs ?? 120_000,
|
|
144
|
+
maxCost: merged.budget?.maxCost ?? null,
|
|
145
|
+
providers: merged.providers === undefined ? null : new Set(merged.providers),
|
|
146
|
+
require: merged.require ?? [],
|
|
147
|
+
};
|
|
148
|
+
}
|