@starci/ai-router 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +13 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.0 - 2026-10-31
|
|
4
|
+
|
|
5
|
+
Many providers below, each with an array of keys, behind one virtual key per app above; a key whose quota is spent or
|
|
6
|
+
whose secret is refused is blacklisted. [docs/design.md](docs/design.md) holds the provider matrix (read from each
|
|
7
|
+
provider's docs on 2026-10-31), the classification table, the blacklist lifecycle, profiles and key scopes, the
|
|
8
|
+
Anthropic adapter and the env naming.
|
|
9
|
+
|
|
10
|
+
- Added: provider kinds, `ProviderConfig.kind` (`anthropic`, `openrouter`, `qwen`, `deepseek`, `openai`, `gemini`, `groq`, `openai-compatible`), each with its default base URL, auth header, embeddings support and error classification (`PROVIDER_PRESETS`); `baseUrl` is optional for a kind that has one.
|
|
11
|
+
- Added: the Anthropic Messages adapter: OpenAI chat-completions in, `/v1/messages` with `x-api-key` and `anthropic-version` out, and back, for chat, SSE streaming (text and `tool_use` deltas, finish reason, usage, error events, a stream cut before `message_stop`), tools and `tool_choice`, tool results, images, `json_schema` as structured output, cache tokens in usage; `defaultMaxTokens` for the `max_tokens` Anthropic requires. Anthropic `anthropic-ratelimit-*` headers (RFC 3339 resets) and OpenRouter's epoch-millisecond `X-RateLimit-Reset` feed headroom.
|
|
12
|
+
- Added: the error class `quota` and per-provider classification (`classifyFailure`, `readProviderError`) of every error shape the providers document: OpenAI, Anthropic, OpenRouter (402 by `limit_source`, errors in a 200 body and mid-stream), DashScope codes, DeepSeek, Gemini (object or list, per-minute vs per-day), Groq (498, daily requests).
|
|
13
|
+
- Added: the key blacklist. A spent quota blacklists the key until the provider's window ends (next UTC month for Anthropic's spend cap and OpenAI's monthly limits, next Pacific midnight for Gemini's daily quota, next UTC midnight for OpenRouter's free daily cap, Groq's request reset), else `resilience.quotaBlacklistMs` (default 6 hours); a refused or missing secret until the secret changes. A 64-bit fingerprint of the refused secret is kept (never the secret), and a new secret lifts the blacklist at the next pick with the new `key-restored` event. `key-disabled` carries the provider's `code`; `KeyHealth.disabledUntil`.
|
|
14
|
+
- Added: when every key that could serve is blacklisted (or its provider past its budget), the call fails at once with class `quota` or `auth`, naming the keys, with `retryAt` at the earliest return; the gateway answers 503 `pool_quota_exhausted` or 503 `pool_auth_failed` with `Retry-After` (300 seconds when no return is known).
|
|
15
|
+
- Added: durable state stores, so a restart does not hammer dead keys: `openSqliteStateStore(path)` and `redisStateStore(client)` over a five-command `RedisLike`; `stateStore: { type: sqlite, path }` in the config file; `toHealthRecord` and `parseHealthRecord` for an adapter; `MemoryStateStore` and `EMPTY_RECORD` are exported again.
|
|
16
|
+
- Added: profile candidates, `ModelPolicy.candidates`: ordered tiers of `{ model, provider?, weight? }`, a tier that is a list sharing its load by weight; `pinned` profiles whose candidates must be one model `identity` and `dimensions` (`ModelConfig.identity`, `ModelConfig.dimensions`), which a call cannot override; an embeddings vector of another length fails its lane; every embeddings call stays on one model identity, and the gateway gives a direct embeddings model no fallback. `CatalogProfile.pinned` and `serves`; a profile on the wrong route is 400 `model_not_supported`.
|
|
17
|
+
- Added: provider budgets, `ProviderConfig.budget` (`costPerDay`, `costPerMonth`), kept in the store; `ProviderHealth.spend` and `kind`.
|
|
18
|
+
- Added: config from env: `poolFromEnv`, `providersFromEnv` and `keysFromEnv` read `<PREFIX>_API_KEY`, `_2` ... `_<n>`, their `_FILE` twins, a `<PREFIX>_API_KEYS` list and `<PREFIX>_BASE_URL` (prefixes ANTHROPIC, OPENROUTER, DASHSCOPE, DEEPSEEK, OPENAI, GEMINI, GROQ) as references read at each attempt, keep the providers that have keys and cut every profile to what they serve; `virtualKeyFromEnv` mounts the app's `STARCI_AI_KEY` (or `STARCI_AI_KEY_FILE`); `keysFromEnv: <PREFIX>` on a config-file provider.
|
|
19
|
+
- Added: the fake provider answers `/messages` in the Anthropic shape (unary and SSE), reads `x-api-key`, scripts `toolCalls`, and `PROVIDER_ERRORS` (from `@starci/ai-router/testing`) holds each provider's real error answers.
|
|
20
|
+
- Fixed: a provider or call header differing from the router's own only in case (`Anthropic-Version`) is one header, not two.
|
|
21
|
+
|
|
22
|
+
### Migration from 0.4
|
|
23
|
+
|
|
24
|
+
- A 0.4 pool routes as before: `kind` defaults to `openai-compatible`, whose classification is 0.4's plus `quota` for 402 and the quota codes.
|
|
25
|
+
- `ErrorClass` has `quota`: a spent quota was a `rate_limit` with a 10-second cooldown and is now a blacklist; a `switch` over `errorClass` adds the case.
|
|
26
|
+
- When every key is blacklisted the router fails with `quota` or `auth`, no longer `no_candidate`, and the gateway answers 503 where `pool_auth_failed` was 502.
|
|
27
|
+
- A key refused for its secret comes back when the secret changes, without `reset()`; a missing secret when it is set.
|
|
28
|
+
- `KeyDisabledEvent.reason` adds `quota`, the event may carry `code`; `KeyHealth` adds `disabledUntil`, `ProviderHealth` `kind` (and `spend`), `CatalogProfile` `pinned` and `serves`, `RouterEvents` `key-restored`, `HealthRecord` its blacklist and budget fields (records 0.4 stored read with them empty).
|
|
29
|
+
- Embeddings never fall back to another model identity.
|
|
30
|
+
- Apps replace a hand-rolled `<PROVIDER>_API_KEY` plus `_2` ... `_9` list with `poolFromEnv`, keep `STARCI_AI_KEY` as their one virtual key (`virtualKeyFromEnv`), pass no admin token, and pass a durable `store`.
|
|
31
|
+
|
|
32
|
+
## 0.4.1 - 2026-10-30
|
|
33
|
+
|
|
34
|
+
- Changed: the lint configuration is TypeScript (eslint.config.ts, loaded through jiti 2.7.0, an exact devDependency) and typechecked with the package; no runtime change.
|
|
35
|
+
|
|
3
36
|
## 0.4.0 - 2026-10-30
|
|
4
37
|
|
|
5
38
|
- Added: mounted keys, `GatewayOptions.mountedKeys` (and so `AiRouterGatewayModule`'s options): virtual keys from secrets the app already holds (`{ label, secret: string | (() => string), owner?, profiles?, models?, allowFallback?, quota?, enabled? }`, the `MountedKeySpec` type), so a deploy needs no `keys create`. Each secret is read once at startup, must start with `sk-starci-<label>-` followed by at least 32 letters, digits, `_` or `-`, and is kept only as its SHA-256 with its bindings in memory; a mounted key is never written to the key store. A bearer token is matched against the mounted keys before the store. Each mounted key has the stable id `key_` + 16 hex of `sha256("mounted:<label>")`, so its quota counters and usage rows survive a new secret and every deploy.
|
package/README.md
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# @starci/ai-router
|
|
2
2
|
|
|
3
|
-
A router for OpenAI-compatible calls across
|
|
3
|
+
A router for OpenAI-compatible calls across many providers, each with an array of keys, behind one virtual key per app.
|
|
4
|
+
It load-balances below (Anthropic, OpenRouter, Qwen, DeepSeek, OpenAI, Gemini, Groq and any OpenAI-compatible endpoint),
|
|
5
|
+
blacklists a key whose quota is spent or whose secret is refused, and exposes one OpenAI-compatible API above. It is
|
|
6
|
+
usable three ways over one engine:
|
|
4
7
|
|
|
5
8
|
- **as a gateway service** (`run.js serve`): an OpenAI-compatible HTTP endpoint (`/v1/chat/completions`, streaming
|
|
6
9
|
included, `/v1/embeddings`, `/v1/models`) with its own virtual API keys, per-key quotas and usage accounting, so any
|
|
@@ -9,15 +12,19 @@ A router for OpenAI-compatible calls across a pool of keys, models and providers
|
|
|
9
12
|
and `embeddings`; `AiRouterGatewayModule` hosts the gateway in the same process;
|
|
10
13
|
- **as a plain Node library**: `createRouter(config)` from `@starci/ai-router/gateway`, with no Nest at all.
|
|
11
14
|
|
|
12
|
-
The router picks a healthy key for each attempt, backs off what is failing, honours rate limits,
|
|
13
|
-
|
|
14
|
-
|
|
15
|
+
The router picks a healthy key for each attempt, backs off what is failing, honours rate limits, blacklists a key whose
|
|
16
|
+
quota or credit is spent until its window ends (or its secret changes), retries on a different key or provider, falls
|
|
17
|
+
back across providers and models by profile, keeps embeddings on one model, and tells you what it did without ever
|
|
18
|
+
exposing a key or a prompt. [docs/design.md](docs/design.md) is the design: the provider matrix, the error
|
|
19
|
+
classification per provider, the blacklist lifecycle, profiles and key scopes, the Anthropic adapter and the env naming.
|
|
15
20
|
|
|
16
21
|
- ESM, typed, Node 20 or newer (the durable key store needs Node 22.13 or newer for `node:sqlite`). One runtime
|
|
17
22
|
dependency, `yaml`, for the config file. `@nestjs/common`, `@nestjs/core` (11 or 12) and `reflect-metadata` are
|
|
18
23
|
optional peer dependencies: the root entry needs them, `@starci/ai-router/gateway`, `@starci/ai-router/cli` and the `run.js` process entry do not.
|
|
19
|
-
- OpenAI chat-completions and embeddings shapes in and out
|
|
20
|
-
|
|
24
|
+
- OpenAI chat-completions and embeddings shapes in and out. A provider's `kind` sets its protocol, auth header, default
|
|
25
|
+
base URL and error classification: `anthropic` (the Messages API, through a protocol adapter for chat, streaming,
|
|
26
|
+
tools and JSON), `openrouter`, `qwen` (DashScope compatible mode), `deepseek`, `openai`, `gemini` (its OpenAI
|
|
27
|
+
endpoint), `groq`, and `openai-compatible` (the default: Ollama, vLLM, Together, LiteLLM...).
|
|
21
28
|
- Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
|
|
22
29
|
everywhere else.
|
|
23
30
|
|
|
@@ -35,6 +42,55 @@ npm install @starci/ai-router # the gat
|
|
|
35
42
|
npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata # in a Nest app
|
|
36
43
|
```
|
|
37
44
|
|
|
45
|
+
## Many providers below, one virtual key above
|
|
46
|
+
|
|
47
|
+
An app declares every provider it may use, each with its models; `poolFromEnv` keeps the ones whose keys the
|
|
48
|
+
environment holds (`<PREFIX>_API_KEY`, `<PREFIX>_API_KEY_2`, ..., their `_FILE` twins, or a `<PREFIX>_API_KEYS` list)
|
|
49
|
+
and cuts every profile down to what they serve. Clients send `model: "work"` or `model: "embed"` with the app's one
|
|
50
|
+
virtual key, `STARCI_AI_KEY`.
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
import { AiRouterGatewayModule, AiRouterModule, openSqliteKeyStore, openSqliteStateStore, poolFromEnv, virtualKeyFromEnv } from "@starci/ai-router"
|
|
54
|
+
|
|
55
|
+
AiRouterModule.registerAsync({
|
|
56
|
+
isGlobal: true,
|
|
57
|
+
useFactory: async () =>
|
|
58
|
+
poolFromEnv({
|
|
59
|
+
providers: [
|
|
60
|
+
{ kind: "anthropic", models: [{ id: "claude-sonnet-5-5", cost: { input: 3, output: 15 } }] },
|
|
61
|
+
{ kind: "openrouter", models: [{ id: "anthropic/claude-sonnet-5-5" }, { id: "openai/text-embedding-3-small", identity: "text-embedding-3-small", dimensions: 1536, capabilities: ["embeddings"] }] },
|
|
62
|
+
{ kind: "deepseek", models: [{ id: "deepseek-v4-pro" }] },
|
|
63
|
+
{ kind: "qwen", models: [{ id: "qwen3.8-max" }] },
|
|
64
|
+
{ kind: "openai", models: [{ id: "text-embedding-3-small", dimensions: 1536, capabilities: ["embeddings"] }] },
|
|
65
|
+
],
|
|
66
|
+
profiles: {
|
|
67
|
+
// Tiers in order; a list is one tier sharing its load by weight.
|
|
68
|
+
work: { candidates: [[{ provider: "anthropic", model: "claude-sonnet-5-5", weight: 3 }, { provider: "openrouter", model: "anthropic/claude-sonnet-5-5" }], { provider: "deepseek", model: "deepseek-v4-pro" }, { provider: "qwen", model: "qwen3.8-max" }] },
|
|
69
|
+
// Pinned: keys and providers rotate, the model (and its vector space) never changes.
|
|
70
|
+
embed: { pinned: true, candidates: [[{ provider: "openai", model: "text-embedding-3-small" }, { provider: "openrouter", model: "openai/text-embedding-3-small" }]] },
|
|
71
|
+
},
|
|
72
|
+
// Durable: a blacklisted key stays out across a restart.
|
|
73
|
+
store: await openSqliteStateStore("/data/ai-router-state.sqlite"),
|
|
74
|
+
}),
|
|
75
|
+
})
|
|
76
|
+
AiRouterGatewayModule.registerAsync({
|
|
77
|
+
useFactory: async () => ({ keys: await openSqliteKeyStore("/data/keys.sqlite"), mountedKeys: [await virtualKeyFromEnv({ label: "app", quota: { costPerMonth: 50 } })], listen: true }),
|
|
78
|
+
})
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
| env variable | meaning |
|
|
82
|
+
|---|---|
|
|
83
|
+
| `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `DASHSCOPE_API_KEY` (Qwen), `DEEPSEEK_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY` | the first key of each provider, alias `<kind>-1` |
|
|
84
|
+
| `<PREFIX>_API_KEY_2` ... `<PREFIX>_API_KEY_<n>` | more keys, aliases `<kind>-<n>`; gaps allowed, no upper bound |
|
|
85
|
+
| `<PREFIX>_API_KEY_FILE`, `<PREFIX>_API_KEY_<n>_FILE` | a key mounted as a file (its first line) |
|
|
86
|
+
| `<PREFIX>_API_KEYS` | several keys in one variable, split at commas and whitespace |
|
|
87
|
+
| `<PREFIX>_BASE_URL` | another endpoint: a Qwen region or workspace host, an Ollama host |
|
|
88
|
+
| `STARCI_AI_KEY` (or `STARCI_AI_KEY_FILE`) | the app's virtual key, `sk-starci-<label>-...`, bound to `work` and `embed` |
|
|
89
|
+
|
|
90
|
+
A virtual key carries the profiles it may call and, optionally, an allowlist of exact model ids (`models`, empty by
|
|
91
|
+
default). A model it names that is neither is refused 403 `model_not_allowed`, never remapped; `/v1/models` lists only
|
|
92
|
+
what it may call; its quotas count every call whatever the model. See [key scopes](docs/design.md#key-scopes).
|
|
93
|
+
|
|
38
94
|
## Run it as a gateway
|
|
39
95
|
|
|
40
96
|
One process (one container) holds the pool, the virtual keys and the admin API. The package declares no `bin` (the
|
|
@@ -61,9 +117,14 @@ gateway:
|
|
|
61
117
|
adminToken: { env: AI_ROUTER_ADMIN_TOKEN } # absent: the admin API is off
|
|
62
118
|
keyStore: { type: sqlite, path: /data/keys.sqlite } # or { type: memory }; a relative path is from this file
|
|
63
119
|
maxBodyBytes: 4194304 # default 4 MiB
|
|
120
|
+
stateStore: { type: sqlite, path: /data/state.sqlite } # breakers and blacklists survive a restart; default memory
|
|
64
121
|
providers:
|
|
122
|
+
- id: anthropic
|
|
123
|
+
kind: anthropic # base URL, x-api-key auth and the Messages adapter come with the kind
|
|
124
|
+
keysFromEnv: ANTHROPIC # ANTHROPIC_API_KEY, ANTHROPIC_API_KEY_2, ... and their _FILE twins
|
|
125
|
+
models: [{ id: claude-sonnet-5-5, aliases: [smart] }]
|
|
65
126
|
- id: openai
|
|
66
|
-
|
|
127
|
+
kind: openai
|
|
67
128
|
keys:
|
|
68
129
|
- { alias: team-a, env: OPENAI_KEY_A }
|
|
69
130
|
- { alias: team-b, file: /run/secrets/openai_key_b, maxConcurrency: 8 }
|
|
@@ -110,8 +171,9 @@ Errors are OpenAI error objects, `{ "error": { "message", "type", "param", "code
|
|
|
110
171
|
| 413 | `request_too_large` | a body past `maxBodyBytes` |
|
|
111
172
|
| 429 | `requests_per_minute_exceeded`, `tokens_per_day_exceeded`, `cost_per_day_exceeded`, `cost_per_month_exceeded` | the key's quota, with `Retry-After` |
|
|
112
173
|
| 4xx | `upstream_invalid_request` | the provider refused the request (its status and message, e.g. a context-length error) |
|
|
113
|
-
| 502 | `upstream_error
|
|
174
|
+
| 502 | `upstream_error` | every attempt failed upstream |
|
|
114
175
|
| 503 | `pool_unavailable`, `pool_rate_limited` | no lane can serve it, or every key is rate limited (`Retry-After` when known) |
|
|
176
|
+
| 503 | `pool_quota_exhausted`, `pool_auth_failed` | every key that could serve is blacklisted: its quota or credit spent, or its secret refused (`Retry-After` until the first returns, else 300) |
|
|
115
177
|
| 504 | `pool_timeout` | the policy's deadline passed |
|
|
116
178
|
|
|
117
179
|
### Virtual keys
|
|
@@ -405,10 +467,10 @@ the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in
|
|
|
405
467
|
| `chat(request, options?)` | sends a chat-completions request; resolves to the OpenAI completion plus `routing` |
|
|
406
468
|
| `stream(request, options?)` | returns an async iterable of OpenAI chunks, with `requestId`; see [Streaming](#streaming) |
|
|
407
469
|
| `embeddings(request, options?)` | sends an embeddings request; resolves to the OpenAI response plus `routing` |
|
|
408
|
-
| `health()` | reads every breaker,
|
|
470
|
+
| `health()` | reads every breaker, blacklisted key (reason and until), headroom, budget spend and latency EWMA |
|
|
409
471
|
| `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
|
|
410
|
-
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-
|
|
411
|
-
| `reset(target?)` | closes breakers,
|
|
472
|
+
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker`, `key-disabled` or `key-restored` until shutdown; returns the unsubscribe function |
|
|
473
|
+
| `reset(target?)` | closes breakers, lifts blacklists and clears headroom for everything, one provider or one key |
|
|
412
474
|
| `catalog()` | the pool's profiles (resolved model and fallbacks) and models (ids, aliases, providers), with no key |
|
|
413
475
|
|
|
414
476
|
`options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
|
|
@@ -447,19 +509,23 @@ A named registration provides no default instance; `@InjectAiRouterService()` wi
|
|
|
447
509
|
| field | meaning | default |
|
|
448
510
|
|---|---|---|
|
|
449
511
|
| `providers[].id` | a name, unique, without `/` | required |
|
|
450
|
-
| `providers[].
|
|
451
|
-
| `providers[].
|
|
512
|
+
| `providers[].kind` | `anthropic`, `openrouter`, `qwen`, `deepseek`, `openai`, `gemini`, `groq` or `openai-compatible`: protocol, auth header, default base URL, embeddings support and error classification ([provider matrix](docs/design.md#provider-matrix)) | `openai-compatible` |
|
|
513
|
+
| `providers[].baseUrl` | the URL the API paths hang from (`.../v1`) | the kind's; required for `openai-compatible` |
|
|
514
|
+
| `providers[].headers` | extra headers for every request (the auth header, `authorization` or `x-api-key`, is ignored here) | none |
|
|
452
515
|
| `providers[].weight` | multiplies the round-robin share of every lane of the provider | 1 |
|
|
453
516
|
| `providers[].maxConcurrency` | in-flight requests across the provider's keys, in this process | unbounded |
|
|
454
|
-
| `providers[].
|
|
455
|
-
| `providers[].
|
|
517
|
+
| `providers[].defaultMaxTokens` | the `max_tokens` sent to Anthropic, which requires one, when a request sets none | 4096 |
|
|
518
|
+
| `providers[].budget` | `{ costPerDay?, costPerMonth? }` in the models' cost unit; past it the provider is out until the next UTC day or month | none |
|
|
519
|
+
| `providers[].keys[]` | `{ alias, env }` or `{ alias, resolve }`, plus `weight`, `maxConcurrency` (`keysFromEnv` builds them) | at least one |
|
|
520
|
+
| `providers[].models[]` | `{ id, aliases?, contextWindow?, cost?, capabilities?, weight?, identity?, dimensions? }` | at least one |
|
|
456
521
|
| `defaults` | the policy every call starts from | none |
|
|
457
522
|
| `profiles` | named policies a call selects with `{ profile }` | none |
|
|
458
523
|
| `resilience.keyBreaker` | `{ failureThreshold, cooldownMs, maxCooldownMs }` per key | 3, 15000, 300000 |
|
|
459
524
|
| `resilience.providerBreaker` | the same per provider, fed by all its keys | 8, 30000, 300000 |
|
|
460
525
|
| `resilience.backoff` | `{ baseMs, factor, maxMs }` between attempts after a transient failure | 100, 2, 5000 |
|
|
461
|
-
| `resilience.authDisableMs` | how long a
|
|
526
|
+
| `resilience.authDisableMs` | how long a key whose secret was refused stays out; `null` until its secret changes or `reset()` | `null` |
|
|
462
527
|
| `resilience.rateLimitCooldownMs` | a 429's cooldown when it has no `retry-after` | 10000 |
|
|
528
|
+
| `resilience.quotaBlacklistMs` | how long a key whose quota is spent stays out when the provider names no window | 21600000 (6 h) |
|
|
463
529
|
| `selection.latencyTolerance`, `latencySlackMs` | how far above the best latency EWMA a lane still ties | 1.5, 50 |
|
|
464
530
|
| `selection.ewmaAlpha` | the weight of the newest latency sample | 0.3 |
|
|
465
531
|
| `selection.lowHeadroom` | remaining requests at or below which a lane is used only when nothing else is | 1 |
|
|
@@ -471,7 +537,9 @@ A named registration provides no default instance; `@InjectAiRouterService()` wi
|
|
|
471
537
|
A model's `id` is the name sent to the provider; a caller may name it by `id` or any alias, and an alias shared by
|
|
472
538
|
several providers makes them interchangeable for that call. `cost` is per one million tokens, `input` and `output`, in
|
|
473
539
|
whatever unit you budget in. `capabilities` defaults to `chat`, `streaming`, `tools` and `json`; an embeddings model
|
|
474
|
-
declares `["embeddings"]
|
|
540
|
+
declares `["embeddings"]` (refused on a kind with no embeddings API: `anthropic`, `deepseek`, `groq`). `identity` names
|
|
541
|
+
one model served under different ids (`openai/text-embedding-3-small` at OpenRouter is `text-embedding-3-small`), and
|
|
542
|
+
`dimensions` the vector length an embeddings model answers: a response of another length fails the attempt.
|
|
475
543
|
|
|
476
544
|
## Policies
|
|
477
545
|
|
|
@@ -490,6 +558,11 @@ defaults'.
|
|
|
490
558
|
| `budget.maxCost` | a model whose worst-case cost for this request is higher is no candidate | none |
|
|
491
559
|
| `providers` | restrict the call to these provider ids | all |
|
|
492
560
|
| `require` | capabilities needed beyond those the request implies | none |
|
|
561
|
+
| `candidates` | in place of `model` and `fallbacks`: ordered tiers of `{ model, provider?, weight? }`; a tier that is a list shares its load by weight; `allowFallback: false` keeps the first tier | none |
|
|
562
|
+
| `pinned` | a profile whose candidates are all one model `identity` and `dimensions` (refused at startup otherwise); a call cannot name another model | `false` |
|
|
563
|
+
|
|
564
|
+
A call's own `model` wins over a profile's candidates, except on a pinned profile. An embeddings call always stays on
|
|
565
|
+
the model identity of its first lane: another model is another vector space.
|
|
493
566
|
|
|
494
567
|
The request implies its own needs: `tools` needs `tools`, a non-text `response_format` needs `json`, `stream()` needs
|
|
495
568
|
`streaming`, `embeddings()` needs `embeddings`. A model is also no candidate when its `contextWindow` is smaller than
|
|
@@ -520,10 +593,11 @@ waits for it rather than failing. When none will be, it fails at once with the c
|
|
|
520
593
|
|
|
521
594
|
| what happened | class | effect | retried |
|
|
522
595
|
|---|---|---|---|
|
|
523
|
-
| 429, or a rate
|
|
524
|
-
|
|
|
525
|
-
|
|
|
526
|
-
|
|
|
596
|
+
| 429, or a rate error object | `rate_limit` | the lane (key and model) cools down until `retry-after` / `retry-after-ms`, else `rateLimitCooldownMs` | yes, at once, on another lane |
|
|
597
|
+
| 402, or a quota, credit or billing code or message (`insufficient_quota`, `billing_error`, `Arrearage`, ...; per provider in [the classification table](docs/design.md#error-classification)) | `quota` | the key is blacklisted until the provider's window ends (a monthly cap, a daily free tier), else `quotaBlacklistMs`, or until its secret changes; `key-disabled` event with the provider's `code` | yes, at once, on another key |
|
|
598
|
+
| 401, 403, an auth error object | `auth` | the key is blacklisted until its secret changes (or `authDisableMs`); `key-disabled` event | yes, at once, on another key |
|
|
599
|
+
| a resolver that returns nothing or throws, an unset env var | `auth` | the key is blacklisted with reason `missing` until a secret is set; nothing is sent | yes |
|
|
600
|
+
| 408, 409, 425, 498, 5xx (Anthropic's 529) | `transient` | counts toward the key's and the provider's breakers; a `retry-after` also cools the lane | yes, after backoff |
|
|
527
601
|
| connection failure | `network` | as transient | yes, after backoff |
|
|
528
602
|
| attempt timeout | `timeout` | as transient | yes, after backoff |
|
|
529
603
|
| broken or malformed stream before the first token | `stream` (or the streamed error's class) | as transient | yes, after backoff |
|
|
@@ -538,8 +612,13 @@ A successful probe closes it; a failed one reopens it with the cooldown doubled,
|
|
|
538
612
|
invalid requests and cancellations never count against a breaker.
|
|
539
613
|
|
|
540
614
|
Backoff after a transient failure is `min(maxMs, baseMs * factor^(n-1))` scaled by a random factor in `[0.5, 1)`, and
|
|
541
|
-
the next attempt goes to a different lane by the tiers above. Rate and auth failures belong to one key, so the
|
|
542
|
-
is tried without waiting.
|
|
615
|
+
the next attempt goes to a different lane by the tiers above. Rate, quota and auth failures belong to one key, so the
|
|
616
|
+
next key is tried without waiting.
|
|
617
|
+
|
|
618
|
+
A blacklisted key keeps a 64-bit fingerprint of the secret it was refused with (never the secret): each pick resolves
|
|
619
|
+
its secret again, and a new one (a rotated variable, a re-sealed file) lifts the blacklist at once with a
|
|
620
|
+
`key-restored` event. When every key that could serve is blacklisted and none returns before the deadline, the call
|
|
621
|
+
fails at once with class `quota` or `auth`, naming the keys, with `retryAt` at the earliest return.
|
|
543
622
|
|
|
544
623
|
## Streaming
|
|
545
624
|
|
|
@@ -581,7 +660,8 @@ AiRouterModule.register({
|
|
|
581
660
|
| `attempt` | every attempt | `requestId`, `operation`, `attempt`, `provider`, `key` alias, `model`, `latencyMs`, `outcome`, `errorClass`, `status`, `usage`, `cost` |
|
|
582
661
|
| `request` | every call, once | `requestId`, `operation`, `outcome`, `attempts`, `latencyMs`, the answering lane or the `errorClass` |
|
|
583
662
|
| `breaker` | a breaker changes state | `scope` (`key` or `provider`), `provider`, `key`, `from`, `to` |
|
|
584
|
-
| `key-disabled` | a key is
|
|
663
|
+
| `key-disabled` | a key is blacklisted | `provider`, `key`, `reason` (`auth`, `missing` or `quota`), `until` (null: until its secret changes or `reset()`), `code` (the provider's error code, when it sent one) |
|
|
664
|
+
| `key-restored` | a blacklisted key's secret changed | `provider`, `key`, `was` (the reason it was out), `reason` (`secret_changed`) |
|
|
585
665
|
|
|
586
666
|
Events never carry message content unless `telemetry.includeContent` is `true`; then `attempt` events add `content`
|
|
587
667
|
with the request body and, for `chat` and `embeddings`, the response. A handler that throws is isolated from the call.
|
|
@@ -589,13 +669,19 @@ The module itself logs nothing.
|
|
|
589
669
|
|
|
590
670
|
## Sharing state between processes
|
|
591
671
|
|
|
592
|
-
Breakers,
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
672
|
+
Breakers, blacklisted keys, headroom and budgets live in the `store` option; the default keeps them in memory, so a
|
|
673
|
+
restart forgets which keys are dead and tries them again. Two durable stores ship:
|
|
674
|
+
|
|
675
|
+
- `openSqliteStateStore(path)` (`node:sqlite`, Node 22.13 or newer): one file, which processes on one host may share
|
|
676
|
+
(it may be the key store's file); `stateStore: { type: sqlite, path }` in the config file.
|
|
677
|
+
- `redisStateStore(client)`: every gateway process of an app sees the same blacklist; `client` is five commands
|
|
678
|
+
(`get`, `set`, `setIfAbsent` as `SET NX PX`, `del`, `keys`), a few lines over node-redis or ioredis.
|
|
679
|
+
|
|
680
|
+
Another adapter implements the exported `StateStore` type (`get`, `set`, `delete`, `keys(prefix)`, and an atomic
|
|
681
|
+
`lease(key, ttlMs)` / `release(key)` pair for the half-open probe), parsing what it stored with `toHealthRecord` or
|
|
682
|
+
`parseHealthRecord`. Methods may return values or promises. In-flight counts, latency EWMAs and round-robin positions
|
|
683
|
+
stay per process. Updates are read-modify-write, so two processes failing at the same instant may count one failure
|
|
684
|
+
between them: breakers are a health signal, not a ledger.
|
|
599
685
|
|
|
600
686
|
## Testing
|
|
601
687
|
|
|
@@ -629,19 +715,25 @@ const reply = await app.get(AiRouterService).chat({ model: "m", messages: [{ rol
|
|
|
629
715
|
await app.close()
|
|
630
716
|
```
|
|
631
717
|
|
|
632
|
-
A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `
|
|
633
|
-
`stream: { chunks, delayMs, failAfter, failWith: "network" |
|
|
634
|
-
|
|
718
|
+
A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `toolCalls` (`{ id, name, arguments,
|
|
719
|
+
argumentChunks? }`), `usage`, `embedding`, a raw `body`, and `stream: { chunks, delayMs, failAfter, failWith: "network" |
|
|
720
|
+
"error-event", stallAfter, stallMs, includeUsage }`. A request to `/messages` (an `anthropic` provider) is answered in
|
|
721
|
+
the Anthropic shape, unary and SSE, and its key is read from `x-api-key`. `PROVIDER_ERRORS[kind][case]` holds each
|
|
722
|
+
provider's real error answers (`rate_limited`, `quota_exhausted`, `invalid_key`, `overloaded`, `bad_request`, and a
|
|
723
|
+
provider's second shapes such as `anthropic.spend_cap` or `openrouter.free_daily`), to script with
|
|
724
|
+
`fake.on({ key }, PROVIDER_ERRORS.anthropic.quota_exhausted)`. Everything is deterministic except the latencies you
|
|
725
|
+
script.
|
|
635
726
|
|
|
636
727
|
## Security notes
|
|
637
728
|
|
|
638
729
|
- A key is given as an environment variable name or a resolver; the package never needs a literal in configuration.
|
|
639
|
-
It is read at each attempt (so rotation needs no restart) and is sent only in
|
|
640
|
-
provider's
|
|
730
|
+
It is read at each attempt (so rotation needs no restart) and is sent only in its provider's auth header
|
|
731
|
+
(`authorization`, or `x-api-key` for Anthropic) to that provider's base URL.
|
|
732
|
+
- A blacklist record keeps a 64-bit fingerprint of the refused secret to notice a new one, never the secret.
|
|
641
733
|
- Keys are never logged, never put in events, errors, `health()` or `stats()`; every surface names them by alias. An
|
|
642
734
|
error body that echoes the key has it replaced by `[redacted]`.
|
|
643
735
|
- Prompts and responses are not in events unless you set `telemetry.includeContent`.
|
|
644
|
-
- `headers` in a provider or a call cannot override
|
|
736
|
+
- `headers` in a provider or a call cannot override the auth header.
|
|
645
737
|
- The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
|
|
646
738
|
- The gateway never forwards a client's `Authorization` upstream, never answers with `routing` (the key alias), and keeps
|
|
647
739
|
virtual keys as SHA-256 hashes; the admin token is compared in constant time.
|
|
@@ -654,14 +746,15 @@ app it is also a module you call directly, and `AiRouterGatewayModule` serves th
|
|
|
654
746
|
**Can I use it outside Nest?** Yes: through the gateway over HTTP from any language, or in Node with `createRouter` and
|
|
655
747
|
`createGateway` from `@starci/ai-router/gateway`, which loads no Nest code.
|
|
656
748
|
|
|
657
|
-
**Does it work with providers that are not OpenAI-compatible?**
|
|
658
|
-
|
|
749
|
+
**Does it work with providers that are not OpenAI-compatible?** Anthropic, through its own protocol adapter (`kind:
|
|
750
|
+
anthropic`: chat, streaming, tools, structured output). Callers always speak OpenAI chat-completions and embeddings.
|
|
659
751
|
|
|
660
752
|
**Why did my call wait instead of failing?** A lane was going to be usable before the deadline (a `retry-after`, a
|
|
661
753
|
breaker cooldown, a concurrency slot). Lower `deadlineMs` to fail sooner.
|
|
662
754
|
|
|
663
755
|
**Why is a key never used again?** It answered 401/403, or its resolver returned nothing, and `authDisableMs` is `null`
|
|
664
|
-
by default
|
|
756
|
+
by default; or its quota is spent until its window ends (`health()` shows `disabledReason` and `disabledUntil`). Give
|
|
757
|
+
it a new secret (it comes back at the next pick) or call `service.reset({ provider, key })`.
|
|
665
758
|
|
|
666
759
|
**How do I keep calls on one model?** `allowFallback: false` in the policy or a profile. `providers: [...]` keeps them
|
|
667
760
|
on chosen providers.
|
|
@@ -16,13 +16,13 @@ export declare class AiRouterService<Profiles extends string = string> implement
|
|
|
16
16
|
stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
|
|
17
17
|
/** Sends an embeddings request through the pool. */
|
|
18
18
|
embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
|
|
19
|
-
/** Reads every breaker,
|
|
19
|
+
/** Reads every breaker, blacklisted key, headroom and budget from the store. */
|
|
20
20
|
health(): Promise<HealthSnapshot>;
|
|
21
21
|
/** Returns this process's counters. */
|
|
22
22
|
stats(): StatsSnapshot;
|
|
23
23
|
/** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
|
|
24
24
|
on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
|
|
25
|
-
/** Closes breakers,
|
|
25
|
+
/** Closes breakers, lifts blacklists and clears headroom for the target (everything by default). */
|
|
26
26
|
reset(target?: ResetTarget): Promise<void>;
|
|
27
27
|
/** Lists the pool's profiles and models by name. */
|
|
28
28
|
catalog(): RouterCatalog;
|
|
@@ -15,6 +15,8 @@ export class AiRouterService {
|
|
|
15
15
|
this.on("breaker", handlers.breaker);
|
|
16
16
|
if (handlers["key-disabled"] !== undefined)
|
|
17
17
|
this.on("key-disabled", handlers["key-disabled"]);
|
|
18
|
+
if (handlers["key-restored"] !== undefined)
|
|
19
|
+
this.on("key-restored", handlers["key-restored"]);
|
|
18
20
|
if (handlers.request !== undefined)
|
|
19
21
|
this.on("request", handlers.request);
|
|
20
22
|
}
|
|
@@ -34,7 +36,7 @@ export class AiRouterService {
|
|
|
34
36
|
embeddings(request, options = {}) {
|
|
35
37
|
return this.router.embeddings(request, this.#options(options));
|
|
36
38
|
}
|
|
37
|
-
/** Reads every breaker,
|
|
39
|
+
/** Reads every breaker, blacklisted key, headroom and budget from the store. */
|
|
38
40
|
health() {
|
|
39
41
|
return this.router.health();
|
|
40
42
|
}
|
|
@@ -52,7 +54,7 @@ export class AiRouterService {
|
|
|
52
54
|
this.#releases.add(release);
|
|
53
55
|
return release;
|
|
54
56
|
}
|
|
55
|
-
/** Closes breakers,
|
|
57
|
+
/** Closes breakers, lifts blacklists and clears headroom for the target (everything by default). */
|
|
56
58
|
reset(target) {
|
|
57
59
|
return this.router.reset(target);
|
|
58
60
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { QuotaWindow } from "./providers.js";
|
|
2
|
+
import type { RateLimitInfo } from "./headers.js";
|
|
3
|
+
import type { ErrorClass } from "./types.js";
|
|
4
|
+
/** What an `AttemptError` may carry beyond its class and message. */
|
|
5
|
+
export interface AttemptErrorOptions {
|
|
6
|
+
readonly status?: number;
|
|
7
|
+
readonly info?: RateLimitInfo | null;
|
|
8
|
+
readonly missingKey?: boolean;
|
|
9
|
+
/** The provider's error code, for the blacklist event. */
|
|
10
|
+
readonly code?: string;
|
|
11
|
+
/** A quota window the provider's answer named. */
|
|
12
|
+
readonly window?: QuotaWindow;
|
|
13
|
+
}
|
|
14
|
+
/** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
|
|
15
|
+
export declare class AttemptError extends Error {
|
|
16
|
+
readonly errorClass: ErrorClass;
|
|
17
|
+
readonly status: number | undefined;
|
|
18
|
+
readonly info: RateLimitInfo | null;
|
|
19
|
+
readonly missingKey: boolean;
|
|
20
|
+
readonly code: string | undefined;
|
|
21
|
+
readonly window: QuotaWindow | undefined;
|
|
22
|
+
/** Builds the error. */
|
|
23
|
+
constructor(errorClass: ErrorClass, message: string, options?: AttemptErrorOptions);
|
|
24
|
+
/** The same failure with another message (a scrubbed one). */
|
|
25
|
+
withMessage(message: string): AttemptError;
|
|
26
|
+
}
|
package/dist/attempt.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
|
|
2
|
+
export class AttemptError extends Error {
|
|
3
|
+
errorClass;
|
|
4
|
+
status;
|
|
5
|
+
info;
|
|
6
|
+
missingKey;
|
|
7
|
+
code;
|
|
8
|
+
window;
|
|
9
|
+
/** Builds the error. */
|
|
10
|
+
constructor(errorClass, message, options = {}) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.errorClass = errorClass;
|
|
13
|
+
this.status = options.status;
|
|
14
|
+
this.info = options.info ?? null;
|
|
15
|
+
this.missingKey = options.missingKey ?? false;
|
|
16
|
+
this.code = options.code;
|
|
17
|
+
this.window = options.window;
|
|
18
|
+
}
|
|
19
|
+
/** The same failure with another message (a scrubbed one). */
|
|
20
|
+
withMessage(message) {
|
|
21
|
+
return new AttemptError(this.errorClass, message, {
|
|
22
|
+
...(this.status === undefined ? {} : { status: this.status }),
|
|
23
|
+
info: this.info,
|
|
24
|
+
missingKey: this.missingKey,
|
|
25
|
+
...(this.code === undefined ? {} : { code: this.code }),
|
|
26
|
+
...(this.window === undefined ? {} : { window: this.window }),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { QuotaWindow } from "./providers.js";
|
|
2
|
+
import type { ErrorClass, ProviderKind } from "./types.js";
|
|
3
|
+
/** The fields of a provider error, whatever shape it came in (docs/design.md, error shapes). */
|
|
4
|
+
export interface ProviderErrorInfo {
|
|
5
|
+
/** The HTTP status, or the numeric `code` OpenRouter puts inside a 200 body. */
|
|
6
|
+
readonly status: number | undefined;
|
|
7
|
+
/** `error.type` (Anthropic, OpenAI) or Gemini's gRPC `status` (`RESOURCE_EXHAUSTED`). */
|
|
8
|
+
readonly type: string;
|
|
9
|
+
/** `error.code` as text (`insufficient_quota`, `Arrearage`, `quota_exceeded`), or a top-level `code` (DashScope). */
|
|
10
|
+
readonly code: string;
|
|
11
|
+
/** A sub-code: Anthropic `error.details.error_code`, OpenRouter `error.metadata.limit_source` or `error_type`. */
|
|
12
|
+
readonly detail: string;
|
|
13
|
+
readonly message: string;
|
|
14
|
+
}
|
|
15
|
+
/** How one failed attempt is classified, and for a quota, until when its key stays out when the provider said so. */
|
|
16
|
+
export interface Classified {
|
|
17
|
+
readonly errorClass: ErrorClass;
|
|
18
|
+
/** The provider's most specific code (detail, code or type), for telemetry. */
|
|
19
|
+
readonly code: string | undefined;
|
|
20
|
+
/** A quota window the answer named; absent: the pool's `quotaBlacklistMs`. */
|
|
21
|
+
readonly window?: QuotaWindow;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Reads a provider error body into its fields. It knows the OpenAI `{ error: { message, type, code, param } }`, the
|
|
25
|
+
* Anthropic `{ type: "error", error: { type, message, details } }`, OpenRouter's `{ error: { code: <status>, message,
|
|
26
|
+
* metadata } }`, Gemini's object or one-element list `[{ error: { code, message, status } }]` and DashScope's top-level
|
|
27
|
+
* `{ code, message }`. Anything else leaves the fields empty and the raw text as the message.
|
|
28
|
+
*/
|
|
29
|
+
export declare function readProviderError(body: unknown, status: number | undefined): ProviderErrorInfo;
|
|
30
|
+
/**
|
|
31
|
+
* Classifies a failed attempt from its status and error body, per provider kind (docs/design.md, the classification
|
|
32
|
+
* table). Order: a refused secret, then a spent quota (which some providers send as 429 or 400), then a rate limit,
|
|
33
|
+
* then the status alone. Classes: `auth` and `quota` blacklist the key, `rate_limit` cools it down, `transient` feeds
|
|
34
|
+
* the breakers, `invalid_request` is the caller's and is not retried.
|
|
35
|
+
*/
|
|
36
|
+
export declare function classifyFailure(kind: ProviderKind, status: number | undefined, body: unknown): Classified;
|
|
37
|
+
/** The next midnight in a time zone after `now` (to the second; a DST day is corrected once). */
|
|
38
|
+
export declare function nextMidnight(now: number, timeZone: string): number;
|
|
39
|
+
/** When a quota window ends after `now`. */
|
|
40
|
+
export declare function windowEnd(window: QuotaWindow, now: number): number;
|