@starci/ai-router 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +133 -40
  3. package/dist/ai-router.service.d.ts +2 -2
  4. package/dist/ai-router.service.js +4 -2
  5. package/dist/attempt.d.ts +26 -0
  6. package/dist/attempt.js +29 -0
  7. package/dist/classify.d.ts +40 -0
  8. package/dist/classify.js +145 -0
  9. package/dist/config.d.ts +30 -5
  10. package/dist/config.js +184 -24
  11. package/dist/env.d.ts +70 -0
  12. package/dist/env.js +152 -0
  13. package/dist/errors.d.ts +1 -3
  14. package/dist/errors.js +4 -23
  15. package/dist/fake.d.ts +624 -5
  16. package/dist/fake.js +177 -21
  17. package/dist/gateway/config-file.d.ts +8 -3
  18. package/dist/gateway/config-file.js +53 -8
  19. package/dist/gateway/gateway.d.ts +2 -0
  20. package/dist/gateway/gateway.js +19 -3
  21. package/dist/gateway/index.d.ts +10 -1
  22. package/dist/gateway/index.js +5 -0
  23. package/dist/gateway/serve.js +7 -2
  24. package/dist/headers.d.ts +1 -1
  25. package/dist/headers.js +12 -6
  26. package/dist/pool.d.ts +37 -16
  27. package/dist/pool.js +173 -48
  28. package/dist/providers.d.ts +34 -0
  29. package/dist/providers.js +22 -0
  30. package/dist/router.js +108 -105
  31. package/dist/secrets.d.ts +8 -0
  32. package/dist/secrets.js +22 -0
  33. package/dist/shapes.d.ts +11 -0
  34. package/dist/shapes.js +124 -0
  35. package/dist/state-stores.d.ts +38 -0
  36. package/dist/state-stores.js +101 -0
  37. package/dist/store.d.ts +26 -7
  38. package/dist/store.js +64 -2
  39. package/dist/testing.d.ts +2 -2
  40. package/dist/testing.js +1 -1
  41. package/dist/types.d.ts +126 -38
  42. package/dist/wire/anthropic.d.ts +21 -0
  43. package/dist/wire/anthropic.js +322 -0
  44. package/dist/wire/index.d.ts +5 -0
  45. package/dist/wire/index.js +7 -0
  46. package/dist/wire/wire.d.ts +42 -0
  47. package/dist/wire/wire.js +35 -0
  48. package/docs/design.md +279 -0
  49. package/package.json +12 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,34 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.5.0 - 2026-10-31
4
+
5
+ Many providers below, each with an array of keys, behind one virtual key per app above; a key whose quota is spent or
6
+ whose secret is refused is blacklisted. [docs/design.md](docs/design.md) holds the provider matrix (read from each
7
+ provider's docs on 2026-10-31), the classification table, the blacklist lifecycle, profiles and key scopes, the
8
+ Anthropic adapter and the env naming.
9
+
10
+ - Added: provider kinds, `ProviderConfig.kind` (`anthropic`, `openrouter`, `qwen`, `deepseek`, `openai`, `gemini`, `groq`, `openai-compatible`), each with its default base URL, auth header, embeddings support and error classification (`PROVIDER_PRESETS`); `baseUrl` is optional for a kind that has one.
11
+ - Added: the Anthropic Messages adapter: OpenAI chat-completions in, `/v1/messages` with `x-api-key` and `anthropic-version` out, and back, for chat, SSE streaming (text and `tool_use` deltas, finish reason, usage, error events, a stream cut before `message_stop`), tools and `tool_choice`, tool results, images, `json_schema` as structured output, cache tokens in usage; `defaultMaxTokens` for the `max_tokens` Anthropic requires. Anthropic `anthropic-ratelimit-*` headers (RFC 3339 resets) and OpenRouter's epoch-millisecond `X-RateLimit-Reset` feed headroom.
12
+ - Added: the error class `quota` and per-provider classification (`classifyFailure`, `readProviderError`) of every error shape the providers document: OpenAI, Anthropic, OpenRouter (402 by `limit_source`, errors in a 200 body and mid-stream), DashScope codes, DeepSeek, Gemini (object or list, per-minute vs per-day), Groq (498, daily requests).
13
+ - Added: the key blacklist. A spent quota blacklists the key until the provider's window ends (next UTC month for Anthropic's spend cap and OpenAI's monthly limits, next Pacific midnight for Gemini's daily quota, next UTC midnight for OpenRouter's free daily cap, Groq's request reset), else `resilience.quotaBlacklistMs` (default 6 hours); a refused or missing secret until the secret changes. A 64-bit fingerprint of the refused secret is kept (never the secret), and a new secret lifts the blacklist at the next pick with the new `key-restored` event. `key-disabled` carries the provider's `code`; `KeyHealth.disabledUntil`.
14
+ - Added: when every key that could serve is blacklisted (or its provider past its budget), the call fails at once with class `quota` or `auth`, naming the keys, with `retryAt` at the earliest return; the gateway answers 503 `pool_quota_exhausted` or 503 `pool_auth_failed` with `Retry-After` (300 seconds when no return is known).
15
+ - Added: durable state stores, so a restart does not hammer dead keys: `openSqliteStateStore(path)` and `redisStateStore(client)` over a five-command `RedisLike`; `stateStore: { type: sqlite, path }` in the config file; `toHealthRecord` and `parseHealthRecord` for an adapter; `MemoryStateStore` and `EMPTY_RECORD` are exported again.
16
+ - Added: profile candidates, `ModelPolicy.candidates`: ordered tiers of `{ model, provider?, weight? }`, a tier that is a list sharing its load by weight; `pinned` profiles whose candidates must be one model `identity` and `dimensions` (`ModelConfig.identity`, `ModelConfig.dimensions`), which a call cannot override; an embeddings vector of another length fails its lane; every embeddings call stays on one model identity, and the gateway gives a direct embeddings model no fallback. `CatalogProfile.pinned` and `serves`; a profile on the wrong route is 400 `model_not_supported`.
17
+ - Added: provider budgets, `ProviderConfig.budget` (`costPerDay`, `costPerMonth`), kept in the store; `ProviderHealth.spend` and `kind`.
18
+ - Added: config from env: `poolFromEnv`, `providersFromEnv` and `keysFromEnv` read `<PREFIX>_API_KEY`, `_2` ... `_<n>`, their `_FILE` twins, a `<PREFIX>_API_KEYS` list and `<PREFIX>_BASE_URL` (prefixes ANTHROPIC, OPENROUTER, DASHSCOPE, DEEPSEEK, OPENAI, GEMINI, GROQ) as references read at each attempt, keep the providers that have keys and cut every profile to what they serve; `virtualKeyFromEnv` mounts the app's `STARCI_AI_KEY` (or `STARCI_AI_KEY_FILE`); `keysFromEnv: <PREFIX>` on a config-file provider.
19
+ - Added: the fake provider answers `/messages` in the Anthropic shape (unary and SSE), reads `x-api-key`, scripts `toolCalls`, and `PROVIDER_ERRORS` (from `@starci/ai-router/testing`) holds each provider's real error answers.
20
+ - Fixed: a provider or call header differing from the router's own only in case (`Anthropic-Version`) is one header, not two.
21
+
22
+ ### Migration from 0.4
23
+
24
+ - A 0.4 pool routes as before: `kind` defaults to `openai-compatible`, whose classification is 0.4's plus `quota` for 402 and the quota codes.
25
+ - `ErrorClass` has `quota`: a spent quota was a `rate_limit` with a 10-second cooldown and is now a blacklist; a `switch` over `errorClass` adds the case.
26
+ - When every key is blacklisted the router fails with `quota` or `auth`, no longer `no_candidate`, and the gateway answers 503 where `pool_auth_failed` was 502.
27
+ - A key refused for its secret comes back when the secret changes, without `reset()`; a missing secret when it is set.
28
+ - `KeyDisabledEvent.reason` adds `quota`, the event may carry `code`; `KeyHealth` adds `disabledUntil`, `ProviderHealth` `kind` (and `spend`), `CatalogProfile` `pinned` and `serves`, `RouterEvents` `key-restored`, `HealthRecord` its blacklist and budget fields (records 0.4 stored read with them empty).
29
+ - Embeddings never fall back to another model identity.
30
+ - Apps replace a hand-rolled `<PROVIDER>_API_KEY` plus `_2` ... `_9` list with `poolFromEnv`, keep `STARCI_AI_KEY` as their one virtual key (`virtualKeyFromEnv`), pass no admin token, and pass a durable `store`.
31
+
3
32
  ## 0.4.1 - 2026-10-30
4
33
 
5
34
  - Changed: the lint configuration is TypeScript (eslint.config.ts, loaded through jiti 2.7.0, an exact devDependency) and typechecked with the package; no runtime change.
package/README.md CHANGED
@@ -1,6 +1,9 @@
1
1
  # @starci/ai-router
2
2
 
3
- A router for OpenAI-compatible calls across a pool of keys, models and providers, usable three ways over one engine:
3
+ A router for OpenAI-compatible calls across many providers, each with an array of keys, behind one virtual key per app.
4
+ It load-balances below (Anthropic, OpenRouter, Qwen, DeepSeek, OpenAI, Gemini, Groq and any OpenAI-compatible endpoint),
5
+ blacklists a key whose quota is spent or whose secret is refused, and exposes one OpenAI-compatible API above. It is
6
+ usable three ways over one engine:
4
7
 
5
8
  - **as a gateway service** (`run.js serve`): an OpenAI-compatible HTTP endpoint (`/v1/chat/completions`, streaming
6
9
  included, `/v1/embeddings`, `/v1/models`) with its own virtual API keys, per-key quotas and usage accounting, so any
@@ -9,15 +12,19 @@ A router for OpenAI-compatible calls across a pool of keys, models and providers
9
12
  and `embeddings`; `AiRouterGatewayModule` hosts the gateway in the same process;
10
13
  - **as a plain Node library**: `createRouter(config)` from `@starci/ai-router/gateway`, with no Nest at all.
11
14
 
12
- The router picks a healthy key for each attempt, backs off what is failing, honours rate limits, retries on a different
13
- key or provider, falls back to another model when you allow it, and tells you what it did without ever exposing a key or
14
- a prompt.
15
+ The router picks a healthy key for each attempt, backs off what is failing, honours rate limits, blacklists a key whose
16
+ quota or credit is spent until its window ends (or its secret changes), retries on a different key or provider, falls
17
+ back across providers and models by profile, keeps embeddings on one model, and tells you what it did without ever
18
+ exposing a key or a prompt. [docs/design.md](docs/design.md) is the design: the provider matrix, the error
19
+ classification per provider, the blacklist lifecycle, profiles and key scopes, the Anthropic adapter and the env naming.
15
20
 
16
21
  - ESM, typed, Node 20 or newer (the durable key store needs Node 22.13 or newer for `node:sqlite`). One runtime
17
22
  dependency, `yaml`, for the config file. `@nestjs/common`, `@nestjs/core` (11 or 12) and `reflect-metadata` are
18
23
  optional peer dependencies: the root entry needs them, `@starci/ai-router/gateway`, `@starci/ai-router/cli` and the `run.js` process entry do not.
19
- - OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
20
- behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
24
+ - OpenAI chat-completions and embeddings shapes in and out. A provider's `kind` sets its protocol, auth header, default
25
+ base URL and error classification: `anthropic` (the Messages API, through a protocol adapter for chat, streaming,
26
+ tools and JSON), `openrouter`, `qwen` (DashScope compatible mode), `deepseek`, `openai`, `gemini` (its OpenAI
27
+ endpoint), `groq`, and `openai-compatible` (the default: Ollama, vLLM, Together, LiteLLM...).
21
28
  - Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
22
29
  everywhere else.
23
30
 
@@ -35,6 +42,55 @@ npm install @starci/ai-router # the gat
35
42
  npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata # in a Nest app
36
43
  ```
37
44
 
45
+ ## Many providers below, one virtual key above
46
+
47
+ An app declares every provider it may use, each with its models; `poolFromEnv` keeps the ones whose keys the
48
+ environment holds (`<PREFIX>_API_KEY`, `<PREFIX>_API_KEY_2`, ..., their `_FILE` twins, or a `<PREFIX>_API_KEYS` list)
49
+ and cuts every profile down to what they serve. Clients send `model: "work"` or `model: "embed"` with the app's one
50
+ virtual key, `STARCI_AI_KEY`.
51
+
52
+ ```ts
53
+ import { AiRouterGatewayModule, AiRouterModule, openSqliteKeyStore, openSqliteStateStore, poolFromEnv, virtualKeyFromEnv } from "@starci/ai-router"
54
+
55
+ AiRouterModule.registerAsync({
56
+ isGlobal: true,
57
+ useFactory: async () =>
58
+ poolFromEnv({
59
+ providers: [
60
+ { kind: "anthropic", models: [{ id: "claude-sonnet-5-5", cost: { input: 3, output: 15 } }] },
61
+ { kind: "openrouter", models: [{ id: "anthropic/claude-sonnet-5-5" }, { id: "openai/text-embedding-3-small", identity: "text-embedding-3-small", dimensions: 1536, capabilities: ["embeddings"] }] },
62
+ { kind: "deepseek", models: [{ id: "deepseek-v4-pro" }] },
63
+ { kind: "qwen", models: [{ id: "qwen3.8-max" }] },
64
+ { kind: "openai", models: [{ id: "text-embedding-3-small", dimensions: 1536, capabilities: ["embeddings"] }] },
65
+ ],
66
+ profiles: {
67
+ // Tiers in order; a list is one tier sharing its load by weight.
68
+ work: { candidates: [[{ provider: "anthropic", model: "claude-sonnet-5-5", weight: 3 }, { provider: "openrouter", model: "anthropic/claude-sonnet-5-5" }], { provider: "deepseek", model: "deepseek-v4-pro" }, { provider: "qwen", model: "qwen3.8-max" }] },
69
+ // Pinned: keys and providers rotate, the model (and its vector space) never changes.
70
+ embed: { pinned: true, candidates: [[{ provider: "openai", model: "text-embedding-3-small" }, { provider: "openrouter", model: "openai/text-embedding-3-small" }]] },
71
+ },
72
+ // Durable: a blacklisted key stays out across a restart.
73
+ store: await openSqliteStateStore("/data/ai-router-state.sqlite"),
74
+ }),
75
+ })
76
+ AiRouterGatewayModule.registerAsync({
77
+ useFactory: async () => ({ keys: await openSqliteKeyStore("/data/keys.sqlite"), mountedKeys: [await virtualKeyFromEnv({ label: "app", quota: { costPerMonth: 50 } })], listen: true }),
78
+ })
79
+ ```
80
+
81
+ | env variable | meaning |
82
+ |---|---|
83
+ | `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `DASHSCOPE_API_KEY` (Qwen), `DEEPSEEK_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY` | the first key of each provider, alias `<kind>-1` |
84
+ | `<PREFIX>_API_KEY_2` ... `<PREFIX>_API_KEY_<n>` | more keys, aliases `<kind>-<n>`; gaps allowed, no upper bound |
85
+ | `<PREFIX>_API_KEY_FILE`, `<PREFIX>_API_KEY_<n>_FILE` | a key mounted as a file (its first line) |
86
+ | `<PREFIX>_API_KEYS` | several keys in one variable, split at commas and whitespace |
87
+ | `<PREFIX>_BASE_URL` | another endpoint: a Qwen region or workspace host, an Ollama host |
88
+ | `STARCI_AI_KEY` (or `STARCI_AI_KEY_FILE`) | the app's virtual key, `sk-starci-<label>-...`, bound to `work` and `embed` |
89
+
90
+ A virtual key carries the profiles it may call and, optionally, an allowlist of exact model ids (`models`, empty by
91
+ default). A model it names that is neither is refused 403 `model_not_allowed`, never remapped; `/v1/models` lists only
92
+ what it may call; its quotas count every call whatever the model. See [key scopes](docs/design.md#key-scopes).
93
+
38
94
  ## Run it as a gateway
39
95
 
40
96
  One process (one container) holds the pool, the virtual keys and the admin API. The package declares no `bin` (the
@@ -61,9 +117,14 @@ gateway:
61
117
  adminToken: { env: AI_ROUTER_ADMIN_TOKEN } # absent: the admin API is off
62
118
  keyStore: { type: sqlite, path: /data/keys.sqlite } # or { type: memory }; a relative path is from this file
63
119
  maxBodyBytes: 4194304 # default 4 MiB
120
+ stateStore: { type: sqlite, path: /data/state.sqlite } # breakers and blacklists survive a restart; default memory
64
121
  providers:
122
+ - id: anthropic
123
+ kind: anthropic # base URL, x-api-key auth and the Messages adapter come with the kind
124
+ keysFromEnv: ANTHROPIC # ANTHROPIC_API_KEY, ANTHROPIC_API_KEY_2, ... and their _FILE twins
125
+ models: [{ id: claude-sonnet-5-5, aliases: [smart] }]
65
126
  - id: openai
66
- baseUrl: https://api.openai.com/v1
127
+ kind: openai
67
128
  keys:
68
129
  - { alias: team-a, env: OPENAI_KEY_A }
69
130
  - { alias: team-b, file: /run/secrets/openai_key_b, maxConcurrency: 8 }
@@ -110,8 +171,9 @@ Errors are OpenAI error objects, `{ "error": { "message", "type", "param", "code
110
171
  | 413 | `request_too_large` | a body past `maxBodyBytes` |
111
172
  | 429 | `requests_per_minute_exceeded`, `tokens_per_day_exceeded`, `cost_per_day_exceeded`, `cost_per_month_exceeded` | the key's quota, with `Retry-After` |
112
173
  | 4xx | `upstream_invalid_request` | the provider refused the request (its status and message, e.g. a context-length error) |
113
- | 502 | `upstream_error`, `pool_auth_failed` | every attempt failed upstream, or no upstream key was accepted |
174
+ | 502 | `upstream_error` | every attempt failed upstream |
114
175
  | 503 | `pool_unavailable`, `pool_rate_limited` | no lane can serve it, or every key is rate limited (`Retry-After` when known) |
176
+ | 503 | `pool_quota_exhausted`, `pool_auth_failed` | every key that could serve is blacklisted: its quota or credit spent, or its secret refused (`Retry-After` until the first returns, else 300) |
115
177
  | 504 | `pool_timeout` | the policy's deadline passed |
116
178
 
117
179
  ### Virtual keys
@@ -405,10 +467,10 @@ the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in
405
467
  | `chat(request, options?)` | sends a chat-completions request; resolves to the OpenAI completion plus `routing` |
406
468
  | `stream(request, options?)` | returns an async iterable of OpenAI chunks, with `requestId`; see [Streaming](#streaming) |
407
469
  | `embeddings(request, options?)` | sends an embeddings request; resolves to the OpenAI response plus `routing` |
408
- | `health()` | reads every breaker, disabled key, headroom and latency EWMA |
470
+ | `health()` | reads every breaker, blacklisted key (reason and until), headroom, budget spend and latency EWMA |
409
471
  | `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
410
- | `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled` until shutdown; returns the unsubscribe function |
411
- | `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
472
+ | `on(event, handler)` | subscribes to `attempt`, `request`, `breaker`, `key-disabled` or `key-restored` until shutdown; returns the unsubscribe function |
473
+ | `reset(target?)` | closes breakers, lifts blacklists and clears headroom for everything, one provider or one key |
412
474
  | `catalog()` | the pool's profiles (resolved model and fallbacks) and models (ids, aliases, providers), with no key |
413
475
 
414
476
  `options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
@@ -447,19 +509,23 @@ A named registration provides no default instance; `@InjectAiRouterService()` wi
447
509
  | field | meaning | default |
448
510
  |---|---|---|
449
511
  | `providers[].id` | a name, unique, without `/` | required |
450
- | `providers[].baseUrl` | the URL the OpenAI paths hang from (`.../v1`) | required |
451
- | `providers[].headers` | extra headers for every request (an `authorization` here is ignored) | none |
512
+ | `providers[].kind` | `anthropic`, `openrouter`, `qwen`, `deepseek`, `openai`, `gemini`, `groq` or `openai-compatible`: protocol, auth header, default base URL, embeddings support and error classification ([provider matrix](docs/design.md#provider-matrix)) | `openai-compatible` |
513
+ | `providers[].baseUrl` | the URL the API paths hang from (`.../v1`) | the kind's; required for `openai-compatible` |
514
+ | `providers[].headers` | extra headers for every request (the auth header, `authorization` or `x-api-key`, is ignored here) | none |
452
515
  | `providers[].weight` | multiplies the round-robin share of every lane of the provider | 1 |
453
516
  | `providers[].maxConcurrency` | in-flight requests across the provider's keys, in this process | unbounded |
454
- | `providers[].keys[]` | `{ alias, env }` or `{ alias, resolve }`, plus `weight`, `maxConcurrency` | at least one |
455
- | `providers[].models[]` | `{ id, aliases?, contextWindow?, cost?, capabilities?, weight? }` | at least one |
517
+ | `providers[].defaultMaxTokens` | the `max_tokens` sent to Anthropic, which requires one, when a request sets none | 4096 |
518
+ | `providers[].budget` | `{ costPerDay?, costPerMonth? }` in the models' cost unit; past it the provider is out until the next UTC day or month | none |
519
+ | `providers[].keys[]` | `{ alias, env }` or `{ alias, resolve }`, plus `weight`, `maxConcurrency` (`keysFromEnv` builds them) | at least one |
520
+ | `providers[].models[]` | `{ id, aliases?, contextWindow?, cost?, capabilities?, weight?, identity?, dimensions? }` | at least one |
456
521
  | `defaults` | the policy every call starts from | none |
457
522
  | `profiles` | named policies a call selects with `{ profile }` | none |
458
523
  | `resilience.keyBreaker` | `{ failureThreshold, cooldownMs, maxCooldownMs }` per key | 3, 15000, 300000 |
459
524
  | `resilience.providerBreaker` | the same per provider, fed by all its keys | 8, 30000, 300000 |
460
525
  | `resilience.backoff` | `{ baseMs, factor, maxMs }` between attempts after a transient failure | 100, 2, 5000 |
461
- | `resilience.authDisableMs` | how long a 401/403 key stays out; `null` until `reset()` | `null` |
526
+ | `resilience.authDisableMs` | how long a key whose secret was refused stays out; `null` until its secret changes or `reset()` | `null` |
462
527
  | `resilience.rateLimitCooldownMs` | a 429's cooldown when it has no `retry-after` | 10000 |
528
+ | `resilience.quotaBlacklistMs` | how long a key whose quota is spent stays out when the provider names no window | 21600000 (6 h) |
463
529
  | `selection.latencyTolerance`, `latencySlackMs` | how far above the best latency EWMA a lane still ties | 1.5, 50 |
464
530
  | `selection.ewmaAlpha` | the weight of the newest latency sample | 0.3 |
465
531
  | `selection.lowHeadroom` | remaining requests at or below which a lane is used only when nothing else is | 1 |
@@ -471,7 +537,9 @@ A named registration provides no default instance; `@InjectAiRouterService()` wi
471
537
  A model's `id` is the name sent to the provider; a caller may name it by `id` or any alias, and an alias shared by
472
538
  several providers makes them interchangeable for that call. `cost` is per one million tokens, `input` and `output`, in
473
539
  whatever unit you budget in. `capabilities` defaults to `chat`, `streaming`, `tools` and `json`; an embeddings model
474
- declares `["embeddings"]`.
540
+ declares `["embeddings"]` (refused on a kind with no embeddings API: `anthropic`, `deepseek`, `groq`). `identity` names
541
+ one model served under different ids (`openai/text-embedding-3-small` at OpenRouter is `text-embedding-3-small`), and
542
+ `dimensions` the vector length an embeddings model answers: a response of another length fails the attempt.
475
543
 
476
544
  ## Policies
477
545
 
@@ -490,6 +558,11 @@ defaults'.
490
558
  | `budget.maxCost` | a model whose worst-case cost for this request is higher is no candidate | none |
491
559
  | `providers` | restrict the call to these provider ids | all |
492
560
  | `require` | capabilities needed beyond those the request implies | none |
561
+ | `candidates` | in place of `model` and `fallbacks`: ordered tiers of `{ model, provider?, weight? }`; a tier that is a list shares its load by weight; `allowFallback: false` keeps the first tier | none |
562
+ | `pinned` | a profile whose candidates are all one model `identity` and `dimensions` (refused at startup otherwise); a call cannot name another model | `false` |
563
+
564
+ A call's own `model` wins over a profile's candidates, except on a pinned profile. An embeddings call always stays on
565
+ the model identity of its first lane: another model is another vector space.
493
566
 
494
567
  The request implies its own needs: `tools` needs `tools`, a non-text `response_format` needs `json`, `stream()` needs
495
568
  `streaming`, `embeddings()` needs `embeddings`. A model is also no candidate when its `contextWindow` is smaller than
@@ -520,10 +593,11 @@ waits for it rather than failing. When none will be, it fails at once with the c
520
593
 
521
594
  | what happened | class | effect | retried |
522
595
  |---|---|---|---|
523
- | 429, or a rate/quota error object | `rate_limit` | the lane (key and model) cools down until `retry-after` / `retry-after-ms`, else `rateLimitCooldownMs` | yes, at once, on another lane |
524
- | 401, 403, an auth error object | `auth` | the key is disabled (`authDisableMs`, default until `reset()`); `key-disabled` event | yes, at once, on another key |
525
- | a resolver that returns nothing or throws, an unset env var | `auth` | the key is disabled with reason `missing`; nothing is sent | yes |
526
- | 408, 409, 425, 5xx | `transient` | counts toward the key's and the provider's breakers; a `retry-after` also cools the lane | yes, after backoff |
596
+ | 429, or a rate error object | `rate_limit` | the lane (key and model) cools down until `retry-after` / `retry-after-ms`, else `rateLimitCooldownMs` | yes, at once, on another lane |
597
+ | 402, or a quota, credit or billing code or message (`insufficient_quota`, `billing_error`, `Arrearage`, ...; per provider in [the classification table](docs/design.md#error-classification)) | `quota` | the key is blacklisted until the provider's window ends (a monthly cap, a daily free tier), else `quotaBlacklistMs`, or until its secret changes; `key-disabled` event with the provider's `code` | yes, at once, on another key |
598
+ | 401, 403, an auth error object | `auth` | the key is blacklisted until its secret changes (or `authDisableMs`); `key-disabled` event | yes, at once, on another key |
599
+ | a resolver that returns nothing or throws, an unset env var | `auth` | the key is blacklisted with reason `missing` until a secret is set; nothing is sent | yes |
600
+ | 408, 409, 425, 498, 5xx (Anthropic's 529) | `transient` | counts toward the key's and the provider's breakers; a `retry-after` also cools the lane | yes, after backoff |
527
601
  | connection failure | `network` | as transient | yes, after backoff |
528
602
  | attempt timeout | `timeout` | as transient | yes, after backoff |
529
603
  | broken or malformed stream before the first token | `stream` (or the streamed error's class) | as transient | yes, after backoff |
@@ -538,8 +612,13 @@ A successful probe closes it; a failed one reopens it with the cooldown doubled,
538
612
  invalid requests and cancellations never count against a breaker.
539
613
 
540
614
  Backoff after a transient failure is `min(maxMs, baseMs * factor^(n-1))` scaled by a random factor in `[0.5, 1)`, and
541
- the next attempt goes to a different lane by the tiers above. Rate and auth failures belong to one key, so the next key
542
- is tried without waiting.
615
+ the next attempt goes to a different lane by the tiers above. Rate, quota and auth failures belong to one key, so the
616
+ next key is tried without waiting.
617
+
618
+ A blacklisted key keeps a 64-bit fingerprint of the secret it was refused with (never the secret): each pick resolves
619
+ its secret again, and a new one (a rotated variable, a re-sealed file) lifts the blacklist at once with a
620
+ `key-restored` event. When every key that could serve is blacklisted and none returns before the deadline, the call
621
+ fails at once with class `quota` or `auth`, naming the keys, with `retryAt` at the earliest return.
543
622
 
544
623
  ## Streaming
545
624
 
@@ -581,7 +660,8 @@ AiRouterModule.register({
581
660
  | `attempt` | every attempt | `requestId`, `operation`, `attempt`, `provider`, `key` alias, `model`, `latencyMs`, `outcome`, `errorClass`, `status`, `usage`, `cost` |
582
661
  | `request` | every call, once | `requestId`, `operation`, `outcome`, `attempts`, `latencyMs`, the answering lane or the `errorClass` |
583
662
  | `breaker` | a breaker changes state | `scope` (`key` or `provider`), `provider`, `key`, `from`, `to` |
584
- | `key-disabled` | a key is disabled | `provider`, `key`, `reason` (`auth` or `missing`), `until` (null: until `reset()`) |
663
+ | `key-disabled` | a key is blacklisted | `provider`, `key`, `reason` (`auth`, `missing` or `quota`), `until` (null: until its secret changes or `reset()`), `code` (the provider's error code, when it sent one) |
664
+ | `key-restored` | a blacklisted key's secret changed | `provider`, `key`, `was` (the reason it was out), `reason` (`secret_changed`) |
585
665
 
586
666
  Events never carry message content unless `telemetry.includeContent` is `true`; then `attempt` events add `content`
587
667
  with the request body and, for `chat` and `embeddings`, the response. A handler that throws is isolated from the call.
@@ -589,13 +669,19 @@ The module itself logs nothing.
589
669
 
590
670
  ## Sharing state between processes
591
671
 
592
- Breakers, disabled keys and headroom live in the `store` option; the default keeps them in memory. Several processes
593
- (or several registrations) share them through one store: an adapter implements the exported `StateStore` type (`get`,
594
- `set`, `delete`, `keys(prefix)`, and an atomic `lease(key, ttlMs)` / `release(key)` pair for the half-open probe).
595
- Methods may return values or promises. Records are plain JSON (`HealthRecord`); a Redis adapter is a `GET`/`SET`, a
596
- `SCAN` and a `SET NX PX` lease. In-flight counts, latency EWMAs and round-robin positions stay per process. Updates
597
- are read-modify-write, so two processes failing at the same instant may count one failure between them: breakers are a
598
- health signal, not a ledger.
672
+ Breakers, blacklisted keys, headroom and budgets live in the `store` option; the default keeps them in memory, so a
673
+ restart forgets which keys are dead and tries them again. Two durable stores ship:
674
+
675
+ - `openSqliteStateStore(path)` (`node:sqlite`, Node 22.13 or newer): one file, which processes on one host may share
676
+ (it may be the key store's file); `stateStore: { type: sqlite, path }` in the config file.
677
+ - `redisStateStore(client)`: every gateway process of an app sees the same blacklist; `client` is five commands
678
+ (`get`, `set`, `setIfAbsent` as `SET NX PX`, `del`, `keys`), a few lines over node-redis or ioredis.
679
+
680
+ Another adapter implements the exported `StateStore` type (`get`, `set`, `delete`, `keys(prefix)`, and an atomic
681
+ `lease(key, ttlMs)` / `release(key)` pair for the half-open probe), parsing what it stored with `toHealthRecord` or
682
+ `parseHealthRecord`. Methods may return values or promises. In-flight counts, latency EWMAs and round-robin positions
683
+ stay per process. Updates are read-modify-write, so two processes failing at the same instant may count one failure
684
+ between them: breakers are a health signal, not a ledger.
599
685
 
600
686
  ## Testing
601
687
 
@@ -629,19 +715,25 @@ const reply = await app.get(AiRouterService).chat({ model: "m", messages: [{ rol
629
715
  await app.close()
630
716
  ```
631
717
 
632
- A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `usage`, `embedding`, a raw `body`, and
633
- `stream: { chunks, delayMs, failAfter, failWith: "network" | "error-event", stallAfter, stallMs, includeUsage }`.
634
- Everything is deterministic except the latencies you script.
718
+ A reply may set `status`, `headers`, `latencyMs`, `error: "network"`, `text`, `toolCalls` (`{ id, name, arguments,
719
+ argumentChunks? }`), `usage`, `embedding`, a raw `body`, and `stream: { chunks, delayMs, failAfter, failWith: "network" |
720
+ "error-event", stallAfter, stallMs, includeUsage }`. A request to `/messages` (an `anthropic` provider) is answered in
721
+ the Anthropic shape, unary and SSE, and its key is read from `x-api-key`. `PROVIDER_ERRORS[kind][case]` holds each
722
+ provider's real error answers (`rate_limited`, `quota_exhausted`, `invalid_key`, `overloaded`, `bad_request`, and a
723
+ provider's second shapes such as `anthropic.spend_cap` or `openrouter.free_daily`), to script with
724
+ `fake.on({ key }, PROVIDER_ERRORS.anthropic.quota_exhausted)`. Everything is deterministic except the latencies you
725
+ script.
635
726
 
636
727
  ## Security notes
637
728
 
638
729
  - A key is given as an environment variable name or a resolver; the package never needs a literal in configuration.
639
- It is read at each attempt (so rotation needs no restart) and is sent only in the `authorization` header to its own
640
- provider's `baseUrl`.
730
+ It is read at each attempt (so rotation needs no restart) and is sent only in its provider's auth header
731
+ (`authorization`, or `x-api-key` for Anthropic) to that provider's base URL.
732
+ - A blacklist record keeps a 64-bit fingerprint of the refused secret to notice a new one, never the secret.
641
733
  - Keys are never logged, never put in events, errors, `health()` or `stats()`; every surface names them by alias. An
642
734
  error body that echoes the key has it replaced by `[redacted]`.
643
735
  - Prompts and responses are not in events unless you set `telemetry.includeContent`.
644
- - `headers` in a provider or a call cannot override `authorization`.
736
+ - `headers` in a provider or a call cannot override the auth header.
645
737
  - The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
646
738
  - The gateway never forwards a client's `Authorization` upstream, never answers with `routing` (the key alias), and keeps
647
739
  virtual keys as SHA-256 hashes; the admin token is compared in constant time.
@@ -654,14 +746,15 @@ app it is also a module you call directly, and `AiRouterGatewayModule` serves th
654
746
  **Can I use it outside Nest?** Yes: through the gateway over HTTP from any language, or in Node with `createRouter` and
655
747
  `createGateway` from `@starci/ai-router/gateway`, which loads no Nest code.
656
748
 
657
- **Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
658
- the OpenAI chat-completions and embeddings API.
749
+ **Does it work with providers that are not OpenAI-compatible?** Anthropic, through its own protocol adapter (`kind:
750
+ anthropic`: chat, streaming, tools, structured output). Callers always speak OpenAI chat-completions and embeddings.
659
751
 
660
752
  **Why did my call wait instead of failing?** A lane was going to be usable before the deadline (a `retry-after`, a
661
753
  breaker cooldown, a concurrency slot). Lower `deadlineMs` to fail sooner.
662
754
 
663
755
  **Why is a key never used again?** It answered 401/403, or its resolver returned nothing, and `authDisableMs` is `null`
664
- by default. Fix the key and call `service.reset({ provider, key })`, or set `authDisableMs`.
756
+ by default; or its quota is spent until its window ends (`health()` shows `disabledReason` and `disabledUntil`). Give
757
+ it a new secret (it comes back at the next pick) or call `service.reset({ provider, key })`.
665
758
 
666
759
  **How do I keep calls on one model?** `allowFallback: false` in the policy or a profile. `providers: [...]` keeps them
667
760
  on chosen providers.
@@ -16,13 +16,13 @@ export declare class AiRouterService<Profiles extends string = string> implement
16
16
  stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
17
17
  /** Sends an embeddings request through the pool. */
18
18
  embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
19
- /** Reads every breaker, disabled key and headroom from the store. */
19
+ /** Reads every breaker, blacklisted key, headroom and budget from the store. */
20
20
  health(): Promise<HealthSnapshot>;
21
21
  /** Returns this process's counters. */
22
22
  stats(): StatsSnapshot;
23
23
  /** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
24
24
  on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
25
- /** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
25
+ /** Closes breakers, lifts blacklists and clears headroom for the target (everything by default). */
26
26
  reset(target?: ResetTarget): Promise<void>;
27
27
  /** Lists the pool's profiles and models by name. */
28
28
  catalog(): RouterCatalog;
@@ -15,6 +15,8 @@ export class AiRouterService {
15
15
  this.on("breaker", handlers.breaker);
16
16
  if (handlers["key-disabled"] !== undefined)
17
17
  this.on("key-disabled", handlers["key-disabled"]);
18
+ if (handlers["key-restored"] !== undefined)
19
+ this.on("key-restored", handlers["key-restored"]);
18
20
  if (handlers.request !== undefined)
19
21
  this.on("request", handlers.request);
20
22
  }
@@ -34,7 +36,7 @@ export class AiRouterService {
34
36
  embeddings(request, options = {}) {
35
37
  return this.router.embeddings(request, this.#options(options));
36
38
  }
37
- /** Reads every breaker, disabled key and headroom from the store. */
39
+ /** Reads every breaker, blacklisted key, headroom and budget from the store. */
38
40
  health() {
39
41
  return this.router.health();
40
42
  }
@@ -52,7 +54,7 @@ export class AiRouterService {
52
54
  this.#releases.add(release);
53
55
  return release;
54
56
  }
55
- /** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
57
+ /** Closes breakers, lifts blacklists and clears headroom for the target (everything by default). */
56
58
  reset(target) {
57
59
  return this.router.reset(target);
58
60
  }
@@ -0,0 +1,26 @@
1
+ import type { QuotaWindow } from "./providers.js";
2
+ import type { RateLimitInfo } from "./headers.js";
3
+ import type { ErrorClass } from "./types.js";
4
+ /** What an `AttemptError` may carry beyond its class and message. */
5
+ export interface AttemptErrorOptions {
6
+ readonly status?: number;
7
+ readonly info?: RateLimitInfo | null;
8
+ readonly missingKey?: boolean;
9
+ /** The provider's error code, for the blacklist event. */
10
+ readonly code?: string;
11
+ /** A quota window the provider's answer named. */
12
+ readonly window?: QuotaWindow;
13
+ }
14
+ /** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
15
+ export declare class AttemptError extends Error {
16
+ readonly errorClass: ErrorClass;
17
+ readonly status: number | undefined;
18
+ readonly info: RateLimitInfo | null;
19
+ readonly missingKey: boolean;
20
+ readonly code: string | undefined;
21
+ readonly window: QuotaWindow | undefined;
22
+ /** Builds the error. */
23
+ constructor(errorClass: ErrorClass, message: string, options?: AttemptErrorOptions);
24
+ /** The same failure with another message (a scrubbed one). */
25
+ withMessage(message: string): AttemptError;
26
+ }
@@ -0,0 +1,29 @@
1
+ /** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
2
+ export class AttemptError extends Error {
3
+ errorClass;
4
+ status;
5
+ info;
6
+ missingKey;
7
+ code;
8
+ window;
9
+ /** Builds the error. */
10
+ constructor(errorClass, message, options = {}) {
11
+ super(message);
12
+ this.errorClass = errorClass;
13
+ this.status = options.status;
14
+ this.info = options.info ?? null;
15
+ this.missingKey = options.missingKey ?? false;
16
+ this.code = options.code;
17
+ this.window = options.window;
18
+ }
19
+ /** The same failure with another message (a scrubbed one). */
20
+ withMessage(message) {
21
+ return new AttemptError(this.errorClass, message, {
22
+ ...(this.status === undefined ? {} : { status: this.status }),
23
+ info: this.info,
24
+ missingKey: this.missingKey,
25
+ ...(this.code === undefined ? {} : { code: this.code }),
26
+ ...(this.window === undefined ? {} : { window: this.window }),
27
+ });
28
+ }
29
+ }
@@ -0,0 +1,40 @@
1
+ import type { QuotaWindow } from "./providers.js";
2
+ import type { ErrorClass, ProviderKind } from "./types.js";
3
+ /** The fields of a provider error, whatever shape it came in (docs/design.md, error shapes). */
4
+ export interface ProviderErrorInfo {
5
+ /** The HTTP status, or the numeric `code` OpenRouter puts inside a 200 body. */
6
+ readonly status: number | undefined;
7
+ /** `error.type` (Anthropic, OpenAI) or Gemini's gRPC `status` (`RESOURCE_EXHAUSTED`). */
8
+ readonly type: string;
9
+ /** `error.code` as text (`insufficient_quota`, `Arrearage`, `quota_exceeded`), or a top-level `code` (DashScope). */
10
+ readonly code: string;
11
+ /** A sub-code: Anthropic `error.details.error_code`, OpenRouter `error.metadata.limit_source` or `error_type`. */
12
+ readonly detail: string;
13
+ readonly message: string;
14
+ }
15
+ /** How one failed attempt is classified, and for a quota, until when its key stays out when the provider said so. */
16
+ export interface Classified {
17
+ readonly errorClass: ErrorClass;
18
+ /** The provider's most specific code (detail, code or type), for telemetry. */
19
+ readonly code: string | undefined;
20
+ /** A quota window the answer named; absent: the pool's `quotaBlacklistMs`. */
21
+ readonly window?: QuotaWindow;
22
+ }
23
+ /**
24
+ * Reads a provider error body into its fields. It knows the OpenAI `{ error: { message, type, code, param } }`, the
25
+ * Anthropic `{ type: "error", error: { type, message, details } }`, OpenRouter's `{ error: { code: <status>, message,
26
+ * metadata } }`, Gemini's object or one-element list `[{ error: { code, message, status } }]` and DashScope's top-level
27
+ * `{ code, message }`. Anything else leaves the fields empty and the raw text as the message.
28
+ */
29
+ export declare function readProviderError(body: unknown, status: number | undefined): ProviderErrorInfo;
30
+ /**
31
+ * Classifies a failed attempt from its status and error body, per provider kind (docs/design.md, the classification
32
+ * table). Order: a refused secret, then a spent quota (which some providers send as 429 or 400), then a rate limit,
33
+ * then the status alone. Classes: `auth` and `quota` blacklist the key, `rate_limit` cools it down, `transient` feeds
34
+ * the breakers, `invalid_request` is the caller's and is not retried.
35
+ */
36
+ export declare function classifyFailure(kind: ProviderKind, status: number | undefined, body: unknown): Classified;
37
+ /** The next midnight in a time zone after `now` (to the second; a DST day is corrected once). */
38
+ export declare function nextMidnight(now: number, timeZone: string): number;
39
+ /** When a quota window ends after `now`. */
40
+ export declare function windowEnd(window: QuotaWindow, now: number): number;