@starci/ai-router 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +256 -14
  3. package/dist/ai-router-gateway.listen.d.ts +30 -0
  4. package/dist/ai-router-gateway.listen.js +67 -0
  5. package/dist/ai-router-gateway.module-definition.d.ts +21 -0
  6. package/dist/ai-router-gateway.module-definition.js +6 -0
  7. package/dist/ai-router-gateway.module.d.ts +19 -0
  8. package/dist/ai-router-gateway.module.js +75 -0
  9. package/dist/ai-router-testing.module.d.ts +2 -2
  10. package/dist/ai-router.module-definition.d.ts +3 -3
  11. package/dist/ai-router.module.d.ts +2 -2
  12. package/dist/ai-router.options.d.ts +1 -1
  13. package/dist/ai-router.service.d.ts +4 -2
  14. package/dist/ai-router.service.js +4 -0
  15. package/dist/ai-router.tokens.d.ts +1 -1
  16. package/dist/config.d.ts +3 -1
  17. package/dist/config.js +41 -0
  18. package/dist/errors.d.ts +1 -1
  19. package/dist/events.d.ts +1 -1
  20. package/dist/fake.d.ts +1 -1
  21. package/dist/gateway/cli.d.ts +13 -0
  22. package/dist/gateway/cli.js +241 -0
  23. package/dist/gateway/config-file.d.ts +37 -0
  24. package/dist/gateway/config-file.js +238 -0
  25. package/dist/gateway/gateway.d.ts +34 -0
  26. package/dist/gateway/gateway.js +527 -0
  27. package/dist/gateway/index.d.ts +14 -0
  28. package/dist/gateway/index.js +7 -0
  29. package/dist/gateway/keys.d.ts +166 -0
  30. package/dist/gateway/keys.js +268 -0
  31. package/dist/gateway/run.d.ts +1 -0
  32. package/dist/gateway/run.js +14 -0
  33. package/dist/gateway/serve.d.ts +25 -0
  34. package/dist/gateway/serve.js +38 -0
  35. package/dist/gateway/sqlite-key-store.d.ts +8 -0
  36. package/dist/gateway/sqlite-key-store.js +167 -0
  37. package/dist/index.d.ts +9 -9
  38. package/dist/index.js +2 -1
  39. package/dist/pool.d.ts +5 -5
  40. package/dist/router.d.ts +1 -1
  41. package/dist/router.js +3 -1
  42. package/dist/store.d.ts +1 -1
  43. package/dist/testing.d.ts +4 -4
  44. package/dist/types.d.ts +29 -1
  45. package/package.json +30 -4
package/CHANGELOG.md CHANGED
@@ -1,5 +1,20 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0 - 2026-10-28
4
+
5
+ - Added: an OpenAI-compatible HTTP gateway over the same router (`createGateway(router, options)`): `POST /v1/chat/completions` (SSE streaming in OpenAI's chunk format, ending with `[DONE]`), `POST /v1/embeddings`, `GET /v1/models` (the profiles and pool models the calling key may use), `GET /health`, and OpenAI error objects with 400, 401, 403, 404, 413, 429 (with `Retry-After`), 502, 503 and 504. A request's `model` names a profile or a pool model id or alias.
6
+ - Added: virtual API keys (`sk-starci-<label>-<random>`), stored as SHA-256 only and shown once at create and rotate, each bound to profiles, models, fallback, quotas (requests per minute, tokens per day, cost per day and month), enabled and revoked state, label and owner; usage per key and UTC day (requests, outcomes, tokens, estimated cost, latency), with no prompt or end-user identifier. `KeyAdmin` (create, list, get, update, revoke, rotate, usage), the `KeyStore` interface, `MemoryKeyStore`, and the durable `openSqliteKeyStore(path)` over `node:sqlite` with its `KEY_STORE_SCHEMA`.
7
+ - Added: the admin API under `/admin/keys`, guarded by an admin token, and the commands `serve --config <pool.yaml|json>` and `keys create|list|get|update|revoke|rotate|usage`, on the durable store (`--config`) or through a running gateway (`--url`), run by the process entry `dist/gateway/run.js` or in process with `main` from `@starci/ai-router/cli`; the package declares no `bin`, since the harness has one CLI. The config file names every secret by reference (`{ env }` or `{ file }`) and refuses one written inline. `loadGatewayConfig`, `parseGatewayConfig`, `serveGateway`, `openKeyStore`.
8
+ - Added: `AiRouterGatewayModule.register` / `registerAsync` hosting the gateway in a Nest app over a registered router, `AI_ROUTER_GATEWAY`, `getAiRouterGatewayToken` and `@InjectAiRouterGateway()`.
9
+ - Added: the gateway's port follows the StarCi port convention (`basePort + portOffset`: FE base+0, API base+1): `AiRouterGatewayModule`'s `listen` takes `{ port }` (explicit), or `true` / `{ offset?, host? }` for the app's own bound port plus `offset` (default 1, so an API on 3001 serves its gateway on 3002), on `127.0.0.1` unless `host` is set. The app's port is read from its HTTP server once it listens (the module waits for `listening`, since Nest bootstraps before `app.listen`), from `PORT` when the app has no HTTP server, and bootstrap refuses with an `AiRouterError` when neither is known; a bind failure goes to `onError`. `serve` takes `gateway.appPort` + `gateway.portOffset` in place of `gateway.port`.
10
+ - Added: `catalog()` on the router and `AiRouterService`: the pool's profiles and models by name.
11
+ - Changed: `createRouter(config)` is exported again, from the root and from the new Nest-free entry `@starci/ai-router/gateway`. 0.2.0 removed it to make the Nest module the one entry, not because the engine changed; with the gateway it is again the one engine behind the service, the gateway and plain Node apps.
12
+ - Changed: the Nest peer dependencies are optional, since the gateway and cli entries load no Nest code; `yaml` is a dependency for the config file.
13
+
14
+ ## 0.2.1 - 2026-10-28
15
+
16
+ - Fixed: 0.2.0 shipped corrupt declarations: the build rewrote `.ts` specifiers with a `$1` that sh expanded to nothing, so every `dist/*.d.ts` read `export { X } .js";` and a consumer tsc failed. The declaration step now names `.js` with a replacer function and `"`, and reads alike under sh and cmd.
17
+
3
18
  ## 0.2.0 - 2026-10-27
4
19
 
5
20
  - Breaking: the package is a NestJS module. The root entry is the Nest API (`AiRouterModule`, `AiRouterService`, the tokens and decorators, the options, config, policy and event types, `AiRouterError`, the `StateStore` type); `createRouter`, `definePool`, `validateConfig`, `MemoryStateStore`, `parseSse`, `parseDuration`, `readRateLimit` and `classifyStatus` are no longer exported, and the router engine is reached only through the service.
package/README.md CHANGED
@@ -1,13 +1,21 @@
1
1
  # @starci/ai-router
2
2
 
3
- A NestJS module that routes OpenAI-compatible calls across a pool. Register many keys, many models and many providers
4
- once at the app root, inject `AiRouterService`, and call `chat`, `stream` and `embeddings`: the router picks a healthy
5
- key for each attempt, backs off what is failing, honours rate limits, retries on a different key or provider, falls
6
- back to another model when you allow it, and tells you what it did without ever exposing a key or a prompt.
7
-
8
- - A Nest package: `@nestjs/common` and `@nestjs/core` 11 or 12, and `reflect-metadata`, are peer dependencies your app
9
- already has. No other runtime dependency: `fetch`, `AbortController` and `ReadableStream`. ESM, typed, Node 20 or
10
- newer.
3
+ A router for OpenAI-compatible calls across a pool of keys, models and providers, usable three ways over one engine:
4
+
5
+ - **as a gateway service** (`run.js serve`): an OpenAI-compatible HTTP endpoint (`/v1/chat/completions`, streaming
6
+ included, `/v1/embeddings`, `/v1/models`) with its own virtual API keys, per-key quotas and usage accounting, so any
7
+ app (LangChain, an OpenAI SDK, curl, a non-Node service) uses the pool by pointing its base URL at it;
8
+ - **as a NestJS module**: register the pool once at the app root, inject `AiRouterService`, and call `chat`, `stream`
9
+ and `embeddings`; `AiRouterGatewayModule` hosts the gateway in the same process;
10
+ - **as a plain Node library**: `createRouter(config)` from `@starci/ai-router/gateway`, with no Nest at all.
11
+
12
+ The router picks a healthy key for each attempt, backs off what is failing, honours rate limits, retries on a different
13
+ key or provider, falls back to another model when you allow it, and tells you what it did without ever exposing a key or
14
+ a prompt.
15
+
16
+ - ESM, typed, Node 20 or newer (the durable key store needs Node 22.13 or newer for `node:sqlite`). One runtime
17
+ dependency, `yaml`, for the config file. `@nestjs/common`, `@nestjs/core` (11 or 12) and `reflect-metadata` are
18
+ optional peer dependencies: the root entry needs them, `@starci/ai-router/gateway`, `@starci/ai-router/cli` and the `run.js` process entry do not.
11
19
  - OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
12
20
  behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
13
21
  - Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
@@ -17,16 +25,245 @@ back to another model when you allow it, and tells you what it did without ever
17
25
 
18
26
  One key hits its rate limit, another is revoked, a provider has a bad five minutes, and a model is briefly overloaded.
19
27
  Hand-written retry loops get each of those slightly wrong: they retry a 400, hammer a key that answered 401, ignore
20
- `retry-after`, or retry the same overloaded key. This package owns that loop once, as a Nest capability, with a testing
21
- module to prove your use of it.
28
+ `retry-after`, or retry the same overloaded key. This package owns that loop once, behind one gateway every app can
29
+ call, or as a Nest capability with a testing module to prove your use of it.
22
30
 
23
31
  ## Install
24
32
 
25
33
  ```sh
26
- npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata
34
+ npm install @starci/ai-router # the gateway and plain Node
35
+ npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata # in a Nest app
27
36
  ```
28
37
 
29
- ## Register
38
+ ## Run it as a gateway
39
+
40
+ One process (one container) holds the pool, the virtual keys and the admin API. The package declares no `bin` (the
41
+ harness has one CLI); the process entry is `dist/gateway/run.js`, and `main(argv, io)` from `@starci/ai-router/cli` runs
42
+ the same commands in process:
43
+
44
+ ```sh
45
+ node node_modules/@starci/ai-router/dist/gateway/run.js serve --config pool.yaml
46
+ # {"listening":"http://0.0.0.0:8080"}
47
+ ```
48
+
49
+ ### The config file
50
+
51
+ YAML (or JSON, by the `.json` extension). It is the pool of the [Configuration reference](#configuration-reference) plus
52
+ a `gateway` section; every secret is a reference by name, `{ env: NAME }` or `{ file: path }` (a mounted Docker or
53
+ Kubernetes secret, re-read at each attempt so rotation needs no restart). A field that would hold a secret's value
54
+ (`apiKey`, `value`, `token`, ...) or a provider header that would carry one (`authorization`, `x-api-key`, ...) is
55
+ refused when the file loads, as is any unknown field.
56
+
57
+ ```yaml
58
+ gateway:
59
+ port: 8080 # default 8080; or derive it from the app: appPort: 3001 (+ portOffset, default 1) serves on 3002
60
+ host: 0.0.0.0 # default 127.0.0.1; 0.0.0.0 in a container
61
+ adminToken: { env: AI_ROUTER_ADMIN_TOKEN } # absent: the admin API is off
62
+ keyStore: { type: sqlite, path: /data/keys.sqlite } # or { type: memory }; a relative path is from this file
63
+ maxBodyBytes: 4194304 # default 4 MiB
64
+ providers:
65
+ - id: openai
66
+ baseUrl: https://api.openai.com/v1
67
+ keys:
68
+ - { alias: team-a, env: OPENAI_KEY_A }
69
+ - { alias: team-b, file: /run/secrets/openai_key_b, maxConcurrency: 8 }
70
+ models:
71
+ - { id: gpt-4o-mini, aliases: [fast], cost: { input: 0.15, output: 0.6 } }
72
+ - { id: gpt-4o, aliases: [smart], cost: { input: 2.5, output: 10 } }
73
+ - { id: text-embedding-3-small, capabilities: [embeddings] }
74
+ - id: openrouter
75
+ baseUrl: https://openrouter.ai/api/v1
76
+ keys: [{ alias: or-1, env: OPENROUTER_KEY }]
77
+ models: [{ id: openai/gpt-4o-mini, aliases: [fast] }]
78
+ defaults: { fallbacks: [smart], deadlineMs: 60000 }
79
+ profiles:
80
+ lesson: { model: fast, fallbacks: [smart] }
81
+ grading: { model: smart, allowFallback: false, maxAttempts: 3 }
82
+ ```
83
+
84
+ ### Routes
85
+
86
+ | route | auth | does |
87
+ |---|---|---|
88
+ | `POST /v1/chat/completions` | virtual key | a chat completion; `stream: true` answers OpenAI `chat.completion.chunk` events over SSE ending with `data: [DONE]` |
89
+ | `POST /v1/embeddings` | virtual key | an embeddings response |
90
+ | `GET /v1/models`, `GET /v1/models/{id}` | virtual key | the profiles and pool models (ids and aliases) this key may call |
91
+ | `GET /health` | none; the admin token adds the pool's breakers and keys | `{ "status": "ok" }`, or `degraded` when no provider has a usable key |
92
+ | `/admin/keys...` | admin token | the [admin API](#admin-api) |
93
+
94
+ A request's `model` names a profile (`lesson`) or a pool model by id or alias (`gpt-4o-mini`, `fast`); a profile wins
95
+ when a name is both. A profile call routes by the profile's policy; a direct model call falls back only to the pool
96
+ defaults' fallbacks that the key may call itself. The answer is the provider's OpenAI body without the router's
97
+ `routing`; the response carries `x-request-id`, and a unary one `x-ai-router-model` (the model that answered) and
98
+ `x-ai-router-attempts`. For a stream the gateway always asks the provider for usage (for accounting) and forwards the
99
+ usage chunk only when the client set `stream_options.include_usage`. Nothing is sent before the first chunk, so a pool
100
+ failure before the first token is an ordinary error response; after it, the stream ends with an OpenAI error event.
101
+
102
+ Errors are OpenAI error objects, `{ "error": { "message", "type", "param", "code" } }`:
103
+
104
+ | status | code | when |
105
+ |---|---|---|
106
+ | 400 | `invalid_json`, `invalid_request`, `missing_model`, `model_not_supported` | a body the gateway cannot route |
107
+ | 401 | `missing_api_key`, `invalid_api_key` | no key, an unknown, revoked or disabled key |
108
+ | 403 | `model_not_allowed` | a profile or model the key is not bound to |
109
+ | 404 | `model_not_found`, `not_found` | a name the pool does not serve, an unknown route |
110
+ | 413 | `request_too_large` | a body past `maxBodyBytes` |
111
+ | 429 | `requests_per_minute_exceeded`, `tokens_per_day_exceeded`, `cost_per_day_exceeded`, `cost_per_month_exceeded` | the key's quota, with `Retry-After` |
112
+ | 4xx | `upstream_invalid_request` | the provider refused the request (its status and message, e.g. a context-length error) |
113
+ | 502 | `upstream_error`, `pool_auth_failed` | every attempt failed upstream, or no upstream key was accepted |
114
+ | 503 | `pool_unavailable`, `pool_rate_limited` | no lane can serve it, or every key is rate limited (`Retry-After` when known) |
115
+ | 504 | `pool_timeout` | the policy's deadline passed |
116
+
117
+ ### Virtual keys
118
+
119
+ The gateway issues its own keys, `sk-starci-<label>-<48 hex>`, and stores only their SHA-256: the secret is shown once,
120
+ when the key is created or rotated, and never again. Each key is bound to:
121
+
122
+ | field | meaning | default |
123
+ |---|---|---|
124
+ | `label` | lowercase letters, digits, `_`; part of the secret | required |
125
+ | `owner` | free text naming who holds it | none |
126
+ | `profiles` | profile names it may call, or `["*"]` | `[]` |
127
+ | `models` | pool model ids or aliases it may call directly, or `["*"]` | `[]` (at least one profile or model) |
128
+ | `allowFallback` | `false` pins every call to the profile's or named model | `true` |
129
+ | `quota.requestsPerMinute` | requests per UTC minute | none |
130
+ | `quota.tokensPerDay` | prompt plus completion tokens per UTC day | none |
131
+ | `quota.costPerDay`, `quota.costPerMonth` | estimated cost per UTC day or month, from usage and the model's `cost` (a model without one counts 0) | none |
132
+ | `enabled` | a disabled key answers 401 until enabled | `true` |
133
+
134
+ Tokens and cost are counted after a call, so a quota stops the request after the one that crossed it. Every request a
135
+ key makes is accounted per UTC day: requests, successes, failures, prompt and completion tokens, estimated cost, summed
136
+ latency and the outcomes (`success`, `error:<class>`, `rejected:<code>`). No prompt, response or end-user identifier is
137
+ stored.
138
+
139
+ Keys live in a key store. `{ type: memory }` loses them on restart; `{ type: sqlite, path }` is the durable store
140
+ (`node:sqlite`, WAL, several processes may share the file). Another database implements the exported `KeyStore`
141
+ interface; `KEY_STORE_SCHEMA` is the reference schema (it carries over to Postgres with `BIGINT` times,
142
+ `DOUBLE PRECISION` cost and `JSONB` lists).
143
+
144
+ ### Admin API
145
+
146
+ Guarded by `Authorization: Bearer <admin token>` (403 `admin_disabled` when no token is configured).
147
+
148
+ | route | does |
149
+ |---|---|
150
+ | `POST /admin/keys` | create from a key spec; 201 `{ key, secret }` |
151
+ | `GET /admin/keys` | list (never a secret or hash) |
152
+ | `GET /admin/keys/{id}`, `PATCH /admin/keys/{id}` | read, or change bindings (`owner`, `profiles`, `models`, `allowFallback`, `quota`, `enabled`) |
153
+ | `POST /admin/keys/{id}/revoke` | revoke for good |
154
+ | `POST /admin/keys/{id}/rotate` | new secret, same id, bindings and usage; 200 `{ key, secret }` |
155
+ | `GET /admin/keys/{id}/usage?from=YYYY-MM-DD&to=YYYY-MM-DD` | day rows and totals (default: this month) |
156
+
157
+ ### CLI
158
+
159
+ ```sh
160
+ RUN="node node_modules/@starci/ai-router/dist/gateway/run.js"
161
+ $RUN keys create --label lesson_app --profiles lesson --rpm 60 --cost-per-month 20 --config pool.yaml
162
+ $RUN keys list --config pool.yaml
163
+ $RUN keys usage key_1a2b3c4d5e6f7a8b --from 2026-10-01 --config pool.yaml
164
+ $RUN keys rotate key_1a2b3c4d5e6f7a8b --url http://gateway:8080 # admin token from AI_ROUTER_ADMIN_TOKEN
165
+ $RUN keys revoke key_1a2b3c4d5e6f7a8b --url http://gateway:8080 --admin-token-env MY_ADMIN_TOKEN
166
+ ```
167
+
168
+ `keys create|list|get|update|revoke|rotate|usage` run on the config's durable store directly (`--config`), or through a
169
+ running gateway's admin API (`--url`, required for a memory store). Every command prints JSON; `--help` lists
170
+ the flags.
171
+
172
+ ### Call it from any OpenAI client
173
+
174
+ LangChain (JS):
175
+
176
+ ```ts
177
+ import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"
178
+
179
+ const llm = new ChatOpenAI({
180
+ model: "lesson", // a profile, or a pool model such as "gpt-4o-mini"
181
+ apiKey: process.env.STARCI_AI_KEY, // sk-starci-...
182
+ configuration: { baseURL: "http://127.0.0.1:3002/v1" },
183
+ })
184
+ const reply = await llm.invoke("Explain closures in one paragraph.")
185
+ for await (const chunk of await llm.stream("Count to five.")) process.stdout.write(String(chunk.content))
186
+
187
+ const embeddings = new OpenAIEmbeddings({ model: "text-embedding-3-small", apiKey: process.env.STARCI_AI_KEY, configuration: { baseURL: "http://127.0.0.1:3002/v1" } })
188
+ ```
189
+
190
+ LangChain (Python): `ChatOpenAI(model="lesson", api_key=..., base_url="http://127.0.0.1:3002/v1")`.
191
+
192
+ The OpenAI SDK, or any compatible client:
193
+
194
+ ```ts
195
+ import OpenAI from "openai"
196
+
197
+ const client = new OpenAI({ apiKey: process.env.STARCI_AI_KEY, baseURL: "http://127.0.0.1:3002/v1" })
198
+ const completion = await client.chat.completions.create({ model: "lesson", messages: [{ role: "user", content: "hi" }] })
199
+ ```
200
+
201
+ ```sh
202
+ curl http://127.0.0.1:3002/v1/chat/completions -H "Authorization: Bearer $STARCI_AI_KEY" \
203
+ -H "content-type: application/json" -d '{"model":"lesson","stream":true,"messages":[{"role":"user","content":"hi"}]}'
204
+ ```
205
+
206
+ ### Deploy
207
+
208
+ One container or process: `node node_modules/@starci/ai-router/dist/gateway/run.js serve --config
209
+ /etc/ai-router/pool.yaml` as its command, `host: 0.0.0.0` in the config, the port published,
210
+ a volume for the SQLite file, and SIGTERM for a graceful stop. The environment holds only what the config names: the
211
+ upstream key variables (`OPENAI_KEY_A`, `OPENROUTER_KEY`, ... or mounted files), and the admin token variable
212
+ (`AI_ROUTER_ADMIN_TOKEN` by convention). The CLI's `--url` mode reads the admin token from `AI_ROUTER_ADMIN_TOKEN`
213
+ unless `--admin-token-env` names another variable. Breakers and headroom are per process unless the pool's `store` is
214
+ shared; run one gateway per pool, or several behind a load balancer sharing the SQLite file on one host.
215
+
216
+ ### Plain Node and in-process hosting
217
+
218
+ ```ts
219
+ import { createGateway, createRouter, openSqliteKeyStore } from "@starci/ai-router/gateway"
220
+
221
+ const router = createRouter(pool) // the same engine AiRouterService wraps: chat, stream, embeddings, health, stats, on, reset, catalog
222
+ const gateway = createGateway(router, { keys: await openSqliteKeyStore("keys.sqlite"), adminToken: () => process.env.AI_ROUTER_ADMIN_TOKEN })
223
+ await gateway.listen(8080, "0.0.0.0") // or mount gateway.handle on an existing node:http, Express or Nest adapter
224
+ ```
225
+
226
+ In a Nest app, `AiRouterGatewayModule` fronts a registered router (the default one, or `name` for a named one) and
227
+ listens on its own port when `listen` is given, closing at shutdown; the gateway is injected with
228
+ `@InjectAiRouterGateway(name?)` (token `AI_ROUTER_GATEWAY`):
229
+
230
+ ```ts
231
+ @Module({
232
+ imports: [
233
+ AiRouterModule.register({ isGlobal: true, ...pool }),
234
+ AiRouterGatewayModule.registerAsync({
235
+ useFactory: async () => ({ keys: await openSqliteKeyStore("/data/keys.sqlite"), adminToken: () => process.env.AI_ROUTER_ADMIN_TOKEN, listen: true }),
236
+ }),
237
+ ],
238
+ })
239
+ export class AppModule {}
240
+ ```
241
+
242
+ #### The port convention
243
+
244
+ The gateway's port is derived from the app's, so nobody configures it by hand. The StarCi port convention is
245
+ `basePort + portOffset`: the front end is base+0 (3000), the API is base+1 (3001), and the gateway is the API's port
246
+ plus an offset, 1 by default (3002).
247
+
248
+ | `listen` | gateway listens on |
249
+ |---|---|
250
+ | `true` | the app's port + 1, on `127.0.0.1` |
251
+ | `{ offset?: n, host?: h }` | the app's port + `n` (default 1, at least 1), on `h` (default `127.0.0.1`) |
252
+ | `{ port: p, host?: h }` | `p`, explicitly; the app's port is not read |
253
+ | absent | nothing; mount `gateway.handle` yourself |
254
+
255
+ The app's port is the port its HTTP server is actually bound to. Nest bootstraps before `app.listen` binds, so the
256
+ module waits for the server's `listening` event (or reads the address at once when it is already bound) and then
257
+ starts the gateway; a failure to bind the gateway's port (`EADDRINUSE`) goes to the `onError` option, not out of
258
+ `app.listen`. An application with no HTTP server (a standalone context) reads `PORT` instead; with neither, bootstrap
259
+ fails with an `AiRouterError` saying the port is unknown and `listen.port` is the way out. The host is local to the app
260
+ (`127.0.0.1`) and is `0.0.0.0` only when you set it. With `serve`, `gateway.appPort` plus `gateway.portOffset`
261
+ (default 1) does the same; it cannot be combined with `gateway.port`.
262
+
263
+ A client of an API on 3001 uses `http://127.0.0.1:3002/v1` as its OpenAI base URL: LangChain (below), OpenClaw, or any
264
+ OpenAI-compatible agent, with a virtual `sk-starci-...` key as its API key.
265
+
266
+ ## Use it in a Nest app
30
267
 
31
268
  The module follows the Nest configurable-module shape: `AiRouterModule.register(options)` or
32
269
  `AiRouterModule.registerAsync(...)`, registered once at the app root. The options are the pool
@@ -138,6 +375,7 @@ the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in
138
375
  | `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
139
376
  | `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled` until shutdown; returns the unsubscribe function |
140
377
  | `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
378
+ | `catalog()` | the pool's profiles (resolved model and fallbacks) and models (ids, aliases, providers), with no key |
141
379
 
142
380
  `options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
143
381
  `x-request-id` and `idempotency-key` on every attempt of the call, so a provider that deduplicates sees one request.
@@ -371,12 +609,16 @@ Everything is deterministic except the latencies you script.
371
609
  - Prompts and responses are not in events unless you set `telemetry.includeContent`.
372
610
  - `headers` in a provider or a call cannot override `authorization`.
373
611
  - The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
612
+ - The gateway never forwards a client's `Authorization` upstream, never answers with `routing` (the key alias), and keeps
613
+ virtual keys as SHA-256 hashes; the admin token is compared in constant time.
374
614
 
375
615
  ## FAQ
376
616
 
377
- **Is this a proxy server?** No, it is a module inside your Nest process. Expose it through a controller if you need one.
617
+ **Is this a proxy server?** It can be: `run.js serve` is an OpenAI-compatible gateway with virtual keys. Inside a Nest
618
+ app it is also a module you call directly, and `AiRouterGatewayModule` serves the gateway from that process.
378
619
 
379
- **Can I use it outside Nest?** No. The package is the Nest module; the engine behind the service is not exported.
620
+ **Can I use it outside Nest?** Yes: through the gateway over HTTP from any language, or in Node with `createRouter` and
621
+ `createGateway` from `@starci/ai-router/gateway`, which loads no Nest code.
380
622
 
381
623
  **Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
382
624
  the OpenAI chat-completions and embeddings API.
@@ -0,0 +1,30 @@
1
+ import type { AddressInfo } from "node:net";
2
+ /** The offset from the app's port the gateway listens at when none is given: the API on 3001 serves its gateway on 3002. */
3
+ export declare const DEFAULT_GATEWAY_PORT_OFFSET = 1;
4
+ /** Where the gateway listens: an explicit port, or a port derived from the app's own (`true`: offset 1 on 127.0.0.1). */
5
+ export type AiRouterGatewayListen = true | {
6
+ readonly port: number;
7
+ readonly host?: string;
8
+ readonly offset?: never;
9
+ } | {
10
+ readonly offset?: number;
11
+ readonly host?: string;
12
+ readonly port?: never;
13
+ };
14
+ /** The HTTP server of the Nest application, as far as the gateway needs it. */
15
+ export interface AppServer {
16
+ readonly listening: boolean;
17
+ address(): AddressInfo | string | null;
18
+ once(event: "listening", listener: () => void): unknown;
19
+ off(event: "listening", listener: () => void): unknown;
20
+ }
21
+ /** The port a listen setting means for an app on `appPort`, or the explicit port; refuses an unusable one. */
22
+ export declare function gatewayPort(listen: AiRouterGatewayListen, appPort: number | undefined): number;
23
+ /** The host of a listen setting: `undefined` (the gateway's 127.0.0.1) unless given. */
24
+ export declare function gatewayHost(listen: AiRouterGatewayListen): string | undefined;
25
+ /**
26
+ * Calls `start(port)` once the app's port is known: at once when the server is bound, when it emits `listening` when
27
+ * it is not yet (Nest bootstraps before `app.listen` binds), or from `PORT` when the app has no HTTP server. Returns
28
+ * a function that cancels a wait still pending. It throws when no port can be known.
29
+ */
30
+ export declare function whenAppPort(server: AppServer | undefined, env: NodeJS.ProcessEnv, start: (port: number) => void): () => void;
@@ -0,0 +1,67 @@
1
+ import { AiRouterError } from "./errors.js";
2
+ /** The offset from the app's port the gateway listens at when none is given: the API on 3001 serves its gateway on 3002. */
3
+ export const DEFAULT_GATEWAY_PORT_OFFSET = 1;
4
+ function failure(message) {
5
+ return new AiRouterError({ errorClass: "config", message });
6
+ }
7
+ /** The port a listen setting means for an app on `appPort`, or the explicit port; refuses an unusable one. */
8
+ export function gatewayPort(listen, appPort) {
9
+ if (listen !== true && listen.port !== undefined)
10
+ return listen.port;
11
+ const offset = listen === true ? DEFAULT_GATEWAY_PORT_OFFSET : (listen.offset ?? DEFAULT_GATEWAY_PORT_OFFSET);
12
+ if (!Number.isInteger(offset) || offset < 1)
13
+ throw failure("AiRouterGatewayModule listen.offset must be an integer of 1 or more");
14
+ if (appPort === undefined)
15
+ throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the app has no known port: listen with the application first, or set listen.port");
16
+ const port = appPort + offset;
17
+ if (port > 65535)
18
+ throw failure(`AiRouterGatewayModule listen: app port ${String(appPort)} + offset ${String(offset)} is not a port number`);
19
+ return port;
20
+ }
21
+ /** The host of a listen setting: `undefined` (the gateway's 127.0.0.1) unless given. */
22
+ export function gatewayHost(listen) {
23
+ return listen === true ? undefined : listen.host;
24
+ }
25
+ /** The port the application's HTTP server is bound to, or `undefined` when it is not bound to a TCP port. */
26
+ function boundPort(server) {
27
+ const address = server.address();
28
+ return typeof address === "object" && address !== null ? address.port : undefined;
29
+ }
30
+ /** The app's port from the `PORT` environment variable, when it is a port number. */
31
+ function envPort(env) {
32
+ const value = env.PORT;
33
+ if (value === undefined || !/^\d+$/.test(value))
34
+ return undefined;
35
+ const port = Number(value);
36
+ return port <= 65535 ? port : undefined;
37
+ }
38
+ /**
39
+ * Calls `start(port)` once the app's port is known: at once when the server is bound, when it emits `listening` when
40
+ * it is not yet (Nest bootstraps before `app.listen` binds), or from `PORT` when the app has no HTTP server. Returns
41
+ * a function that cancels a wait still pending. It throws when no port can be known.
42
+ */
43
+ export function whenAppPort(server, env, start) {
44
+ if (server === undefined) {
45
+ const port = envPort(env);
46
+ if (port === undefined)
47
+ throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the application has no HTTP server and PORT is not set: set listen.port");
48
+ start(port);
49
+ return () => undefined;
50
+ }
51
+ if (server.listening) {
52
+ const port = boundPort(server);
53
+ if (port === undefined)
54
+ throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the application's server is not on a TCP port: set listen.port");
55
+ start(port);
56
+ return () => undefined;
57
+ }
58
+ const onListening = () => {
59
+ const port = boundPort(server);
60
+ if (port !== undefined)
61
+ start(port);
62
+ };
63
+ server.once("listening", onListening);
64
+ return () => {
65
+ server.off("listening", onListening);
66
+ };
67
+ }
@@ -0,0 +1,21 @@
1
+ import type { AiRouterGatewayListen } from "./ai-router-gateway.listen.js";
2
+ import type { GatewayOptions } from "./gateway/gateway.js";
3
+ /** What the gateway module takes: the gateway's options, and where it listens. */
4
+ export interface AiRouterGatewayModuleOptions extends GatewayOptions {
5
+ /**
6
+ * Listens on its own port and stops at shutdown; absent, mount `gateway.handle` yourself. `{ port }` is explicit;
7
+ * `true` or `{ offset?, host? }` is the app's own port plus `offset` (default 1) once the app listens, on `host`
8
+ * (default 127.0.0.1): an API on 3001 serves its gateway on 3002.
9
+ */
10
+ readonly listen?: AiRouterGatewayListen;
11
+ }
12
+ /** The extras of a registration: its visibility, and the `name` of the `AiRouterModule` registration it fronts (the default router when absent). */
13
+ export interface AiRouterGatewayModuleExtras {
14
+ readonly isGlobal: boolean;
15
+ readonly name: string | undefined;
16
+ }
17
+ export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterGatewayModuleOptions, "register", "createAiRouterGatewayOptions", AiRouterGatewayModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: AiRouterGatewayModuleOptions & Partial<AiRouterGatewayModuleExtras>, ASYNC_OPTIONS_TYPE: import("@nestjs/common").ConfigurableModuleAsyncOptions<AiRouterGatewayModuleOptions, "createAiRouterGatewayOptions"> & Partial<AiRouterGatewayModuleExtras>;
18
+ /** What `AiRouterGatewayModule.register` takes: the options, plus `isGlobal` and `name`. */
19
+ export type AiRouterGatewayRegisterOptions = typeof OPTIONS_TYPE;
20
+ /** What `AiRouterGatewayModule.registerAsync` takes: `imports`, `inject` and one of `useFactory`, `useClass`, `useExisting`, plus `isGlobal` and `name`. */
21
+ export type AiRouterGatewayRegisterAsyncOptions = typeof ASYNC_OPTIONS_TYPE;
@@ -0,0 +1,6 @@
1
+ import { ConfigurableModuleBuilder } from "@nestjs/common";
2
+ export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = new ConfigurableModuleBuilder({ moduleName: "AiRouterGateway" })
3
+ .setClassMethodName("register")
4
+ .setFactoryMethodName("createAiRouterGatewayOptions")
5
+ .setExtras({ isGlobal: false, name: undefined }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
6
+ .build();
@@ -0,0 +1,19 @@
1
+ import type { DynamicModule } from "@nestjs/common";
2
+ import { ConfigurableModuleClass } from "./ai-router-gateway.module-definition.js";
3
+ import type { AiRouterGatewayRegisterAsyncOptions, AiRouterGatewayRegisterOptions } from "./ai-router-gateway.module-definition.js";
4
+ /** The injection token of the default router's gateway. */
5
+ export declare const AI_ROUTER_GATEWAY = "AI_ROUTER_GATEWAY";
6
+ /** The token of a gateway: `AI_ROUTER_GATEWAY` over the default router, `AI_ROUTER_GATEWAY:<name>` over a named one. */
7
+ export declare function getAiRouterGatewayToken(name?: string): string;
8
+ /** Injects the gateway over the default router, or over the router registered under `name`. */
9
+ export declare function InjectAiRouterGateway(name?: string): PropertyDecorator & ParameterDecorator;
10
+ /**
11
+ * The OpenAI-compatible gateway hosted in a Nest application, over the router an `AiRouterModule` registration
12
+ * provides: the router must be visible here (an `isGlobal` registration, or its module in `imports` of `registerAsync`).
13
+ */
14
+ export declare class AiRouterGatewayModule extends ConfigurableModuleClass {
15
+ /** Registers a gateway from options known now. */
16
+ static register(options: AiRouterGatewayRegisterOptions): DynamicModule;
17
+ /** Registers a gateway from options a factory, class or existing provider supplies. */
18
+ static registerAsync(options: AiRouterGatewayRegisterAsyncOptions): DynamicModule;
19
+ }
@@ -0,0 +1,75 @@
1
+ import { Inject, Module } from "@nestjs/common";
2
+ import { HttpAdapterHost } from "@nestjs/core";
3
+ import { gatewayHost, gatewayPort, whenAppPort } from "./ai-router-gateway.listen.js";
4
+ import { getAiRouterToken } from "./ai-router.tokens.js";
5
+ import { createGateway } from "./gateway/gateway.js";
6
+ import { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN } from "./ai-router-gateway.module-definition.js";
7
+ /** The injection token of the default router's gateway. */
8
+ export const AI_ROUTER_GATEWAY = "AI_ROUTER_GATEWAY";
9
+ /** The token of a gateway: `AI_ROUTER_GATEWAY` over the default router, `AI_ROUTER_GATEWAY:<name>` over a named one. */
10
+ export function getAiRouterGatewayToken(name) {
11
+ return name === undefined ? AI_ROUTER_GATEWAY : `${AI_ROUTER_GATEWAY}:${name}`;
12
+ }
13
+ /** Injects the gateway over the default router, or over the router registered under `name`. */
14
+ export function InjectAiRouterGateway(name) {
15
+ return Inject(getAiRouterGatewayToken(name));
16
+ }
17
+ /** Adds the gateway over the named router, and the hooks that listen at bootstrap and close at shutdown. */
18
+ function withGateway(base, name) {
19
+ const token = getAiRouterGatewayToken(name);
20
+ const providers = [
21
+ {
22
+ provide: token,
23
+ inject: [getAiRouterToken(name), MODULE_OPTIONS_TOKEN],
24
+ useFactory: (router, options) => createGateway(router, options),
25
+ },
26
+ {
27
+ // Nest calls the lifecycle hooks of every provider instance that has them.
28
+ provide: `${token}:lifecycle`,
29
+ inject: [token, MODULE_OPTIONS_TOKEN, { token: HttpAdapterHost, optional: true }],
30
+ useFactory: (gateway, options, host) => {
31
+ let cancel = () => undefined;
32
+ return {
33
+ async onApplicationBootstrap() {
34
+ const listen = options.listen;
35
+ if (listen === undefined)
36
+ return;
37
+ if (listen !== true && listen.port !== undefined) {
38
+ await gateway.listen(listen.port, listen.host);
39
+ return;
40
+ }
41
+ // The adapter is unset until an application is created over the module (a standalone context has none).
42
+ const adapter = host?.httpAdapter;
43
+ const server = adapter?.getHttpServer();
44
+ // Bootstrap runs before `app.listen` binds the app, so a port not yet bound is waited for, not awaited.
45
+ cancel = whenAppPort(server, process.env, (appPort) => {
46
+ gateway.listen(gatewayPort(listen, appPort), gatewayHost(listen)).catch((error) => {
47
+ options.onError?.(error);
48
+ });
49
+ });
50
+ },
51
+ onApplicationShutdown() {
52
+ cancel();
53
+ return gateway.close();
54
+ },
55
+ };
56
+ },
57
+ },
58
+ ];
59
+ return { ...base, providers: [...(base.providers ?? []), ...providers], exports: [...(base.exports ?? []), token] };
60
+ }
61
+ /**
62
+ * The OpenAI-compatible gateway hosted in a Nest application, over the router an `AiRouterModule` registration
63
+ * provides: the router must be visible here (an `isGlobal` registration, or its module in `imports` of `registerAsync`).
64
+ */
65
+ export class AiRouterGatewayModule extends ConfigurableModuleClass {
66
+ /** Registers a gateway from options known now. */
67
+ static register(options) {
68
+ return withGateway(super.register(options), options.name);
69
+ }
70
+ /** Registers a gateway from options a factory, class or existing provider supplies. */
71
+ static registerAsync(options) {
72
+ return withGateway(super.registerAsync(options), options.name);
73
+ }
74
+ }
75
+ Module({})(AiRouterGatewayModule);
@@ -1,6 +1,6 @@
1
1
  import type { DynamicModule } from "@nestjs/common";
2
- import type { AiRouterRegisterOptions } .js";
3
- import type { FakeProvider } .js";
2
+ import type { AiRouterRegisterOptions } from "./ai-router.module-definition.js";
3
+ import type { FakeProvider } from "./fake.js";
4
4
  /** What `AiRouterTestingModule.register` takes: a registration whose requests a fake provider answers. */
5
5
  export type AiRouterTestingOptions = Omit<AiRouterRegisterOptions, "fetch"> & {
6
6
  /** The provider every request goes to, in process; script it with `fake.on` and read `fake.calls`. */
@@ -1,11 +1,11 @@
1
- import type { AiRouterModuleOptions } .js";
1
+ import type { AiRouterModuleOptions } from "./ai-router.options.js";
2
2
  /** The extras every registration may set: global visibility, and the name of one of several pools. */
3
3
  export interface AiRouterModuleExtras {
4
4
  readonly isGlobal: boolean;
5
5
  readonly name: string | undefined;
6
6
  }
7
- export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterModuleOptions, "register", "createAiRouterOptions", AiRouterModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: .js").PoolConfig<string> & {
8
- readonly on?: .js").AiRouterHandlers;
7
+ export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterModuleOptions, "register", "createAiRouterOptions", AiRouterModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: import("./types.js").PoolConfig<string> & {
8
+ readonly on?: import("./ai-router.options.js").AiRouterHandlers;
9
9
  } & Partial<AiRouterModuleExtras>, ASYNC_OPTIONS_TYPE: import("@nestjs/common").ConfigurableModuleAsyncOptions<AiRouterModuleOptions, "createAiRouterOptions"> & Partial<AiRouterModuleExtras>;
10
10
  /** What `AiRouterModule.register` takes: the options, plus `isGlobal` and `name`. */
11
11
  export type AiRouterRegisterOptions = typeof OPTIONS_TYPE;
@@ -1,6 +1,6 @@
1
1
  import type { DynamicModule } from "@nestjs/common";
2
- import { ConfigurableModuleClass } .js";
3
- import type { AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } .js";
2
+ import { ConfigurableModuleClass } from "./ai-router.module-definition.js";
3
+ import type { AiRouterRegisterAsyncOptions, AiRouterRegisterOptions } from "./ai-router.module-definition.js";
4
4
  /** A pool router as a Nest capability: registered once at the app root, or once per name for several pools. */
5
5
  export declare class AiRouterModule extends ConfigurableModuleClass {
6
6
  /** Registers a router from options known now. */
@@ -1,4 +1,4 @@
1
- import type { PoolConfig, RouterEvents } .js";
1
+ import type { PoolConfig, RouterEvents } from "./types.js";
2
2
  /** One handler per router event, subscribed when the module starts and released when the application shuts down. */
3
3
  export type AiRouterHandlers = {
4
4
  readonly [E in keyof RouterEvents]?: (payload: RouterEvents[E]) => void;
@@ -1,6 +1,6 @@
1
1
  import type { OnApplicationShutdown } from "@nestjs/common";
2
- import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterEvents, StatsSnapshot } .js";
3
- import type { AiRouterHandlers } .js";
2
+ import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterCatalog, RouterEvents, StatsSnapshot } from "./types.js";
3
+ import type { AiRouterHandlers } from "./ai-router.options.js";
4
4
  /**
5
5
  * The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
6
6
  * handler it subscribed is released then; calls made on the raw router (`AI_ROUTER`) are the caller's to cancel.
@@ -24,6 +24,8 @@ export declare class AiRouterService<Profiles extends string = string> implement
24
24
  on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
25
25
  /** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
26
26
  reset(target?: ResetTarget): Promise<void>;
27
+ /** Lists the pool's profiles and models by name. */
28
+ catalog(): RouterCatalog;
27
29
  /** Aborts every call still in flight through this service and releases its handlers. */
28
30
  onApplicationShutdown(): void;
29
31
  }
@@ -56,6 +56,10 @@ export class AiRouterService {
56
56
  reset(target) {
57
57
  return this.router.reset(target);
58
58
  }
59
+ /** Lists the pool's profiles and models by name. */
60
+ catalog() {
61
+ return this.router.catalog();
62
+ }
59
63
  /** Aborts every call still in flight through this service and releases its handlers. */
60
64
  onApplicationShutdown() {
61
65
  this.#shutdown.abort();