@starci/ai-router 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +256 -14
- package/dist/ai-router-gateway.listen.d.ts +30 -0
- package/dist/ai-router-gateway.listen.js +67 -0
- package/dist/ai-router-gateway.module-definition.d.ts +21 -0
- package/dist/ai-router-gateway.module-definition.js +6 -0
- package/dist/ai-router-gateway.module.d.ts +19 -0
- package/dist/ai-router-gateway.module.js +75 -0
- package/dist/ai-router.service.d.ts +3 -1
- package/dist/ai-router.service.js +4 -0
- package/dist/config.d.ts +3 -1
- package/dist/config.js +41 -0
- package/dist/gateway/cli.d.ts +13 -0
- package/dist/gateway/cli.js +241 -0
- package/dist/gateway/config-file.d.ts +37 -0
- package/dist/gateway/config-file.js +238 -0
- package/dist/gateway/gateway.d.ts +34 -0
- package/dist/gateway/gateway.js +527 -0
- package/dist/gateway/index.d.ts +14 -0
- package/dist/gateway/index.js +7 -0
- package/dist/gateway/keys.d.ts +166 -0
- package/dist/gateway/keys.js +268 -0
- package/dist/gateway/run.d.ts +1 -0
- package/dist/gateway/run.js +14 -0
- package/dist/gateway/serve.d.ts +25 -0
- package/dist/gateway/serve.js +38 -0
- package/dist/gateway/sqlite-key-store.d.ts +8 -0
- package/dist/gateway/sqlite-key-store.js +167 -0
- package/dist/index.d.ts +4 -4
- package/dist/index.js +2 -1
- package/dist/router.js +3 -1
- package/dist/types.d.ts +28 -0
- package/package.json +29 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0 - 2026-10-28
|
|
4
|
+
|
|
5
|
+
- Added: an OpenAI-compatible HTTP gateway over the same router (`createGateway(router, options)`): `POST /v1/chat/completions` (SSE streaming in OpenAI's chunk format, ending with `[DONE]`), `POST /v1/embeddings`, `GET /v1/models` (the profiles and pool models the calling key may use), `GET /health`, and OpenAI error objects with 400, 401, 403, 404, 413, 429 (with `Retry-After`), 502, 503 and 504. A request's `model` names a profile or a pool model id or alias.
|
|
6
|
+
- Added: virtual API keys (`sk-starci-<label>-<random>`), stored as SHA-256 only and shown once at create and rotate, each bound to profiles, models, fallback, quotas (requests per minute, tokens per day, cost per day and month), enabled and revoked state, label and owner; usage per key and UTC day (requests, outcomes, tokens, estimated cost, latency), with no prompt or end-user identifier. `KeyAdmin` (create, list, get, update, revoke, rotate, usage), the `KeyStore` interface, `MemoryKeyStore`, and the durable `openSqliteKeyStore(path)` over `node:sqlite` with its `KEY_STORE_SCHEMA`.
|
|
7
|
+
- Added: the admin API under `/admin/keys`, guarded by an admin token, and the commands `serve --config <pool.yaml|json>` and `keys create|list|get|update|revoke|rotate|usage`, on the durable store (`--config`) or through a running gateway (`--url`), run by the process entry `dist/gateway/run.js` or in process with `main` from `@starci/ai-router/cli`; the package declares no `bin`, since the harness has one CLI. The config file names every secret by reference (`{ env }` or `{ file }`) and refuses one written inline. `loadGatewayConfig`, `parseGatewayConfig`, `serveGateway`, `openKeyStore`.
|
|
8
|
+
- Added: `AiRouterGatewayModule.register` / `registerAsync` hosting the gateway in a Nest app over a registered router, `AI_ROUTER_GATEWAY`, `getAiRouterGatewayToken` and `@InjectAiRouterGateway()`.
|
|
9
|
+
- Added: the gateway's port follows the StarCi port convention (`basePort + portOffset`: FE base+0, API base+1): `AiRouterGatewayModule`'s `listen` takes `{ port }` (explicit), or `true` / `{ offset?, host? }` for the app's own bound port plus `offset` (default 1, so an API on 3001 serves its gateway on 3002), on `127.0.0.1` unless `host` is set. The app's port is read from its HTTP server once it listens (the module waits for `listening`, since Nest bootstraps before `app.listen`), from `PORT` when the app has no HTTP server, and bootstrap refuses with an `AiRouterError` when neither is known; a bind failure goes to `onError`. `serve` takes `gateway.appPort` + `gateway.portOffset` in place of `gateway.port`.
|
|
10
|
+
- Added: `catalog()` on the router and `AiRouterService`: the pool's profiles and models by name.
|
|
11
|
+
- Changed: `createRouter(config)` is exported again, from the root and from the new Nest-free entry `@starci/ai-router/gateway`. 0.2.0 removed it to make the Nest module the one entry, not because the engine changed; with the gateway it is again the one engine behind the service, the gateway and plain Node apps.
|
|
12
|
+
- Changed: the Nest peer dependencies are optional, since the gateway and cli entries load no Nest code; `yaml` is a dependency for the config file.
|
|
13
|
+
|
|
3
14
|
## 0.2.1 - 2026-10-28
|
|
4
15
|
|
|
5
16
|
- Fixed: 0.2.0 shipped corrupt declarations: the build rewrote `.ts` specifiers with a `$1` that sh expanded to nothing, so every `dist/*.d.ts` read `export { X } .js";` and a consumer tsc failed. The declaration step now names `.js` with a replacer function and `"`, and reads alike under sh and cmd.
|
package/README.md
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
# @starci/ai-router
|
|
2
2
|
|
|
3
|
-
A
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
A router for OpenAI-compatible calls across a pool of keys, models and providers, usable three ways over one engine:
|
|
4
|
+
|
|
5
|
+
- **as a gateway service** (`run.js serve`): an OpenAI-compatible HTTP endpoint (`/v1/chat/completions`, streaming
|
|
6
|
+
included, `/v1/embeddings`, `/v1/models`) with its own virtual API keys, per-key quotas and usage accounting, so any
|
|
7
|
+
app (LangChain, an OpenAI SDK, curl, a non-Node service) uses the pool by pointing its base URL at it;
|
|
8
|
+
- **as a NestJS module**: register the pool once at the app root, inject `AiRouterService`, and call `chat`, `stream`
|
|
9
|
+
and `embeddings`; `AiRouterGatewayModule` hosts the gateway in the same process;
|
|
10
|
+
- **as a plain Node library**: `createRouter(config)` from `@starci/ai-router/gateway`, with no Nest at all.
|
|
11
|
+
|
|
12
|
+
The router picks a healthy key for each attempt, backs off what is failing, honours rate limits, retries on a different
|
|
13
|
+
key or provider, falls back to another model when you allow it, and tells you what it did without ever exposing a key or
|
|
14
|
+
a prompt.
|
|
15
|
+
|
|
16
|
+
- ESM, typed, Node 20 or newer (the durable key store needs Node 22.13 or newer for `node:sqlite`). One runtime
|
|
17
|
+
dependency, `yaml`, for the config file. `@nestjs/common`, `@nestjs/core` (11 or 12) and `reflect-metadata` are
|
|
18
|
+
optional peer dependencies: the root entry needs them, `@starci/ai-router/gateway`, `@starci/ai-router/cli` and the `run.js` process entry do not.
|
|
11
19
|
- OpenAI chat-completions and embeddings shapes in and out, so any OpenAI-compatible endpoint (OpenAI, Azure OpenAI
|
|
12
20
|
behind a compatible gateway, OpenRouter, Groq, Together, DeepSeek, vLLM, Ollama, LiteLLM...) is a provider.
|
|
13
21
|
- Keys are references (an environment variable name or a resolver function), read at each attempt and named by alias
|
|
@@ -17,16 +25,245 @@ back to another model when you allow it, and tells you what it did without ever
|
|
|
17
25
|
|
|
18
26
|
One key hits its rate limit, another is revoked, a provider has a bad five minutes, and a model is briefly overloaded.
|
|
19
27
|
Hand-written retry loops get each of those slightly wrong: they retry a 400, hammer a key that answered 401, ignore
|
|
20
|
-
`retry-after`, or retry the same overloaded key. This package owns that loop once,
|
|
21
|
-
module to prove your use of it.
|
|
28
|
+
`retry-after`, or retry the same overloaded key. This package owns that loop once, behind one gateway every app can
|
|
29
|
+
call, or as a Nest capability with a testing module to prove your use of it.
|
|
22
30
|
|
|
23
31
|
## Install
|
|
24
32
|
|
|
25
33
|
```sh
|
|
26
|
-
npm install @starci/ai-router
|
|
34
|
+
npm install @starci/ai-router # the gateway and plain Node
|
|
35
|
+
npm install @starci/ai-router @nestjs/common @nestjs/core reflect-metadata # in a Nest app
|
|
27
36
|
```
|
|
28
37
|
|
|
29
|
-
##
|
|
38
|
+
## Run it as a gateway
|
|
39
|
+
|
|
40
|
+
One process (one container) holds the pool, the virtual keys and the admin API. The package declares no `bin` (the
|
|
41
|
+
harness has one CLI); the process entry is `dist/gateway/run.js`, and `main(argv, io)` from `@starci/ai-router/cli` runs
|
|
42
|
+
the same commands in process:
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
node node_modules/@starci/ai-router/dist/gateway/run.js serve --config pool.yaml
|
|
46
|
+
# {"listening":"http://0.0.0.0:8080"}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### The config file
|
|
50
|
+
|
|
51
|
+
YAML (or JSON, by the `.json` extension). It is the pool of the [Configuration reference](#configuration-reference) plus
|
|
52
|
+
a `gateway` section; every secret is a reference by name, `{ env: NAME }` or `{ file: path }` (a mounted Docker or
|
|
53
|
+
Kubernetes secret, re-read at each attempt so rotation needs no restart). A field that would hold a secret's value
|
|
54
|
+
(`apiKey`, `value`, `token`, ...) or a provider header that would carry one (`authorization`, `x-api-key`, ...) is
|
|
55
|
+
refused when the file loads, as is any unknown field.
|
|
56
|
+
|
|
57
|
+
```yaml
|
|
58
|
+
gateway:
|
|
59
|
+
port: 8080 # default 8080; or derive it from the app: appPort: 3001 (+ portOffset, default 1) serves on 3002
|
|
60
|
+
host: 0.0.0.0 # default 127.0.0.1; 0.0.0.0 in a container
|
|
61
|
+
adminToken: { env: AI_ROUTER_ADMIN_TOKEN } # absent: the admin API is off
|
|
62
|
+
keyStore: { type: sqlite, path: /data/keys.sqlite } # or { type: memory }; a relative path is from this file
|
|
63
|
+
maxBodyBytes: 4194304 # default 4 MiB
|
|
64
|
+
providers:
|
|
65
|
+
- id: openai
|
|
66
|
+
baseUrl: https://api.openai.com/v1
|
|
67
|
+
keys:
|
|
68
|
+
- { alias: team-a, env: OPENAI_KEY_A }
|
|
69
|
+
- { alias: team-b, file: /run/secrets/openai_key_b, maxConcurrency: 8 }
|
|
70
|
+
models:
|
|
71
|
+
- { id: gpt-4o-mini, aliases: [fast], cost: { input: 0.15, output: 0.6 } }
|
|
72
|
+
- { id: gpt-4o, aliases: [smart], cost: { input: 2.5, output: 10 } }
|
|
73
|
+
- { id: text-embedding-3-small, capabilities: [embeddings] }
|
|
74
|
+
- id: openrouter
|
|
75
|
+
baseUrl: https://openrouter.ai/api/v1
|
|
76
|
+
keys: [{ alias: or-1, env: OPENROUTER_KEY }]
|
|
77
|
+
models: [{ id: openai/gpt-4o-mini, aliases: [fast] }]
|
|
78
|
+
defaults: { fallbacks: [smart], deadlineMs: 60000 }
|
|
79
|
+
profiles:
|
|
80
|
+
lesson: { model: fast, fallbacks: [smart] }
|
|
81
|
+
grading: { model: smart, allowFallback: false, maxAttempts: 3 }
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Routes
|
|
85
|
+
|
|
86
|
+
| route | auth | does |
|
|
87
|
+
|---|---|---|
|
|
88
|
+
| `POST /v1/chat/completions` | virtual key | a chat completion; `stream: true` answers OpenAI `chat.completion.chunk` events over SSE ending with `data: [DONE]` |
|
|
89
|
+
| `POST /v1/embeddings` | virtual key | an embeddings response |
|
|
90
|
+
| `GET /v1/models`, `GET /v1/models/{id}` | virtual key | the profiles and pool models (ids and aliases) this key may call |
|
|
91
|
+
| `GET /health` | none; the admin token adds the pool's breakers and keys | `{ "status": "ok" }`, or `degraded` when no provider has a usable key |
|
|
92
|
+
| `/admin/keys...` | admin token | the [admin API](#admin-api) |
|
|
93
|
+
|
|
94
|
+
A request's `model` names a profile (`lesson`) or a pool model by id or alias (`gpt-4o-mini`, `fast`); a profile wins
|
|
95
|
+
when a name is both. A profile call routes by the profile's policy; a direct model call falls back only to the pool
|
|
96
|
+
defaults' fallbacks that the key may call itself. The answer is the provider's OpenAI body without the router's
|
|
97
|
+
`routing`; the response carries `x-request-id`, and a unary one `x-ai-router-model` (the model that answered) and
|
|
98
|
+
`x-ai-router-attempts`. For a stream the gateway always asks the provider for usage (for accounting) and forwards the
|
|
99
|
+
usage chunk only when the client set `stream_options.include_usage`. Nothing is sent before the first chunk, so a pool
|
|
100
|
+
failure before the first token is an ordinary error response; after it, the stream ends with an OpenAI error event.
|
|
101
|
+
|
|
102
|
+
Errors are OpenAI error objects, `{ "error": { "message", "type", "param", "code" } }`:
|
|
103
|
+
|
|
104
|
+
| status | code | when |
|
|
105
|
+
|---|---|---|
|
|
106
|
+
| 400 | `invalid_json`, `invalid_request`, `missing_model`, `model_not_supported` | a body the gateway cannot route |
|
|
107
|
+
| 401 | `missing_api_key`, `invalid_api_key` | no key, an unknown, revoked or disabled key |
|
|
108
|
+
| 403 | `model_not_allowed` | a profile or model the key is not bound to |
|
|
109
|
+
| 404 | `model_not_found`, `not_found` | a name the pool does not serve, an unknown route |
|
|
110
|
+
| 413 | `request_too_large` | a body past `maxBodyBytes` |
|
|
111
|
+
| 429 | `requests_per_minute_exceeded`, `tokens_per_day_exceeded`, `cost_per_day_exceeded`, `cost_per_month_exceeded` | the key's quota, with `Retry-After` |
|
|
112
|
+
| 4xx | `upstream_invalid_request` | the provider refused the request (its status and message, e.g. a context-length error) |
|
|
113
|
+
| 502 | `upstream_error`, `pool_auth_failed` | every attempt failed upstream, or no upstream key was accepted |
|
|
114
|
+
| 503 | `pool_unavailable`, `pool_rate_limited` | no lane can serve it, or every key is rate limited (`Retry-After` when known) |
|
|
115
|
+
| 504 | `pool_timeout` | the policy's deadline passed |
|
|
116
|
+
|
|
117
|
+
### Virtual keys
|
|
118
|
+
|
|
119
|
+
The gateway issues its own keys, `sk-starci-<label>-<48 hex>`, and stores only their SHA-256: the secret is shown once,
|
|
120
|
+
when the key is created or rotated, and never again. Each key is bound to:
|
|
121
|
+
|
|
122
|
+
| field | meaning | default |
|
|
123
|
+
|---|---|---|
|
|
124
|
+
| `label` | lowercase letters, digits, `_`; part of the secret | required |
|
|
125
|
+
| `owner` | free text naming who holds it | none |
|
|
126
|
+
| `profiles` | profile names it may call, or `["*"]` | `[]` |
|
|
127
|
+
| `models` | pool model ids or aliases it may call directly, or `["*"]` | `[]` (at least one profile or model) |
|
|
128
|
+
| `allowFallback` | `false` pins every call to the profile's or named model | `true` |
|
|
129
|
+
| `quota.requestsPerMinute` | requests per UTC minute | none |
|
|
130
|
+
| `quota.tokensPerDay` | prompt plus completion tokens per UTC day | none |
|
|
131
|
+
| `quota.costPerDay`, `quota.costPerMonth` | estimated cost per UTC day or month, from usage and the model's `cost` (a model without one counts 0) | none |
|
|
132
|
+
| `enabled` | a disabled key answers 401 until enabled | `true` |
|
|
133
|
+
|
|
134
|
+
Tokens and cost are counted after a call, so a quota stops the request after the one that crossed it. Every request a
|
|
135
|
+
key makes is accounted per UTC day: requests, successes, failures, prompt and completion tokens, estimated cost, summed
|
|
136
|
+
latency and the outcomes (`success`, `error:<class>`, `rejected:<code>`). No prompt, response or end-user identifier is
|
|
137
|
+
stored.
|
|
138
|
+
|
|
139
|
+
Keys live in a key store. `{ type: memory }` loses them on restart; `{ type: sqlite, path }` is the durable store
|
|
140
|
+
(`node:sqlite`, WAL, several processes may share the file). Another database implements the exported `KeyStore`
|
|
141
|
+
interface; `KEY_STORE_SCHEMA` is the reference schema (it carries over to Postgres with `BIGINT` times,
|
|
142
|
+
`DOUBLE PRECISION` cost and `JSONB` lists).
|
|
143
|
+
|
|
144
|
+
### Admin API
|
|
145
|
+
|
|
146
|
+
Guarded by `Authorization: Bearer <admin token>` (403 `admin_disabled` when no token is configured).
|
|
147
|
+
|
|
148
|
+
| route | does |
|
|
149
|
+
|---|---|
|
|
150
|
+
| `POST /admin/keys` | create from a key spec; 201 `{ key, secret }` |
|
|
151
|
+
| `GET /admin/keys` | list (never a secret or hash) |
|
|
152
|
+
| `GET /admin/keys/{id}`, `PATCH /admin/keys/{id}` | read, or change bindings (`owner`, `profiles`, `models`, `allowFallback`, `quota`, `enabled`) |
|
|
153
|
+
| `POST /admin/keys/{id}/revoke` | revoke for good |
|
|
154
|
+
| `POST /admin/keys/{id}/rotate` | new secret, same id, bindings and usage; 200 `{ key, secret }` |
|
|
155
|
+
| `GET /admin/keys/{id}/usage?from=YYYY-MM-DD&to=YYYY-MM-DD` | day rows and totals (default: this month) |
|
|
156
|
+
|
|
157
|
+
### CLI
|
|
158
|
+
|
|
159
|
+
```sh
|
|
160
|
+
RUN="node node_modules/@starci/ai-router/dist/gateway/run.js"
|
|
161
|
+
$RUN keys create --label lesson_app --profiles lesson --rpm 60 --cost-per-month 20 --config pool.yaml
|
|
162
|
+
$RUN keys list --config pool.yaml
|
|
163
|
+
$RUN keys usage key_1a2b3c4d5e6f7a8b --from 2026-10-01 --config pool.yaml
|
|
164
|
+
$RUN keys rotate key_1a2b3c4d5e6f7a8b --url http://gateway:8080 # admin token from AI_ROUTER_ADMIN_TOKEN
|
|
165
|
+
$RUN keys revoke key_1a2b3c4d5e6f7a8b --url http://gateway:8080 --admin-token-env MY_ADMIN_TOKEN
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`keys create|list|get|update|revoke|rotate|usage` run on the config's durable store directly (`--config`), or through a
|
|
169
|
+
running gateway's admin API (`--url`, required for a memory store). Every command prints JSON; `--help` lists
|
|
170
|
+
the flags.
|
|
171
|
+
|
|
172
|
+
### Call it from any OpenAI client
|
|
173
|
+
|
|
174
|
+
LangChain (JS):
|
|
175
|
+
|
|
176
|
+
```ts
|
|
177
|
+
import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"
|
|
178
|
+
|
|
179
|
+
const llm = new ChatOpenAI({
|
|
180
|
+
model: "lesson", // a profile, or a pool model such as "gpt-4o-mini"
|
|
181
|
+
apiKey: process.env.STARCI_AI_KEY, // sk-starci-...
|
|
182
|
+
configuration: { baseURL: "http://127.0.0.1:3002/v1" },
|
|
183
|
+
})
|
|
184
|
+
const reply = await llm.invoke("Explain closures in one paragraph.")
|
|
185
|
+
for await (const chunk of await llm.stream("Count to five.")) process.stdout.write(String(chunk.content))
|
|
186
|
+
|
|
187
|
+
const embeddings = new OpenAIEmbeddings({ model: "text-embedding-3-small", apiKey: process.env.STARCI_AI_KEY, configuration: { baseURL: "http://127.0.0.1:3002/v1" } })
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
LangChain (Python): `ChatOpenAI(model="lesson", api_key=..., base_url="http://127.0.0.1:3002/v1")`.
|
|
191
|
+
|
|
192
|
+
The OpenAI SDK, or any compatible client:
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
import OpenAI from "openai"
|
|
196
|
+
|
|
197
|
+
const client = new OpenAI({ apiKey: process.env.STARCI_AI_KEY, baseURL: "http://127.0.0.1:3002/v1" })
|
|
198
|
+
const completion = await client.chat.completions.create({ model: "lesson", messages: [{ role: "user", content: "hi" }] })
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
```sh
|
|
202
|
+
curl http://127.0.0.1:3002/v1/chat/completions -H "Authorization: Bearer $STARCI_AI_KEY" \
|
|
203
|
+
-H "content-type: application/json" -d '{"model":"lesson","stream":true,"messages":[{"role":"user","content":"hi"}]}'
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### Deploy
|
|
207
|
+
|
|
208
|
+
One container or process: `node node_modules/@starci/ai-router/dist/gateway/run.js serve --config
|
|
209
|
+
/etc/ai-router/pool.yaml` as its command, `host: 0.0.0.0` in the config, the port published,
|
|
210
|
+
a volume for the SQLite file, and SIGTERM for a graceful stop. The environment holds only what the config names: the
|
|
211
|
+
upstream key variables (`OPENAI_KEY_A`, `OPENROUTER_KEY`, ... or mounted files), and the admin token variable
|
|
212
|
+
(`AI_ROUTER_ADMIN_TOKEN` by convention). The CLI's `--url` mode reads the admin token from `AI_ROUTER_ADMIN_TOKEN`
|
|
213
|
+
unless `--admin-token-env` names another variable. Breakers and headroom are per process unless the pool's `store` is
|
|
214
|
+
shared; run one gateway per pool, or several behind a load balancer sharing the SQLite file on one host.
|
|
215
|
+
|
|
216
|
+
### Plain Node and in-process hosting
|
|
217
|
+
|
|
218
|
+
```ts
|
|
219
|
+
import { createGateway, createRouter, openSqliteKeyStore } from "@starci/ai-router/gateway"
|
|
220
|
+
|
|
221
|
+
const router = createRouter(pool) // the same engine AiRouterService wraps: chat, stream, embeddings, health, stats, on, reset, catalog
|
|
222
|
+
const gateway = createGateway(router, { keys: await openSqliteKeyStore("keys.sqlite"), adminToken: () => process.env.AI_ROUTER_ADMIN_TOKEN })
|
|
223
|
+
await gateway.listen(8080, "0.0.0.0") // or mount gateway.handle on an existing node:http, Express or Nest adapter
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
In a Nest app, `AiRouterGatewayModule` fronts a registered router (the default one, or `name` for a named one) and
|
|
227
|
+
listens on its own port when `listen` is given, closing at shutdown; the gateway is injected with
|
|
228
|
+
`@InjectAiRouterGateway(name?)` (token `AI_ROUTER_GATEWAY`):
|
|
229
|
+
|
|
230
|
+
```ts
|
|
231
|
+
@Module({
|
|
232
|
+
imports: [
|
|
233
|
+
AiRouterModule.register({ isGlobal: true, ...pool }),
|
|
234
|
+
AiRouterGatewayModule.registerAsync({
|
|
235
|
+
useFactory: async () => ({ keys: await openSqliteKeyStore("/data/keys.sqlite"), adminToken: () => process.env.AI_ROUTER_ADMIN_TOKEN, listen: true }),
|
|
236
|
+
}),
|
|
237
|
+
],
|
|
238
|
+
})
|
|
239
|
+
export class AppModule {}
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
#### The port convention
|
|
243
|
+
|
|
244
|
+
The gateway's port is derived from the app's, so nobody configures it by hand. The StarCi port convention is
|
|
245
|
+
`basePort + portOffset`: the front end is base+0 (3000), the API is base+1 (3001), and the gateway is the API's port
|
|
246
|
+
plus an offset, 1 by default (3002).
|
|
247
|
+
|
|
248
|
+
| `listen` | gateway listens on |
|
|
249
|
+
|---|---|
|
|
250
|
+
| `true` | the app's port + 1, on `127.0.0.1` |
|
|
251
|
+
| `{ offset?: n, host?: h }` | the app's port + `n` (default 1, at least 1), on `h` (default `127.0.0.1`) |
|
|
252
|
+
| `{ port: p, host?: h }` | `p`, explicitly; the app's port is not read |
|
|
253
|
+
| absent | nothing; mount `gateway.handle` yourself |
|
|
254
|
+
|
|
255
|
+
The app's port is the port its HTTP server is actually bound to. Nest bootstraps before `app.listen` binds, so the
|
|
256
|
+
module waits for the server's `listening` event (or reads the address at once when it is already bound) and then
|
|
257
|
+
starts the gateway; a failure to bind the gateway's port (`EADDRINUSE`) goes to the `onError` option, not out of
|
|
258
|
+
`app.listen`. An application with no HTTP server (a standalone context) reads `PORT` instead; with neither, bootstrap
|
|
259
|
+
fails with an `AiRouterError` saying the port is unknown and `listen.port` is the way out. The host is local to the app
|
|
260
|
+
(`127.0.0.1`) and is `0.0.0.0` only when you set it. With `serve`, `gateway.appPort` plus `gateway.portOffset`
|
|
261
|
+
(default 1) does the same; it cannot be combined with `gateway.port`.
|
|
262
|
+
|
|
263
|
+
A client of an API on 3001 uses `http://127.0.0.1:3002/v1` as its OpenAI base URL: LangChain (below), OpenClaw, or any
|
|
264
|
+
OpenAI-compatible agent, with a virtual `sk-starci-...` key as its API key.
|
|
265
|
+
|
|
266
|
+
## Use it in a Nest app
|
|
30
267
|
|
|
31
268
|
The module follows the Nest configurable-module shape: `AiRouterModule.register(options)` or
|
|
32
269
|
`AiRouterModule.registerAsync(...)`, registered once at the app root. The options are the pool
|
|
@@ -138,6 +375,7 @@ the options the same way (`AiRouterModuleOptions<"strict">`) to keep the two in
|
|
|
138
375
|
| `stats()` | this process's counters: requests, attempts, failures by class, tokens and cost per lane |
|
|
139
376
|
| `on(event, handler)` | subscribes to `attempt`, `request`, `breaker` or `key-disabled` until shutdown; returns the unsubscribe function |
|
|
140
377
|
| `reset(target?)` | closes breakers, re-enables keys and clears headroom for everything, one provider or one key |
|
|
378
|
+
| `catalog()` | the pool's profiles (resolved model and fallbacks) and models (ids, aliases, providers), with no key |
|
|
141
379
|
|
|
142
380
|
`options` is `{ profile?, policy?, signal?, requestId?, headers? }`. `requestId` (generated when absent) is sent as
|
|
143
381
|
`x-request-id` and `idempotency-key` on every attempt of the call, so a provider that deduplicates sees one request.
|
|
@@ -371,12 +609,16 @@ Everything is deterministic except the latencies you script.
|
|
|
371
609
|
- Prompts and responses are not in events unless you set `telemetry.includeContent`.
|
|
372
610
|
- `headers` in a provider or a call cannot override `authorization`.
|
|
373
611
|
- The fake provider records the key values your tests send in `fake.calls`; it is a test double, use test values.
|
|
612
|
+
- The gateway never forwards a client's `Authorization` upstream, never answers with `routing` (the key alias), and keeps
|
|
613
|
+
virtual keys as SHA-256 hashes; the admin token is compared in constant time.
|
|
374
614
|
|
|
375
615
|
## FAQ
|
|
376
616
|
|
|
377
|
-
**Is this a proxy server?**
|
|
617
|
+
**Is this a proxy server?** It can be: `run.js serve` is an OpenAI-compatible gateway with virtual keys. Inside a Nest
|
|
618
|
+
app it is also a module you call directly, and `AiRouterGatewayModule` serves the gateway from that process.
|
|
378
619
|
|
|
379
|
-
**Can I use it outside Nest?**
|
|
620
|
+
**Can I use it outside Nest?** Yes: through the gateway over HTTP from any language, or in Node with `createRouter` and
|
|
621
|
+
`createGateway` from `@starci/ai-router/gateway`, which loads no Nest code.
|
|
380
622
|
|
|
381
623
|
**Does it work with providers that are not OpenAI-compatible?** Only through a compatible gateway; the wire format is
|
|
382
624
|
the OpenAI chat-completions and embeddings API.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { AddressInfo } from "node:net";
|
|
2
|
+
/** The offset from the app's port the gateway listens at when none is given: the API on 3001 serves its gateway on 3002. */
|
|
3
|
+
export declare const DEFAULT_GATEWAY_PORT_OFFSET = 1;
|
|
4
|
+
/** Where the gateway listens: an explicit port, or a port derived from the app's own (`true`: offset 1 on 127.0.0.1). */
|
|
5
|
+
export type AiRouterGatewayListen = true | {
|
|
6
|
+
readonly port: number;
|
|
7
|
+
readonly host?: string;
|
|
8
|
+
readonly offset?: never;
|
|
9
|
+
} | {
|
|
10
|
+
readonly offset?: number;
|
|
11
|
+
readonly host?: string;
|
|
12
|
+
readonly port?: never;
|
|
13
|
+
};
|
|
14
|
+
/** The HTTP server of the Nest application, as far as the gateway needs it. */
|
|
15
|
+
export interface AppServer {
|
|
16
|
+
readonly listening: boolean;
|
|
17
|
+
address(): AddressInfo | string | null;
|
|
18
|
+
once(event: "listening", listener: () => void): unknown;
|
|
19
|
+
off(event: "listening", listener: () => void): unknown;
|
|
20
|
+
}
|
|
21
|
+
/** The port a listen setting means for an app on `appPort`, or the explicit port; refuses an unusable one. */
|
|
22
|
+
export declare function gatewayPort(listen: AiRouterGatewayListen, appPort: number | undefined): number;
|
|
23
|
+
/** The host of a listen setting: `undefined` (the gateway's 127.0.0.1) unless given. */
|
|
24
|
+
export declare function gatewayHost(listen: AiRouterGatewayListen): string | undefined;
|
|
25
|
+
/**
|
|
26
|
+
* Calls `start(port)` once the app's port is known: at once when the server is bound, when it emits `listening` when
|
|
27
|
+
* it is not yet (Nest bootstraps before `app.listen` binds), or from `PORT` when the app has no HTTP server. Returns
|
|
28
|
+
* a function that cancels a wait still pending. It throws when no port can be known.
|
|
29
|
+
*/
|
|
30
|
+
export declare function whenAppPort(server: AppServer | undefined, env: NodeJS.ProcessEnv, start: (port: number) => void): () => void;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { AiRouterError } from "./errors.js";
|
|
2
|
+
/** The offset from the app's port the gateway listens at when none is given: the API on 3001 serves its gateway on 3002. */
|
|
3
|
+
export const DEFAULT_GATEWAY_PORT_OFFSET = 1;
|
|
4
|
+
function failure(message) {
|
|
5
|
+
return new AiRouterError({ errorClass: "config", message });
|
|
6
|
+
}
|
|
7
|
+
/** The port a listen setting means for an app on `appPort`, or the explicit port; refuses an unusable one. */
|
|
8
|
+
export function gatewayPort(listen, appPort) {
|
|
9
|
+
if (listen !== true && listen.port !== undefined)
|
|
10
|
+
return listen.port;
|
|
11
|
+
const offset = listen === true ? DEFAULT_GATEWAY_PORT_OFFSET : (listen.offset ?? DEFAULT_GATEWAY_PORT_OFFSET);
|
|
12
|
+
if (!Number.isInteger(offset) || offset < 1)
|
|
13
|
+
throw failure("AiRouterGatewayModule listen.offset must be an integer of 1 or more");
|
|
14
|
+
if (appPort === undefined)
|
|
15
|
+
throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the app has no known port: listen with the application first, or set listen.port");
|
|
16
|
+
const port = appPort + offset;
|
|
17
|
+
if (port > 65535)
|
|
18
|
+
throw failure(`AiRouterGatewayModule listen: app port ${String(appPort)} + offset ${String(offset)} is not a port number`);
|
|
19
|
+
return port;
|
|
20
|
+
}
|
|
21
|
+
/** The host of a listen setting: `undefined` (the gateway's 127.0.0.1) unless given. */
|
|
22
|
+
export function gatewayHost(listen) {
|
|
23
|
+
return listen === true ? undefined : listen.host;
|
|
24
|
+
}
|
|
25
|
+
/** The port the application's HTTP server is bound to, or `undefined` when it is not bound to a TCP port. */
|
|
26
|
+
function boundPort(server) {
|
|
27
|
+
const address = server.address();
|
|
28
|
+
return typeof address === "object" && address !== null ? address.port : undefined;
|
|
29
|
+
}
|
|
30
|
+
/** The app's port from the `PORT` environment variable, when it is a port number. */
|
|
31
|
+
function envPort(env) {
|
|
32
|
+
const value = env.PORT;
|
|
33
|
+
if (value === undefined || !/^\d+$/.test(value))
|
|
34
|
+
return undefined;
|
|
35
|
+
const port = Number(value);
|
|
36
|
+
return port <= 65535 ? port : undefined;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Calls `start(port)` once the app's port is known: at once when the server is bound, when it emits `listening` when
|
|
40
|
+
* it is not yet (Nest bootstraps before `app.listen` binds), or from `PORT` when the app has no HTTP server. Returns
|
|
41
|
+
* a function that cancels a wait still pending. It throws when no port can be known.
|
|
42
|
+
*/
|
|
43
|
+
export function whenAppPort(server, env, start) {
|
|
44
|
+
if (server === undefined) {
|
|
45
|
+
const port = envPort(env);
|
|
46
|
+
if (port === undefined)
|
|
47
|
+
throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the application has no HTTP server and PORT is not set: set listen.port");
|
|
48
|
+
start(port);
|
|
49
|
+
return () => undefined;
|
|
50
|
+
}
|
|
51
|
+
if (server.listening) {
|
|
52
|
+
const port = boundPort(server);
|
|
53
|
+
if (port === undefined)
|
|
54
|
+
throw failure("AiRouterGatewayModule listen derives the gateway port from the app's port, but the application's server is not on a TCP port: set listen.port");
|
|
55
|
+
start(port);
|
|
56
|
+
return () => undefined;
|
|
57
|
+
}
|
|
58
|
+
const onListening = () => {
|
|
59
|
+
const port = boundPort(server);
|
|
60
|
+
if (port !== undefined)
|
|
61
|
+
start(port);
|
|
62
|
+
};
|
|
63
|
+
server.once("listening", onListening);
|
|
64
|
+
return () => {
|
|
65
|
+
server.off("listening", onListening);
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { AiRouterGatewayListen } from "./ai-router-gateway.listen.js";
|
|
2
|
+
import type { GatewayOptions } from "./gateway/gateway.js";
|
|
3
|
+
/** What the gateway module takes: the gateway's options, and where it listens. */
|
|
4
|
+
export interface AiRouterGatewayModuleOptions extends GatewayOptions {
|
|
5
|
+
/**
|
|
6
|
+
* Listens on its own port and stops at shutdown; absent, mount `gateway.handle` yourself. `{ port }` is explicit;
|
|
7
|
+
* `true` or `{ offset?, host? }` is the app's own port plus `offset` (default 1) once the app listens, on `host`
|
|
8
|
+
* (default 127.0.0.1): an API on 3001 serves its gateway on 3002.
|
|
9
|
+
*/
|
|
10
|
+
readonly listen?: AiRouterGatewayListen;
|
|
11
|
+
}
|
|
12
|
+
/** The extras of a registration: its visibility, and the `name` of the `AiRouterModule` registration it fronts (the default router when absent). */
|
|
13
|
+
export interface AiRouterGatewayModuleExtras {
|
|
14
|
+
readonly isGlobal: boolean;
|
|
15
|
+
readonly name: string | undefined;
|
|
16
|
+
}
|
|
17
|
+
export declare const ConfigurableModuleClass: import("@nestjs/common").ConfigurableModuleCls<AiRouterGatewayModuleOptions, "register", "createAiRouterGatewayOptions", AiRouterGatewayModuleExtras>, MODULE_OPTIONS_TOKEN: string | symbol, OPTIONS_TYPE: AiRouterGatewayModuleOptions & Partial<AiRouterGatewayModuleExtras>, ASYNC_OPTIONS_TYPE: import("@nestjs/common").ConfigurableModuleAsyncOptions<AiRouterGatewayModuleOptions, "createAiRouterGatewayOptions"> & Partial<AiRouterGatewayModuleExtras>;
|
|
18
|
+
/** What `AiRouterGatewayModule.register` takes: the options, plus `isGlobal` and `name`. */
|
|
19
|
+
export type AiRouterGatewayRegisterOptions = typeof OPTIONS_TYPE;
|
|
20
|
+
/** What `AiRouterGatewayModule.registerAsync` takes: `imports`, `inject` and one of `useFactory`, `useClass`, `useExisting`, plus `isGlobal` and `name`. */
|
|
21
|
+
export type AiRouterGatewayRegisterAsyncOptions = typeof ASYNC_OPTIONS_TYPE;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { ConfigurableModuleBuilder } from "@nestjs/common";
|
|
2
|
+
export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = new ConfigurableModuleBuilder({ moduleName: "AiRouterGateway" })
|
|
3
|
+
.setClassMethodName("register")
|
|
4
|
+
.setFactoryMethodName("createAiRouterGatewayOptions")
|
|
5
|
+
.setExtras({ isGlobal: false, name: undefined }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
|
|
6
|
+
.build();
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { DynamicModule } from "@nestjs/common";
|
|
2
|
+
import { ConfigurableModuleClass } from "./ai-router-gateway.module-definition.js";
|
|
3
|
+
import type { AiRouterGatewayRegisterAsyncOptions, AiRouterGatewayRegisterOptions } from "./ai-router-gateway.module-definition.js";
|
|
4
|
+
/** The injection token of the default router's gateway. */
|
|
5
|
+
export declare const AI_ROUTER_GATEWAY = "AI_ROUTER_GATEWAY";
|
|
6
|
+
/** The token of a gateway: `AI_ROUTER_GATEWAY` over the default router, `AI_ROUTER_GATEWAY:<name>` over a named one. */
|
|
7
|
+
export declare function getAiRouterGatewayToken(name?: string): string;
|
|
8
|
+
/** Injects the gateway over the default router, or over the router registered under `name`. */
|
|
9
|
+
export declare function InjectAiRouterGateway(name?: string): PropertyDecorator & ParameterDecorator;
|
|
10
|
+
/**
|
|
11
|
+
* The OpenAI-compatible gateway hosted in a Nest application, over the router an `AiRouterModule` registration
|
|
12
|
+
* provides: the router must be visible here (an `isGlobal` registration, or its module in `imports` of `registerAsync`).
|
|
13
|
+
*/
|
|
14
|
+
export declare class AiRouterGatewayModule extends ConfigurableModuleClass {
|
|
15
|
+
/** Registers a gateway from options known now. */
|
|
16
|
+
static register(options: AiRouterGatewayRegisterOptions): DynamicModule;
|
|
17
|
+
/** Registers a gateway from options a factory, class or existing provider supplies. */
|
|
18
|
+
static registerAsync(options: AiRouterGatewayRegisterAsyncOptions): DynamicModule;
|
|
19
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Inject, Module } from "@nestjs/common";
|
|
2
|
+
import { HttpAdapterHost } from "@nestjs/core";
|
|
3
|
+
import { gatewayHost, gatewayPort, whenAppPort } from "./ai-router-gateway.listen.js";
|
|
4
|
+
import { getAiRouterToken } from "./ai-router.tokens.js";
|
|
5
|
+
import { createGateway } from "./gateway/gateway.js";
|
|
6
|
+
import { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN } from "./ai-router-gateway.module-definition.js";
|
|
7
|
+
/** The injection token of the default router's gateway. */
|
|
8
|
+
export const AI_ROUTER_GATEWAY = "AI_ROUTER_GATEWAY";
|
|
9
|
+
/** The token of a gateway: `AI_ROUTER_GATEWAY` over the default router, `AI_ROUTER_GATEWAY:<name>` over a named one. */
|
|
10
|
+
export function getAiRouterGatewayToken(name) {
|
|
11
|
+
return name === undefined ? AI_ROUTER_GATEWAY : `${AI_ROUTER_GATEWAY}:${name}`;
|
|
12
|
+
}
|
|
13
|
+
/** Injects the gateway over the default router, or over the router registered under `name`. */
|
|
14
|
+
export function InjectAiRouterGateway(name) {
|
|
15
|
+
return Inject(getAiRouterGatewayToken(name));
|
|
16
|
+
}
|
|
17
|
+
/** Adds the gateway over the named router, and the hooks that listen at bootstrap and close at shutdown. */
|
|
18
|
+
function withGateway(base, name) {
|
|
19
|
+
const token = getAiRouterGatewayToken(name);
|
|
20
|
+
const providers = [
|
|
21
|
+
{
|
|
22
|
+
provide: token,
|
|
23
|
+
inject: [getAiRouterToken(name), MODULE_OPTIONS_TOKEN],
|
|
24
|
+
useFactory: (router, options) => createGateway(router, options),
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
// Nest calls the lifecycle hooks of every provider instance that has them.
|
|
28
|
+
provide: `${token}:lifecycle`,
|
|
29
|
+
inject: [token, MODULE_OPTIONS_TOKEN, { token: HttpAdapterHost, optional: true }],
|
|
30
|
+
useFactory: (gateway, options, host) => {
|
|
31
|
+
let cancel = () => undefined;
|
|
32
|
+
return {
|
|
33
|
+
async onApplicationBootstrap() {
|
|
34
|
+
const listen = options.listen;
|
|
35
|
+
if (listen === undefined)
|
|
36
|
+
return;
|
|
37
|
+
if (listen !== true && listen.port !== undefined) {
|
|
38
|
+
await gateway.listen(listen.port, listen.host);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
// The adapter is unset until an application is created over the module (a standalone context has none).
|
|
42
|
+
const adapter = host?.httpAdapter;
|
|
43
|
+
const server = adapter?.getHttpServer();
|
|
44
|
+
// Bootstrap runs before `app.listen` binds the app, so a port not yet bound is waited for, not awaited.
|
|
45
|
+
cancel = whenAppPort(server, process.env, (appPort) => {
|
|
46
|
+
gateway.listen(gatewayPort(listen, appPort), gatewayHost(listen)).catch((error) => {
|
|
47
|
+
options.onError?.(error);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
},
|
|
51
|
+
onApplicationShutdown() {
|
|
52
|
+
cancel();
|
|
53
|
+
return gateway.close();
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
];
|
|
59
|
+
return { ...base, providers: [...(base.providers ?? []), ...providers], exports: [...(base.exports ?? []), token] };
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The OpenAI-compatible gateway hosted in a Nest application, over the router an `AiRouterModule` registration
|
|
63
|
+
* provides: the router must be visible here (an `isGlobal` registration, or its module in `imports` of `registerAsync`).
|
|
64
|
+
*/
|
|
65
|
+
export class AiRouterGatewayModule extends ConfigurableModuleClass {
|
|
66
|
+
/** Registers a gateway from options known now. */
|
|
67
|
+
static register(options) {
|
|
68
|
+
return withGateway(super.register(options), options.name);
|
|
69
|
+
}
|
|
70
|
+
/** Registers a gateway from options a factory, class or existing provider supplies. */
|
|
71
|
+
static registerAsync(options) {
|
|
72
|
+
return withGateway(super.registerAsync(options), options.name);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
Module({})(AiRouterGatewayModule);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { OnApplicationShutdown } from "@nestjs/common";
|
|
2
|
-
import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterEvents, StatsSnapshot } from "./types.js";
|
|
2
|
+
import type { AiRouter, CallOptions, ChatCompletion, ChatCompletionRequest, ChatStream, EmbeddingsRequest, EmbeddingsResponse, HealthSnapshot, ResetTarget, RouterCatalog, RouterEvents, StatsSnapshot } from "./types.js";
|
|
3
3
|
import type { AiRouterHandlers } from "./ai-router.options.js";
|
|
4
4
|
/**
|
|
5
5
|
* The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
|
|
@@ -24,6 +24,8 @@ export declare class AiRouterService<Profiles extends string = string> implement
|
|
|
24
24
|
on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
|
|
25
25
|
/** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
|
|
26
26
|
reset(target?: ResetTarget): Promise<void>;
|
|
27
|
+
/** Lists the pool's profiles and models by name. */
|
|
28
|
+
catalog(): RouterCatalog;
|
|
27
29
|
/** Aborts every call still in flight through this service and releases its handlers. */
|
|
28
30
|
onApplicationShutdown(): void;
|
|
29
31
|
}
|
|
@@ -56,6 +56,10 @@ export class AiRouterService {
|
|
|
56
56
|
reset(target) {
|
|
57
57
|
return this.router.reset(target);
|
|
58
58
|
}
|
|
59
|
+
/** Lists the pool's profiles and models by name. */
|
|
60
|
+
catalog() {
|
|
61
|
+
return this.router.catalog();
|
|
62
|
+
}
|
|
59
63
|
/** Aborts every call still in flight through this service and releases its handlers. */
|
|
60
64
|
onApplicationShutdown() {
|
|
61
65
|
this.#shutdown.abort();
|
package/dist/config.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Capability, CallOptions, ModelConfig, PoolConfig } from "./types.js";
|
|
1
|
+
import type { Capability, CallOptions, ModelConfig, PoolConfig, RouterCatalog } from "./types.js";
|
|
2
2
|
/** The capabilities of a model that declares none. */
|
|
3
3
|
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
4
|
/** A breaker's thresholds with every default applied. */
|
|
@@ -44,3 +44,5 @@ export declare function validateConfig(config: PoolConfig): void;
|
|
|
44
44
|
export declare function resolveSettings(config: PoolConfig): Settings;
|
|
45
45
|
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
46
46
|
export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
|
|
47
|
+
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
48
|
+
export declare function buildCatalog(config: PoolConfig): RouterCatalog;
|
package/dist/config.js
CHANGED
|
@@ -146,3 +146,44 @@ export function resolvePolicy(config, options, requestModel) {
|
|
|
146
146
|
require: merged.require ?? [],
|
|
147
147
|
};
|
|
148
148
|
}
|
|
149
|
+
function fallbacksOf(policy) {
|
|
150
|
+
return policy.allowFallback === false ? [] : (policy.fallbacks ?? []);
|
|
151
|
+
}
|
|
152
|
+
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
153
|
+
export function buildCatalog(config) {
|
|
154
|
+
const models = new Map();
|
|
155
|
+
for (const provider of config.providers) {
|
|
156
|
+
for (const model of provider.models) {
|
|
157
|
+
let entry = models.get(model.id);
|
|
158
|
+
if (entry === undefined) {
|
|
159
|
+
entry = { aliases: new Set(), providers: [], capabilities: new Set() };
|
|
160
|
+
models.set(model.id, entry);
|
|
161
|
+
}
|
|
162
|
+
for (const alias of model.aliases ?? [])
|
|
163
|
+
entry.aliases.add(alias);
|
|
164
|
+
entry.providers.push(provider.id);
|
|
165
|
+
for (const capability of model.capabilities ?? DEFAULT_CAPABILITIES)
|
|
166
|
+
entry.capabilities.add(capability);
|
|
167
|
+
if (entry.contextWindow === undefined && model.contextWindow !== undefined)
|
|
168
|
+
entry.contextWindow = model.contextWindow;
|
|
169
|
+
if (entry.cost === undefined && model.cost !== undefined)
|
|
170
|
+
entry.cost = model.cost;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const defaults = config.defaults ?? {};
|
|
174
|
+
return {
|
|
175
|
+
defaults: { ...(defaults.model === undefined ? {} : { model: defaults.model }), fallbacks: fallbacksOf(defaults) },
|
|
176
|
+
profiles: Object.entries(config.profiles ?? {}).map(([name, policy]) => {
|
|
177
|
+
const merged = { ...defaults, ...policy };
|
|
178
|
+
return { name, ...(merged.model === undefined ? {} : { model: merged.model }), fallbacks: fallbacksOf(merged) };
|
|
179
|
+
}),
|
|
180
|
+
models: [...models].map(([id, entry]) => ({
|
|
181
|
+
id,
|
|
182
|
+
aliases: [...entry.aliases],
|
|
183
|
+
providers: entry.providers,
|
|
184
|
+
capabilities: [...entry.capabilities],
|
|
185
|
+
...(entry.contextWindow === undefined ? {} : { contextWindow: entry.contextWindow }),
|
|
186
|
+
...(entry.cost === undefined ? {} : { cost: entry.cost }),
|
|
187
|
+
})),
|
|
188
|
+
};
|
|
189
|
+
}
|