@starci/ai-router 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +256 -14
- package/dist/ai-router-gateway.listen.d.ts +30 -0
- package/dist/ai-router-gateway.listen.js +67 -0
- package/dist/ai-router-gateway.module-definition.d.ts +21 -0
- package/dist/ai-router-gateway.module-definition.js +6 -0
- package/dist/ai-router-gateway.module.d.ts +19 -0
- package/dist/ai-router-gateway.module.js +75 -0
- package/dist/ai-router-testing.module.d.ts +2 -2
- package/dist/ai-router.module-definition.d.ts +3 -3
- package/dist/ai-router.module.d.ts +2 -2
- package/dist/ai-router.options.d.ts +1 -1
- package/dist/ai-router.service.d.ts +4 -2
- package/dist/ai-router.service.js +4 -0
- package/dist/ai-router.tokens.d.ts +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +41 -0
- package/dist/errors.d.ts +1 -1
- package/dist/events.d.ts +1 -1
- package/dist/fake.d.ts +1 -1
- package/dist/gateway/cli.d.ts +13 -0
- package/dist/gateway/cli.js +241 -0
- package/dist/gateway/config-file.d.ts +37 -0
- package/dist/gateway/config-file.js +238 -0
- package/dist/gateway/gateway.d.ts +34 -0
- package/dist/gateway/gateway.js +527 -0
- package/dist/gateway/index.d.ts +14 -0
- package/dist/gateway/index.js +7 -0
- package/dist/gateway/keys.d.ts +166 -0
- package/dist/gateway/keys.js +268 -0
- package/dist/gateway/run.d.ts +1 -0
- package/dist/gateway/run.js +14 -0
- package/dist/gateway/serve.d.ts +25 -0
- package/dist/gateway/serve.js +38 -0
- package/dist/gateway/sqlite-key-store.d.ts +8 -0
- package/dist/gateway/sqlite-key-store.js +167 -0
- package/dist/index.d.ts +9 -9
- package/dist/index.js +2 -1
- package/dist/pool.d.ts +5 -5
- package/dist/router.d.ts +1 -1
- package/dist/router.js +3 -1
- package/dist/store.d.ts +1 -1
- package/dist/testing.d.ts +4 -4
- package/dist/types.d.ts +29 -1
- package/package.json +30 -4
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AiRouterService } .js";
|
|
1
|
+
import { AiRouterService } from "./ai-router.service.js";
|
|
2
2
|
/** The injection token of the default instance's raw router (`AiRouter`). */
|
|
3
3
|
export declare const AI_ROUTER = "AI_ROUTER";
|
|
4
4
|
/** The token of a raw router: `AI_ROUTER` for the default instance, `AI_ROUTER:<name>` for a named one. */
|
package/dist/config.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Capability, CallOptions, ModelConfig, PoolConfig } .js";
|
|
1
|
+
import type { Capability, CallOptions, ModelConfig, PoolConfig, RouterCatalog } from "./types.js";
|
|
2
2
|
/** The capabilities of a model that declares none. */
|
|
3
3
|
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
4
|
/** A breaker's thresholds with every default applied. */
|
|
@@ -44,3 +44,5 @@ export declare function validateConfig(config: PoolConfig): void;
|
|
|
44
44
|
export declare function resolveSettings(config: PoolConfig): Settings;
|
|
45
45
|
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
46
46
|
export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
|
|
47
|
+
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
48
|
+
export declare function buildCatalog(config: PoolConfig): RouterCatalog;
|
package/dist/config.js
CHANGED
|
@@ -146,3 +146,44 @@ export function resolvePolicy(config, options, requestModel) {
|
|
|
146
146
|
require: merged.require ?? [],
|
|
147
147
|
};
|
|
148
148
|
}
|
|
149
|
+
function fallbacksOf(policy) {
|
|
150
|
+
return policy.allowFallback === false ? [] : (policy.fallbacks ?? []);
|
|
151
|
+
}
|
|
152
|
+
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
153
|
+
export function buildCatalog(config) {
|
|
154
|
+
const models = new Map();
|
|
155
|
+
for (const provider of config.providers) {
|
|
156
|
+
for (const model of provider.models) {
|
|
157
|
+
let entry = models.get(model.id);
|
|
158
|
+
if (entry === undefined) {
|
|
159
|
+
entry = { aliases: new Set(), providers: [], capabilities: new Set() };
|
|
160
|
+
models.set(model.id, entry);
|
|
161
|
+
}
|
|
162
|
+
for (const alias of model.aliases ?? [])
|
|
163
|
+
entry.aliases.add(alias);
|
|
164
|
+
entry.providers.push(provider.id);
|
|
165
|
+
for (const capability of model.capabilities ?? DEFAULT_CAPABILITIES)
|
|
166
|
+
entry.capabilities.add(capability);
|
|
167
|
+
if (entry.contextWindow === undefined && model.contextWindow !== undefined)
|
|
168
|
+
entry.contextWindow = model.contextWindow;
|
|
169
|
+
if (entry.cost === undefined && model.cost !== undefined)
|
|
170
|
+
entry.cost = model.cost;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const defaults = config.defaults ?? {};
|
|
174
|
+
return {
|
|
175
|
+
defaults: { ...(defaults.model === undefined ? {} : { model: defaults.model }), fallbacks: fallbacksOf(defaults) },
|
|
176
|
+
profiles: Object.entries(config.profiles ?? {}).map(([name, policy]) => {
|
|
177
|
+
const merged = { ...defaults, ...policy };
|
|
178
|
+
return { name, ...(merged.model === undefined ? {} : { model: merged.model }), fallbacks: fallbacksOf(merged) };
|
|
179
|
+
}),
|
|
180
|
+
models: [...models].map(([id, entry]) => ({
|
|
181
|
+
id,
|
|
182
|
+
aliases: [...entry.aliases],
|
|
183
|
+
providers: entry.providers,
|
|
184
|
+
capabilities: [...entry.capabilities],
|
|
185
|
+
...(entry.contextWindow === undefined ? {} : { contextWindow: entry.contextWindow }),
|
|
186
|
+
...(entry.cost === undefined ? {} : { cost: entry.cost }),
|
|
187
|
+
})),
|
|
188
|
+
};
|
|
189
|
+
}
|
package/dist/errors.d.ts
CHANGED
package/dist/events.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RouterEvents } .js";
|
|
1
|
+
import type { RouterEvents } from "./types.js";
|
|
2
2
|
type Handler<E extends keyof RouterEvents> = (payload: RouterEvents[E]) => void;
|
|
3
3
|
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
4
4
|
export declare class Emitter {
|
package/dist/fake.d.ts
CHANGED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { FetchLike } from "../types.js";
|
|
2
|
+
/** What the CLI reads and writes; the bin passes the process's, a spec passes its own. */
|
|
3
|
+
export interface CliIo {
|
|
4
|
+
readonly out: (text: string) => void;
|
|
5
|
+
readonly err: (text: string) => void;
|
|
6
|
+
readonly env: NodeJS.ProcessEnv;
|
|
7
|
+
/** Stops `serve`; the bin aborts it on SIGINT and SIGTERM. */
|
|
8
|
+
readonly signal?: AbortSignal;
|
|
9
|
+
/** The `fetch` of `keys --url` and of the served router (a spec's fake). Default the global `fetch`. */
|
|
10
|
+
readonly fetch?: FetchLike;
|
|
11
|
+
}
|
|
12
|
+
/** Runs one CLI command and resolves to its exit code: 0 done, 1 failed, 2 misused. */
|
|
13
|
+
export declare function main(argv: readonly string[], io: CliIo): Promise<number>;
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { createRouter } from "../router.js";
|
|
3
|
+
import { loadGatewayConfig } from "./config-file.js";
|
|
4
|
+
import { KeyAdmin } from "./keys.js";
|
|
5
|
+
import { openKeyStore, serveGateway } from "./serve.js";
|
|
6
|
+
/** A usage mistake: the CLI prints the help and exits 2. */
|
|
7
|
+
class UsageError extends Error {
|
|
8
|
+
}
|
|
9
|
+
const HELP = `ai-router: an OpenAI-compatible gateway over a pool of providers, keys and models.
|
|
10
|
+
Run as: node node_modules/@starci/ai-router/dist/gateway/run.js <command>, or in process with main() from @starci/ai-router/cli.
|
|
11
|
+
|
|
12
|
+
serve --config <pool.yaml|json> [--port N] [--host H]
|
|
13
|
+
keys create --label <label> [--owner O] [--profiles a,b] [--models m1,m2] [--no-fallback]
|
|
14
|
+
[--rpm N] [--tokens-per-day N] [--cost-per-day X] [--cost-per-month X] [--disabled]
|
|
15
|
+
keys list
|
|
16
|
+
keys get <id>
|
|
17
|
+
keys update <id> [--owner O] [--profiles ..] [--models ..] [--fallback|--no-fallback]
|
|
18
|
+
[--rpm N] [--tokens-per-day N] [--cost-per-day X] [--cost-per-month X] [--enable|--disable]
|
|
19
|
+
keys revoke <id>
|
|
20
|
+
keys rotate <id>
|
|
21
|
+
keys usage <id> [--from YYYY-MM-DD] [--to YYYY-MM-DD]
|
|
22
|
+
|
|
23
|
+
keys commands run on the config's durable key store (--config <file>), or on a running gateway's admin API
|
|
24
|
+
(--url <gateway url>, the admin token read from the variable --admin-token-env names, AI_ROUTER_ADMIN_TOKEN by default).
|
|
25
|
+
Every command prints JSON. A created or rotated key's secret is printed once and never stored.`;
|
|
26
|
+
const OPTIONS = {
|
|
27
|
+
config: { type: "string" },
|
|
28
|
+
port: { type: "string" },
|
|
29
|
+
host: { type: "string" },
|
|
30
|
+
url: { type: "string" },
|
|
31
|
+
"admin-token-env": { type: "string" },
|
|
32
|
+
label: { type: "string" },
|
|
33
|
+
owner: { type: "string" },
|
|
34
|
+
profiles: { type: "string" },
|
|
35
|
+
models: { type: "string" },
|
|
36
|
+
fallback: { type: "boolean" },
|
|
37
|
+
"no-fallback": { type: "boolean" },
|
|
38
|
+
rpm: { type: "string" },
|
|
39
|
+
"tokens-per-day": { type: "string" },
|
|
40
|
+
"cost-per-day": { type: "string" },
|
|
41
|
+
"cost-per-month": { type: "string" },
|
|
42
|
+
enable: { type: "boolean" },
|
|
43
|
+
disable: { type: "boolean" },
|
|
44
|
+
disabled: { type: "boolean" },
|
|
45
|
+
from: { type: "string" },
|
|
46
|
+
to: { type: "string" },
|
|
47
|
+
help: { type: "boolean", short: "h" },
|
|
48
|
+
};
|
|
49
|
+
function list(value) {
|
|
50
|
+
return value?.split(",").map((name) => name.trim()).filter((name) => name !== "");
|
|
51
|
+
}
|
|
52
|
+
function amount(value, flag) {
|
|
53
|
+
if (value === undefined)
|
|
54
|
+
return undefined;
|
|
55
|
+
const parsed = Number(value);
|
|
56
|
+
if (value.trim() === "" || !Number.isFinite(parsed))
|
|
57
|
+
throw new UsageError(`--${flag} must be a number`);
|
|
58
|
+
return parsed;
|
|
59
|
+
}
|
|
60
|
+
function flag(on, off, name) {
|
|
61
|
+
if (on === true && off === true)
|
|
62
|
+
throw new UsageError(`--${name} and its opposite cannot both be given`);
|
|
63
|
+
return on === true ? true : off === true ? false : undefined;
|
|
64
|
+
}
|
|
65
|
+
function bindings(values) {
|
|
66
|
+
const quota = {};
|
|
67
|
+
for (const [field, name] of [
|
|
68
|
+
["requestsPerMinute", "rpm"],
|
|
69
|
+
["tokensPerDay", "tokens-per-day"],
|
|
70
|
+
["costPerDay", "cost-per-day"],
|
|
71
|
+
["costPerMonth", "cost-per-month"],
|
|
72
|
+
]) {
|
|
73
|
+
const limit = amount(values[name], name);
|
|
74
|
+
if (limit !== undefined)
|
|
75
|
+
quota[field] = limit;
|
|
76
|
+
}
|
|
77
|
+
const profiles = list(values.profiles);
|
|
78
|
+
const models = list(values.models);
|
|
79
|
+
const allowFallback = flag(values.fallback, values["no-fallback"], "fallback");
|
|
80
|
+
const enabled = flag(values.enable, values.disable ?? values.disabled, "enable");
|
|
81
|
+
return {
|
|
82
|
+
...(values.owner === undefined ? {} : { owner: values.owner }),
|
|
83
|
+
...(profiles === undefined ? {} : { profiles }),
|
|
84
|
+
...(models === undefined ? {} : { models }),
|
|
85
|
+
...(allowFallback === undefined ? {} : { allowFallback }),
|
|
86
|
+
...(Object.keys(quota).length === 0 ? {} : { quota }),
|
|
87
|
+
...(enabled === undefined ? {} : { enabled }),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/** The admin API of a running gateway, spoken over HTTP. */
|
|
91
|
+
function remoteAdmin(url, token, fetchImpl) {
|
|
92
|
+
const base = url.replace(/\/+$/, "");
|
|
93
|
+
async function call(method, path, body) {
|
|
94
|
+
const response = await fetchImpl(`${base}/admin/keys${path}`, {
|
|
95
|
+
method,
|
|
96
|
+
headers: { authorization: `Bearer ${token}`, ...(body === undefined ? {} : { "content-type": "application/json" }) },
|
|
97
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
98
|
+
});
|
|
99
|
+
const json = await response.json();
|
|
100
|
+
if (!response.ok) {
|
|
101
|
+
const error = json.error;
|
|
102
|
+
throw new Error(`the gateway answered ${String(response.status)}: ${error?.message ?? "no message"}`);
|
|
103
|
+
}
|
|
104
|
+
return json;
|
|
105
|
+
}
|
|
106
|
+
return {
|
|
107
|
+
create: (spec) => call("POST", "", spec),
|
|
108
|
+
list: async () => (await call("GET", "")).data,
|
|
109
|
+
get: (id) => call("GET", `/${encodeURIComponent(id)}`),
|
|
110
|
+
revoke: (id) => call("POST", `/${encodeURIComponent(id)}/revoke`),
|
|
111
|
+
rotate: (id) => call("POST", `/${encodeURIComponent(id)}/rotate`),
|
|
112
|
+
update: (id, patch) => call("PATCH", `/${encodeURIComponent(id)}`, patch),
|
|
113
|
+
usage: (id, range) => {
|
|
114
|
+
const query = new URLSearchParams({ ...(range.from === undefined ? {} : { from: range.from }), ...(range.to === undefined ? {} : { to: range.to }) }).toString();
|
|
115
|
+
return call("GET", `/${encodeURIComponent(id)}/usage${query === "" ? "" : `?${query}`}`);
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
async function openAdmin(values, io) {
|
|
120
|
+
if (values.url !== undefined && values.config !== undefined)
|
|
121
|
+
throw new UsageError("give --config or --url, not both");
|
|
122
|
+
if (values.url !== undefined) {
|
|
123
|
+
const variable = values["admin-token-env"] ?? "AI_ROUTER_ADMIN_TOKEN";
|
|
124
|
+
const token = io.env[variable]?.trim();
|
|
125
|
+
if (token === undefined || token === "")
|
|
126
|
+
throw new UsageError(`the admin token variable ${variable} is not set`);
|
|
127
|
+
return { admin: remoteAdmin(values.url, token, io.fetch ?? ((input, init) => fetch(input, init))), close: () => Promise.resolve() };
|
|
128
|
+
}
|
|
129
|
+
if (values.config === undefined)
|
|
130
|
+
throw new UsageError("keys commands need --config <file> or --url <gateway url>");
|
|
131
|
+
const file = await loadGatewayConfig(values.config);
|
|
132
|
+
if (file.gateway.keyStore.type === "memory") {
|
|
133
|
+
throw new UsageError("the config keeps keys in memory, which only the running gateway holds: use --url <gateway url> instead");
|
|
134
|
+
}
|
|
135
|
+
// The router is built for its catalog only, so a key can name only the pool's profiles and models; nothing is sent.
|
|
136
|
+
const catalog = createRouter(file.pool).catalog();
|
|
137
|
+
const store = await openKeyStore(file.gateway.keyStore);
|
|
138
|
+
return {
|
|
139
|
+
admin: new KeyAdmin(store, { catalog }),
|
|
140
|
+
close: async () => {
|
|
141
|
+
await store.close?.();
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
async function keys(command, id, values, io) {
|
|
146
|
+
const needsId = () => {
|
|
147
|
+
if (id === undefined)
|
|
148
|
+
throw new UsageError(`keys ${command ?? ""} needs a key id`);
|
|
149
|
+
return id;
|
|
150
|
+
};
|
|
151
|
+
const known = ["create", "list", "get", "update", "revoke", "rotate", "usage"];
|
|
152
|
+
if (command === undefined || !known.includes(command))
|
|
153
|
+
throw new UsageError(`keys needs one of ${known.join(", ")}`);
|
|
154
|
+
if (command === "create" && values.label === undefined)
|
|
155
|
+
throw new UsageError("keys create needs --label");
|
|
156
|
+
const { admin, close } = await openAdmin(values, io);
|
|
157
|
+
try {
|
|
158
|
+
switch (command) {
|
|
159
|
+
case "create":
|
|
160
|
+
return await admin.create({ label: values.label ?? "", ...bindings(values) });
|
|
161
|
+
case "list":
|
|
162
|
+
return await admin.list();
|
|
163
|
+
case "get":
|
|
164
|
+
return await admin.get(needsId());
|
|
165
|
+
case "update":
|
|
166
|
+
return await admin.update(needsId(), bindings(values));
|
|
167
|
+
case "revoke":
|
|
168
|
+
return await admin.revoke(needsId());
|
|
169
|
+
case "rotate":
|
|
170
|
+
return await admin.rotate(needsId());
|
|
171
|
+
default:
|
|
172
|
+
return await admin.usage(needsId(), { ...(values.from === undefined ? {} : { from: values.from }), ...(values.to === undefined ? {} : { to: values.to }) });
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
await close();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
async function serve(values, io) {
|
|
180
|
+
if (values.config === undefined)
|
|
181
|
+
throw new UsageError("serve needs --config <file>");
|
|
182
|
+
const port = amount(values.port, "port");
|
|
183
|
+
const serving = await serveGateway({
|
|
184
|
+
config: values.config,
|
|
185
|
+
env: io.env,
|
|
186
|
+
...(port === undefined ? {} : { port }),
|
|
187
|
+
...(values.host === undefined ? {} : { host: values.host }),
|
|
188
|
+
...(io.fetch === undefined ? {} : { fetch: io.fetch }),
|
|
189
|
+
onError: (error) => {
|
|
190
|
+
io.err(`ai-router: ${error instanceof Error ? error.message : "an unknown failure"}\n`);
|
|
191
|
+
},
|
|
192
|
+
});
|
|
193
|
+
io.out(`${JSON.stringify({ listening: serving.url })}\n`);
|
|
194
|
+
await new Promise((resolve) => {
|
|
195
|
+
if (io.signal === undefined)
|
|
196
|
+
return;
|
|
197
|
+
if (io.signal.aborted)
|
|
198
|
+
resolve();
|
|
199
|
+
else
|
|
200
|
+
io.signal.addEventListener("abort", () => {
|
|
201
|
+
resolve();
|
|
202
|
+
}, { once: true });
|
|
203
|
+
});
|
|
204
|
+
await serving.close();
|
|
205
|
+
}
|
|
206
|
+
/** Runs one CLI command and resolves to its exit code: 0 done, 1 failed, 2 misused. */
|
|
207
|
+
export async function main(argv, io) {
|
|
208
|
+
let parsed;
|
|
209
|
+
try {
|
|
210
|
+
parsed = parseArgs({ args: [...argv], options: OPTIONS, allowPositionals: true, strict: true });
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
io.err(`ai-router: ${error instanceof Error ? error.message : "bad arguments"}\n\n${HELP}\n`);
|
|
214
|
+
return 2;
|
|
215
|
+
}
|
|
216
|
+
const { values, positionals } = parsed;
|
|
217
|
+
const [group, command, id] = positionals;
|
|
218
|
+
if (values.help === true || group === undefined) {
|
|
219
|
+
io.out(`${HELP}\n`);
|
|
220
|
+
return group === undefined && values.help !== true ? 2 : 0;
|
|
221
|
+
}
|
|
222
|
+
try {
|
|
223
|
+
if (group === "serve") {
|
|
224
|
+
await serve(values, io);
|
|
225
|
+
return 0;
|
|
226
|
+
}
|
|
227
|
+
if (group === "keys") {
|
|
228
|
+
io.out(`${JSON.stringify(await keys(command, id, values, io), null, 2)}\n`);
|
|
229
|
+
return 0;
|
|
230
|
+
}
|
|
231
|
+
throw new UsageError(`unknown command "${group}"`);
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
if (error instanceof UsageError) {
|
|
235
|
+
io.err(`ai-router: ${error.message}\n\n${HELP}\n`);
|
|
236
|
+
return 2;
|
|
237
|
+
}
|
|
238
|
+
io.err(`ai-router: ${error instanceof Error ? error.message : "an unknown failure"}\n`);
|
|
239
|
+
return 1;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { PoolConfig } from "../types.js";
|
|
2
|
+
/** A secret named by reference: an environment variable or a file (a mounted Docker or Kubernetes secret), never its value. */
|
|
3
|
+
export type SecretRef = {
|
|
4
|
+
readonly env: string;
|
|
5
|
+
} | {
|
|
6
|
+
readonly file: string;
|
|
7
|
+
};
|
|
8
|
+
/** Where the gateway keeps virtual keys: in memory (lost on restart) or in a SQLite file. */
|
|
9
|
+
export type KeyStoreConfig = {
|
|
10
|
+
readonly type: "memory";
|
|
11
|
+
} | {
|
|
12
|
+
readonly type: "sqlite";
|
|
13
|
+
readonly path: string;
|
|
14
|
+
};
|
|
15
|
+
/** The `gateway` section of a config file, with every default applied and paths resolved from the file's folder. */
|
|
16
|
+
export interface GatewaySettings {
|
|
17
|
+
readonly port: number;
|
|
18
|
+
readonly host: string;
|
|
19
|
+
readonly maxBodyBytes: number;
|
|
20
|
+
readonly adminToken: SecretRef | null;
|
|
21
|
+
readonly keyStore: KeyStoreConfig;
|
|
22
|
+
}
|
|
23
|
+
/** A loaded config file: the pool the router takes, and the gateway's settings. */
|
|
24
|
+
export interface GatewayFileConfig {
|
|
25
|
+
readonly pool: PoolConfig;
|
|
26
|
+
readonly gateway: GatewaySettings;
|
|
27
|
+
}
|
|
28
|
+
/** Reads a referenced secret now: the variable's value or the file's first line, trimmed; undefined when absent or empty. */
|
|
29
|
+
export declare function readSecret(ref: SecretRef, env?: NodeJS.ProcessEnv): Promise<string | undefined>;
|
|
30
|
+
/**
|
|
31
|
+
* Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
|
|
32
|
+
* keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
|
|
33
|
+
* no restart). A field that would hold a secret's value, or a provider header that would carry one, is refused.
|
|
34
|
+
*/
|
|
35
|
+
export declare function parseGatewayConfig(raw: unknown, base?: string): GatewayFileConfig;
|
|
36
|
+
/** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
|
|
37
|
+
export declare function loadGatewayConfig(path: string): Promise<GatewayFileConfig>;
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, extname, resolve } from "node:path";
|
|
3
|
+
import { parse as parseYaml } from "yaml";
|
|
4
|
+
import { AiRouterError } from "../errors.js";
|
|
5
|
+
const CAPABILITIES = { list: { enum: ["chat", "streaming", "tools", "json", "embeddings", "vision"] } };
|
|
6
|
+
const BREAKER = { fields: { failureThreshold: "number", cooldownMs: "number", maxCooldownMs: "number" } };
|
|
7
|
+
const POLICY = {
|
|
8
|
+
fields: {
|
|
9
|
+
model: "string",
|
|
10
|
+
fallbacks: { list: "string" },
|
|
11
|
+
allowFallback: "boolean",
|
|
12
|
+
maxAttempts: "number",
|
|
13
|
+
timeoutMs: "number",
|
|
14
|
+
deadlineMs: "number",
|
|
15
|
+
budget: { fields: { maxCost: "number" } },
|
|
16
|
+
providers: { list: "string" },
|
|
17
|
+
require: CAPABILITIES,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
const SECRET = { fields: { env: "string", file: "string" } };
|
|
21
|
+
const FILE = {
|
|
22
|
+
fields: {
|
|
23
|
+
gateway: {
|
|
24
|
+
fields: {
|
|
25
|
+
port: "number",
|
|
26
|
+
appPort: "number",
|
|
27
|
+
portOffset: "number",
|
|
28
|
+
host: "string",
|
|
29
|
+
maxBodyBytes: "number",
|
|
30
|
+
adminToken: SECRET,
|
|
31
|
+
keyStore: { fields: { type: { enum: ["memory", "sqlite"] }, path: "string" }, required: ["type"] },
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
providers: {
|
|
35
|
+
list: {
|
|
36
|
+
fields: {
|
|
37
|
+
id: "string",
|
|
38
|
+
baseUrl: "string",
|
|
39
|
+
headers: { map: "string" },
|
|
40
|
+
weight: "number",
|
|
41
|
+
maxConcurrency: "number",
|
|
42
|
+
keys: { list: { fields: { alias: "string", env: "string", file: "string", weight: "number", maxConcurrency: "number" }, required: ["alias"] } },
|
|
43
|
+
models: {
|
|
44
|
+
list: {
|
|
45
|
+
fields: {
|
|
46
|
+
id: "string",
|
|
47
|
+
aliases: { list: "string" },
|
|
48
|
+
contextWindow: "number",
|
|
49
|
+
cost: { fields: { input: "number", output: "number" }, required: ["input", "output"] },
|
|
50
|
+
capabilities: CAPABILITIES,
|
|
51
|
+
weight: "number",
|
|
52
|
+
},
|
|
53
|
+
required: ["id"],
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
required: ["id", "baseUrl", "keys", "models"],
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
defaults: POLICY,
|
|
61
|
+
profiles: { map: POLICY },
|
|
62
|
+
resilience: {
|
|
63
|
+
fields: {
|
|
64
|
+
keyBreaker: BREAKER,
|
|
65
|
+
providerBreaker: BREAKER,
|
|
66
|
+
backoff: { fields: { baseMs: "number", factor: "number", maxMs: "number" } },
|
|
67
|
+
authDisableMs: "number|null",
|
|
68
|
+
rateLimitCooldownMs: "number",
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
selection: { fields: { latencyTolerance: "number", latencySlackMs: "number", ewmaAlpha: "number", lowHeadroom: "number" } },
|
|
72
|
+
telemetry: { fields: { includeContent: "boolean" } },
|
|
73
|
+
storeNamespace: "string",
|
|
74
|
+
},
|
|
75
|
+
required: ["providers"],
|
|
76
|
+
};
|
|
77
|
+
/** Field names that would hold a secret's value; a config file names secrets by reference only. */
|
|
78
|
+
const INLINE_SECRET = /^(?:value|key|secret|token|apikey|api_key|password)$/i;
|
|
79
|
+
const SECRET_HEADER = /authorization|api[-_]?key|token|secret|cookie/i;
|
|
80
|
+
function configError(message) {
|
|
81
|
+
return new AiRouterError({ errorClass: "config", message });
|
|
82
|
+
}
|
|
83
|
+
function isRecord(value) {
|
|
84
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
85
|
+
}
|
|
86
|
+
function check(value, schema, where) {
|
|
87
|
+
if (schema === "string") {
|
|
88
|
+
if (typeof value !== "string")
|
|
89
|
+
throw configError(`${where} must be a string`);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (schema === "number" || schema === "number|null") {
|
|
93
|
+
if (schema === "number|null" && value === null)
|
|
94
|
+
return;
|
|
95
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
96
|
+
throw configError(`${where} must be a number`);
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (schema === "boolean") {
|
|
100
|
+
if (typeof value !== "boolean")
|
|
101
|
+
throw configError(`${where} must be true or false`);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if ("enum" in schema) {
|
|
105
|
+
if (typeof value !== "string" || !schema.enum.includes(value))
|
|
106
|
+
throw configError(`${where} must be one of ${schema.enum.join(", ")}`);
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if ("list" in schema) {
|
|
110
|
+
if (!Array.isArray(value))
|
|
111
|
+
throw configError(`${where} must be a list`);
|
|
112
|
+
value.forEach((item, index) => {
|
|
113
|
+
check(item, schema.list, `${where}[${String(index)}]`);
|
|
114
|
+
});
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (!isRecord(value))
|
|
118
|
+
throw configError(`${where} must be a mapping`);
|
|
119
|
+
if ("map" in schema) {
|
|
120
|
+
for (const [name, item] of Object.entries(value))
|
|
121
|
+
check(item, schema.map, `${where}.${name}`);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
for (const [name, item] of Object.entries(value)) {
|
|
125
|
+
const field = schema.fields[name];
|
|
126
|
+
if (field === undefined) {
|
|
127
|
+
if (INLINE_SECRET.test(name))
|
|
128
|
+
throw configError(`${where}.${name} would hold a secret: name it by reference ({ env: NAME } or { file: path }), never inline`);
|
|
129
|
+
throw configError(`${where}.${name} is not a known field`);
|
|
130
|
+
}
|
|
131
|
+
if (item !== undefined)
|
|
132
|
+
check(item, field, `${where}.${name}`);
|
|
133
|
+
}
|
|
134
|
+
for (const name of schema.required ?? [])
|
|
135
|
+
if (value[name] === undefined)
|
|
136
|
+
throw configError(`${where}.${name} is required`);
|
|
137
|
+
}
|
|
138
|
+
function secretRef(value, where, base) {
|
|
139
|
+
const { env, file } = value;
|
|
140
|
+
if (typeof env === "string" && file === undefined)
|
|
141
|
+
return { env };
|
|
142
|
+
if (typeof file === "string" && env === undefined)
|
|
143
|
+
return { file: resolve(base, file) };
|
|
144
|
+
throw configError(`${where} needs exactly one of env or file`);
|
|
145
|
+
}
|
|
146
|
+
/** Reads a referenced secret now: the variable's value or the file's first line, trimmed; undefined when absent or empty. */
|
|
147
|
+
export async function readSecret(ref, env = process.env) {
|
|
148
|
+
let value;
|
|
149
|
+
if ("env" in ref)
|
|
150
|
+
value = env[ref.env];
|
|
151
|
+
else {
|
|
152
|
+
try {
|
|
153
|
+
value = (await readFile(ref.file, "utf8")).split(/\r?\n/)[0];
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
value = undefined;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const trimmed = value?.trim();
|
|
160
|
+
return trimmed === undefined || trimmed === "" ? undefined : trimmed;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
|
|
164
|
+
* keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
|
|
165
|
+
* no restart). A field that would hold a secret's value, or a provider header that would carry one, is refused.
|
|
166
|
+
*/
|
|
167
|
+
export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
168
|
+
check(raw, FILE, "config");
|
|
169
|
+
const file = raw;
|
|
170
|
+
const providers = file.providers.map((provider, index) => {
|
|
171
|
+
const where = `config.providers[${String(index)}]`;
|
|
172
|
+
for (const name of Object.keys(isRecord(provider.headers) ? provider.headers : {})) {
|
|
173
|
+
if (SECRET_HEADER.test(name))
|
|
174
|
+
throw configError(`${where}.headers.${name} would carry a secret in the config file; give keys under keys[] by reference`);
|
|
175
|
+
}
|
|
176
|
+
const keys = provider.keys.map((key, at) => {
|
|
177
|
+
const ref = secretRef({ env: key.env, file: key.file }, `${where}.keys[${String(at)}]`, base);
|
|
178
|
+
const common = {
|
|
179
|
+
alias: key.alias,
|
|
180
|
+
...(key.weight === undefined ? {} : { weight: key.weight }),
|
|
181
|
+
...(key.maxConcurrency === undefined ? {} : { maxConcurrency: key.maxConcurrency }),
|
|
182
|
+
};
|
|
183
|
+
return "env" in ref ? { ...common, env: ref.env } : { ...common, resolve: () => readSecret(ref) };
|
|
184
|
+
});
|
|
185
|
+
return { ...provider, keys };
|
|
186
|
+
});
|
|
187
|
+
const gateway = file.gateway ?? {};
|
|
188
|
+
const store = (gateway.keyStore ?? { type: "memory" });
|
|
189
|
+
if (store.type === "sqlite" && store.path === undefined)
|
|
190
|
+
throw configError("config.gateway.keyStore.path is required for a sqlite store");
|
|
191
|
+
const appPort = gateway.appPort;
|
|
192
|
+
const portOffset = gateway.portOffset;
|
|
193
|
+
if (gateway.port !== undefined && (appPort !== undefined || portOffset !== undefined))
|
|
194
|
+
throw configError("config.gateway.port cannot be combined with appPort or portOffset");
|
|
195
|
+
if (appPort === undefined && portOffset !== undefined)
|
|
196
|
+
throw configError("config.gateway.portOffset needs config.gateway.appPort");
|
|
197
|
+
if (appPort !== undefined && (!Number.isInteger(appPort) || appPort < 1 || appPort > 65535))
|
|
198
|
+
throw configError("config.gateway.appPort must be a port number");
|
|
199
|
+
if (portOffset !== undefined && (!Number.isInteger(portOffset) || portOffset < 1))
|
|
200
|
+
throw configError("config.gateway.portOffset must be an integer of 1 or more");
|
|
201
|
+
const port = appPort === undefined ? (gateway.port ?? 8080) : appPort + (portOffset ?? 1);
|
|
202
|
+
if (!Number.isInteger(port) || port < 0 || port > 65535)
|
|
203
|
+
throw configError(appPort === undefined ? "config.gateway.port must be a port number" : "config.gateway.appPort + portOffset must be a port number");
|
|
204
|
+
const maxBodyBytes = gateway.maxBodyBytes ?? 4 * 1024 * 1024;
|
|
205
|
+
if (!Number.isInteger(maxBodyBytes) || maxBodyBytes <= 0)
|
|
206
|
+
throw configError("config.gateway.maxBodyBytes must be a positive integer");
|
|
207
|
+
const pool = { ...file, providers };
|
|
208
|
+
delete pool.gateway;
|
|
209
|
+
return {
|
|
210
|
+
pool: pool,
|
|
211
|
+
gateway: {
|
|
212
|
+
port,
|
|
213
|
+
host: gateway.host ?? "127.0.0.1",
|
|
214
|
+
maxBodyBytes,
|
|
215
|
+
adminToken: isRecord(gateway.adminToken) ? secretRef(gateway.adminToken, "config.gateway.adminToken", base) : null,
|
|
216
|
+
keyStore: store.type === "sqlite" ? { type: "sqlite", path: store.path === ":memory:" ? ":memory:" : resolve(base, store.path ?? "") } : { type: "memory" },
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
|
|
221
|
+
export async function loadGatewayConfig(path) {
|
|
222
|
+
const full = resolve(path);
|
|
223
|
+
let text;
|
|
224
|
+
try {
|
|
225
|
+
text = await readFile(full, "utf8");
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
throw configError(`cannot read the config file ${full}`);
|
|
229
|
+
}
|
|
230
|
+
let raw;
|
|
231
|
+
try {
|
|
232
|
+
raw = extname(full).toLowerCase() === ".json" ? JSON.parse(text) : parseYaml(text);
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
throw configError(`the config file ${full} does not parse: ${error instanceof Error ? error.message : "unknown error"}`);
|
|
236
|
+
}
|
|
237
|
+
return parseGatewayConfig(raw, dirname(full));
|
|
238
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
2
|
+
import type { AiRouter, MaybePromise } from "../types.js";
|
|
3
|
+
import { KeyAdmin } from "./keys.js";
|
|
4
|
+
import type { KeyStore } from "./keys.js";
|
|
5
|
+
/** How a gateway is built over a router. */
|
|
6
|
+
export interface GatewayOptions {
|
|
7
|
+
/** Where virtual keys and their usage live. */
|
|
8
|
+
readonly keys: KeyStore;
|
|
9
|
+
/** Returns the admin token at each admin request; absent, or an empty answer, turns the admin API off. */
|
|
10
|
+
readonly adminToken?: () => MaybePromise<string | undefined>;
|
|
11
|
+
/** The largest request body read, in bytes. Default 4 MiB. */
|
|
12
|
+
readonly maxBodyBytes?: number;
|
|
13
|
+
/** The clock of quotas and usage days. Default `Date.now`. */
|
|
14
|
+
readonly clock?: () => number;
|
|
15
|
+
/** Receives a failure no response can carry any more (a usage write after the reply was sent). */
|
|
16
|
+
readonly onError?: (error: unknown) => void;
|
|
17
|
+
}
|
|
18
|
+
/** An OpenAI-compatible HTTP front over one router, with virtual keys, quotas, usage and an admin API. */
|
|
19
|
+
export interface AiRouterGateway {
|
|
20
|
+
/** The key administration the admin API and the CLI share. */
|
|
21
|
+
readonly admin: KeyAdmin;
|
|
22
|
+
/** Answers one HTTP request; mount it on any Node HTTP server or framework (a body a framework parsed is used as is). */
|
|
23
|
+
handle(request: IncomingMessage, response: ServerResponse): Promise<void>;
|
|
24
|
+
/** Listens on a port (0: any free port) and resolves to the bound address. */
|
|
25
|
+
listen(port: number, host?: string): Promise<{
|
|
26
|
+
readonly port: number;
|
|
27
|
+
readonly host: string;
|
|
28
|
+
readonly url: string;
|
|
29
|
+
}>;
|
|
30
|
+
/** Stops listening, ends open connections and releases the router subscription; the key store stays open. */
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
/** Creates the gateway over a router; nothing listens until `listen`. */
|
|
34
|
+
export declare function createGateway(router: AiRouter, options: GatewayOptions): AiRouterGateway;
|