@starci/ai-router 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +13 -3
package/dist/router.js
CHANGED
|
@@ -1,26 +1,18 @@
|
|
|
1
|
+
import { AttemptError } from "./attempt.js";
|
|
2
|
+
import { classifyFailure } from "./classify.js";
|
|
1
3
|
import { buildCatalog, resolvePolicy, resolveSettings, validateConfig } from "./config.js";
|
|
2
|
-
import { AiRouterError,
|
|
4
|
+
import { AiRouterError, isRetryable, scrub } from "./errors.js";
|
|
3
5
|
import { Emitter } from "./events.js";
|
|
4
6
|
import { readRateLimit } from "./headers.js";
|
|
5
7
|
import { PoolState } from "./pool.js";
|
|
8
|
+
import { baseUrlOf, presetOf } from "./providers.js";
|
|
9
|
+
import { resolveSecret } from "./secrets.js";
|
|
10
|
+
import { readChunk, readCompletion, readEmbeddings, readUsage, vectorLength } from "./shapes.js";
|
|
6
11
|
import { parseSse } from "./sse.js";
|
|
7
12
|
import { MemoryStateStore } from "./store.js";
|
|
13
|
+
import { wireOf } from "./wire/index.js";
|
|
8
14
|
const BUSY_POLL_MS = 250;
|
|
9
15
|
const ERROR_BODY_LIMIT = 4096;
|
|
10
|
-
/** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
|
|
11
|
-
class AttemptError extends Error {
|
|
12
|
-
errorClass;
|
|
13
|
-
status;
|
|
14
|
-
info;
|
|
15
|
-
missingKey;
|
|
16
|
-
constructor(errorClass, message, options = {}) {
|
|
17
|
-
super(message);
|
|
18
|
-
this.errorClass = errorClass;
|
|
19
|
-
this.status = options.status;
|
|
20
|
-
this.info = options.info ?? null;
|
|
21
|
-
this.missingKey = options.missingKey ?? false;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
16
|
/** One call's lifetime: its deadline, its cancellation, the pairs it tried and the failures it met. */
|
|
25
17
|
class Call {
|
|
26
18
|
requestId;
|
|
@@ -90,7 +82,7 @@ class Call {
|
|
|
90
82
|
class AttemptScope {
|
|
91
83
|
controller = new AbortController();
|
|
92
84
|
cause = null;
|
|
93
|
-
/** The key value this attempt sends, held only to scrub it from error messages. */
|
|
85
|
+
/** The key value this attempt sends, held only to scrub it from error messages and fingerprint a blacklist. */
|
|
94
86
|
secret;
|
|
95
87
|
#timer;
|
|
96
88
|
#call;
|
|
@@ -126,7 +118,7 @@ class AttemptScope {
|
|
|
126
118
|
// Whatever a provider echoed, the key value never leaves the attempt.
|
|
127
119
|
if (this.secret === undefined || !failure.message.includes(this.secret))
|
|
128
120
|
return failure;
|
|
129
|
-
return
|
|
121
|
+
return failure.withMessage(scrub(failure.message, [this.secret]));
|
|
130
122
|
}
|
|
131
123
|
/** Stops the timer and the call listener. */
|
|
132
124
|
dispose() {
|
|
@@ -144,8 +136,8 @@ function textOf(content) {
|
|
|
144
136
|
function estimateTokens(text) {
|
|
145
137
|
return Math.ceil(text.length / 4);
|
|
146
138
|
}
|
|
147
|
-
function chatNeed(request, streaming,
|
|
148
|
-
const capabilities = new Set(["chat", ...require]);
|
|
139
|
+
function chatNeed(request, streaming, policy) {
|
|
140
|
+
const capabilities = new Set(["chat", ...policy.require]);
|
|
149
141
|
if (streaming)
|
|
150
142
|
capabilities.add("streaming");
|
|
151
143
|
if (request.tools !== undefined && request.tools.length > 0)
|
|
@@ -156,61 +148,56 @@ function chatNeed(request, streaming, require) {
|
|
|
156
148
|
capabilities: [...capabilities],
|
|
157
149
|
promptTokens: estimateTokens(request.messages.map((message) => textOf(message.content)).join("\n")),
|
|
158
150
|
outputTokens: request.max_completion_tokens ?? request.max_tokens ?? 0,
|
|
151
|
+
oneIdentity: policy.pinned,
|
|
152
|
+
dimensions: null,
|
|
159
153
|
};
|
|
160
154
|
}
|
|
161
155
|
function hasToken(chunk) {
|
|
162
156
|
return chunk.choices.some((choice) => Object.entries(choice.delta).some(([field, value]) => field !== "role" && value !== null && value !== undefined && value !== "" && !(Array.isArray(value) && value.length === 0)));
|
|
163
157
|
}
|
|
164
158
|
function isRecord(value) {
|
|
165
|
-
return typeof value === "object" && value !== null;
|
|
166
|
-
}
|
|
167
|
-
function usageOf(json) {
|
|
168
|
-
return isRecord(json.usage) && typeof json.usage.prompt_tokens === "number" ? json.usage : undefined;
|
|
159
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
169
160
|
}
|
|
170
161
|
function statusOf(error) {
|
|
171
162
|
return error.status === undefined ? {} : { status: error.status };
|
|
172
163
|
}
|
|
173
|
-
/**
|
|
174
|
-
function
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
throw new AttemptError(classifyProviderError(json.error), message);
|
|
187
|
-
}
|
|
188
|
-
return Array.isArray(json.choices) ? json : null;
|
|
189
|
-
}
|
|
190
|
-
async function resolveSecret(key) {
|
|
191
|
-
const value = "env" in key ? process.env[key.env] : await key.resolve();
|
|
192
|
-
return value === undefined || value.trim() === "" ? undefined : value.trim();
|
|
164
|
+
/** The failure an error body describes: its class by the provider's kind, its message, its code and quota window. */
|
|
165
|
+
function failureOf(lane, status, body, info, fallback) {
|
|
166
|
+
const classified = classifyFailure(presetOf(lane.provider).kind, status, body);
|
|
167
|
+
const root = Array.isArray(body) ? body[0] : body;
|
|
168
|
+
const error = isRecord(root) && isRecord(root.error) ? root.error : root;
|
|
169
|
+
const detail = isRecord(error) && typeof error.message === "string" ? error.message : typeof body === "string" && body !== "" ? body : fallback;
|
|
170
|
+
const message = status === undefined ? detail : `HTTP ${String(status)}: ${detail}`;
|
|
171
|
+
return new AttemptError(classified.errorClass, message, {
|
|
172
|
+
...(status === undefined ? {} : { status }),
|
|
173
|
+
info,
|
|
174
|
+
...(classified.code === undefined ? {} : { code: classified.code }),
|
|
175
|
+
...(classified.window === undefined ? {} : { window: classified.window }),
|
|
176
|
+
});
|
|
193
177
|
}
|
|
194
|
-
async function
|
|
195
|
-
|
|
178
|
+
async function errorOf(lane, response) {
|
|
179
|
+
const info = readRateLimit(response.headers, Date.now());
|
|
180
|
+
let text = "";
|
|
196
181
|
try {
|
|
197
182
|
text = (await response.text()).slice(0, ERROR_BODY_LIMIT);
|
|
198
183
|
}
|
|
199
184
|
catch {
|
|
200
|
-
|
|
185
|
+
// An unreadable body: the status alone decides.
|
|
201
186
|
}
|
|
202
|
-
let
|
|
187
|
+
let body = text;
|
|
203
188
|
try {
|
|
204
|
-
|
|
205
|
-
if (isRecord(json) && isRecord(json.error) && typeof json.error.message === "string")
|
|
206
|
-
message = json.error.message;
|
|
207
|
-
else if (isRecord(json) && typeof json.message === "string")
|
|
208
|
-
message = json.message;
|
|
189
|
+
body = JSON.parse(text);
|
|
209
190
|
}
|
|
210
191
|
catch {
|
|
211
192
|
// Not JSON: the raw text is the message.
|
|
212
193
|
}
|
|
213
|
-
return
|
|
194
|
+
return failureOf(lane, response.status, body, info, response.statusText || "the provider refused the request");
|
|
195
|
+
}
|
|
196
|
+
function blacklistMessage(blacklisted, models) {
|
|
197
|
+
const keys = blacklisted.keys.length === 0 ? "" : ` keys ${blacklisted.keys.join(", ")}`;
|
|
198
|
+
const budgets = blacklisted.budgets.length === 0 ? "" : ` budget of ${blacklisted.budgets.join(", ")}`;
|
|
199
|
+
const what = blacklisted.reason === "quota" ? "quota or budget is spent" : "secret was refused or is missing";
|
|
200
|
+
return `every key serving ${models.join(", ")} is blacklisted (${what}):${keys}${budgets}`;
|
|
214
201
|
}
|
|
215
202
|
/** Creates a router over a pool of providers, keys and models. The pool is validated now; keys are read per attempt. */
|
|
216
203
|
export function createRouter(config) {
|
|
@@ -240,7 +227,8 @@ export function createRouter(config) {
|
|
|
240
227
|
if (call.controller.signal.aborted)
|
|
241
228
|
throw call.stopped();
|
|
242
229
|
if (call.lanes.length === 0) {
|
|
243
|
-
|
|
230
|
+
const dimensions = call.need.dimensions === null ? "" : ` at ${String(call.need.dimensions)} dimensions`;
|
|
231
|
+
throw call.error("no_candidate", `no provider serves ${call.policy.models.join(", ")} with ${call.need.capabilities.join(", ")}${dimensions} within its context window and budget`);
|
|
244
232
|
}
|
|
245
233
|
if (call.attempts >= call.policy.maxAttempts) {
|
|
246
234
|
const last = call.failures.at(-1);
|
|
@@ -251,9 +239,12 @@ export function createRouter(config) {
|
|
|
251
239
|
return picked;
|
|
252
240
|
const now = Date.now();
|
|
253
241
|
if (picked.waitUntil > call.deadlineAt) {
|
|
242
|
+
const retryAt = Number.isFinite(picked.waitUntil) ? { retryAt: picked.waitUntil } : {};
|
|
243
|
+
// Every key is out for its own reason, past the deadline: the call fails at once and says why.
|
|
244
|
+
if (picked.blacklisted !== null)
|
|
245
|
+
throw call.error(picked.blacklisted.reason, blacklistMessage(picked.blacklisted, call.policy.models), retryAt);
|
|
254
246
|
const last = call.failures.at(-1);
|
|
255
247
|
const message = `no candidate is usable before the deadline${last === undefined ? "" : `; last: ${last.message}`}`;
|
|
256
|
-
const retryAt = Number.isFinite(picked.waitUntil) ? { retryAt: picked.waitUntil } : {};
|
|
257
248
|
throw call.error(last?.errorClass ?? "no_candidate", message, retryAt);
|
|
258
249
|
}
|
|
259
250
|
try {
|
|
@@ -265,8 +256,8 @@ export function createRouter(config) {
|
|
|
265
256
|
}
|
|
266
257
|
}
|
|
267
258
|
async function backoff(call, errorClass) {
|
|
268
|
-
// Rate and auth failures belong to one key; the next key owes them no wait. Transient ones back off.
|
|
269
|
-
if (errorClass === "rate_limit" || errorClass === "auth")
|
|
259
|
+
// Rate, quota and auth failures belong to one key; the next key owes them no wait. Transient ones back off.
|
|
260
|
+
if (errorClass === "rate_limit" || errorClass === "quota" || errorClass === "auth")
|
|
270
261
|
return;
|
|
271
262
|
call.transientFailures++;
|
|
272
263
|
const { baseMs, factor, maxMs } = settings.backoff;
|
|
@@ -293,47 +284,45 @@ export function createRouter(config) {
|
|
|
293
284
|
}
|
|
294
285
|
}
|
|
295
286
|
function headers(lane, secret, call, options, accept) {
|
|
287
|
+
const wire = wireOf(lane.provider);
|
|
296
288
|
const out = {};
|
|
289
|
+
// Neither the provider's headers nor the caller's may carry or replace the secret's header.
|
|
297
290
|
for (const [name, value] of Object.entries({ ...lane.provider.headers, ...options.headers })) {
|
|
298
|
-
|
|
299
|
-
|
|
291
|
+
// Names are lower-cased, so `Anthropic-Version` and the wire's `anthropic-version` are one header, not two.
|
|
292
|
+
const lower = name.toLowerCase();
|
|
293
|
+
if (lower !== "authorization" && !wire.secretHeaders.includes(lower))
|
|
294
|
+
out[lower] = value;
|
|
300
295
|
}
|
|
301
296
|
return {
|
|
302
297
|
...out,
|
|
303
298
|
"content-type": "application/json",
|
|
304
299
|
accept,
|
|
305
|
-
|
|
300
|
+
...wire.headers(secret, lane.provider),
|
|
306
301
|
"x-request-id": call.requestId,
|
|
307
302
|
"idempotency-key": call.requestId,
|
|
308
303
|
};
|
|
309
304
|
}
|
|
310
|
-
async function open(call, pick, scope,
|
|
311
|
-
|
|
312
|
-
try {
|
|
313
|
-
secret = await resolveSecret(pick.lane.key);
|
|
314
|
-
}
|
|
315
|
-
catch {
|
|
316
|
-
secret = undefined;
|
|
317
|
-
}
|
|
305
|
+
async function open(call, pick, scope, operation, body, options) {
|
|
306
|
+
const secret = await resolveSecret(pick.lane.key);
|
|
318
307
|
if (secret === undefined)
|
|
319
308
|
throw new AttemptError("auth", `key ${pick.lane.key.alias} resolved to no value`, { missingKey: true });
|
|
320
309
|
scope.secret = secret;
|
|
310
|
+
const wire = wireOf(pick.lane.provider);
|
|
311
|
+
const request = operation === "embeddings" ? wire.embeddings(body, pick.lane.model.id) : wire.chat(body, pick.lane.model.id, operation === "stream", pick.lane.provider);
|
|
321
312
|
let response;
|
|
322
313
|
try {
|
|
323
|
-
response = await fetchImpl(`${pick.lane.provider
|
|
314
|
+
response = await fetchImpl(`${baseUrlOf(pick.lane.provider)}${request.path}`, {
|
|
324
315
|
method: "POST",
|
|
325
|
-
headers: headers(pick.lane, secret, call, options,
|
|
326
|
-
body: JSON.stringify(
|
|
316
|
+
headers: headers(pick.lane, secret, call, options, operation === "stream" ? "text/event-stream" : "application/json"),
|
|
317
|
+
body: JSON.stringify(request.body),
|
|
327
318
|
signal: scope.controller.signal,
|
|
328
319
|
});
|
|
329
320
|
}
|
|
330
321
|
catch (error) {
|
|
331
322
|
throw scope.classify(error);
|
|
332
323
|
}
|
|
333
|
-
if (!response.ok)
|
|
334
|
-
|
|
335
|
-
throw new AttemptError(classifyStatus(response.status), await errorBody(response), { status: response.status, info });
|
|
336
|
-
}
|
|
324
|
+
if (!response.ok)
|
|
325
|
+
throw await errorOf(pick.lane, response);
|
|
337
326
|
return response;
|
|
338
327
|
}
|
|
339
328
|
function begin(call, pick) {
|
|
@@ -343,8 +332,8 @@ export function createRouter(config) {
|
|
|
343
332
|
call.tried.providers.add(pick.lane.provider.id);
|
|
344
333
|
state.count(pick.lane);
|
|
345
334
|
}
|
|
346
|
-
async function fail(call, pick, error, latencyMs, content) {
|
|
347
|
-
await state.failed(pick.lane, error
|
|
335
|
+
async function fail(call, pick, error, secret, latencyMs, content) {
|
|
336
|
+
await state.failed(pick.lane, error, secret);
|
|
348
337
|
call.failures.push({
|
|
349
338
|
provider: pick.lane.provider.id,
|
|
350
339
|
key: pick.lane.key.alias,
|
|
@@ -416,7 +405,7 @@ export function createRouter(config) {
|
|
|
416
405
|
return error;
|
|
417
406
|
return call.error("network", error instanceof Error ? error.message : "the call failed");
|
|
418
407
|
}
|
|
419
|
-
async function unary(call,
|
|
408
|
+
async function unary(call, operation, body, options, parse) {
|
|
420
409
|
try {
|
|
421
410
|
for (;;) {
|
|
422
411
|
const pick = await next(call);
|
|
@@ -424,7 +413,7 @@ export function createRouter(config) {
|
|
|
424
413
|
const scope = new AttemptScope(call, call.policy.timeoutMs);
|
|
425
414
|
const started = Date.now();
|
|
426
415
|
try {
|
|
427
|
-
const response = await open(call, pick, scope,
|
|
416
|
+
const response = await open(call, pick, scope, operation, body, options);
|
|
428
417
|
let json;
|
|
429
418
|
try {
|
|
430
419
|
json = await response.json();
|
|
@@ -432,13 +421,12 @@ export function createRouter(config) {
|
|
|
432
421
|
catch (error) {
|
|
433
422
|
throw scope.cause === null ? new AttemptError("transient", "the provider answered a body that is not JSON") : scope.classify(error);
|
|
434
423
|
}
|
|
435
|
-
if (!isRecord(json))
|
|
424
|
+
if (!isRecord(json) && !Array.isArray(json))
|
|
436
425
|
throw new AttemptError("transient", "the provider answered a body that is not an object");
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
throw
|
|
440
|
-
|
|
441
|
-
const parsed = parse(json);
|
|
426
|
+
// An error inside a 200 body (OpenRouter does this) is classified like an error status.
|
|
427
|
+
if (!isRecord(json) || (json.error !== undefined && json.error !== null))
|
|
428
|
+
throw failureOf(pick.lane, undefined, json, null, "the provider answered an error");
|
|
429
|
+
const parsed = parse(json, pick.lane);
|
|
442
430
|
const latencyMs = Date.now() - started;
|
|
443
431
|
const routing = await succeed(call, pick, readRateLimit(response.headers, Date.now()), latencyMs, parsed.usage, { request: body, response: json });
|
|
444
432
|
settle(call, routing, null);
|
|
@@ -446,7 +434,7 @@ export function createRouter(config) {
|
|
|
446
434
|
}
|
|
447
435
|
catch (error) {
|
|
448
436
|
const failure = scope.classify(error);
|
|
449
|
-
await fail(call, pick, failure, Date.now() - started, body);
|
|
437
|
+
await fail(call, pick, failure, scope.secret, Date.now() - started, body);
|
|
450
438
|
if (failure.errorClass === "cancelled" || failure.errorClass === "deadline")
|
|
451
439
|
throw call.stopped();
|
|
452
440
|
if (!isRetryable(failure.errorClass)) {
|
|
@@ -480,19 +468,22 @@ export function createRouter(config) {
|
|
|
480
468
|
// "pending" in the finally block means the consumer stopped reading: no catch ran and no success did.
|
|
481
469
|
let outcome = "pending";
|
|
482
470
|
try {
|
|
483
|
-
const response = await open(call, pick, scope, "
|
|
471
|
+
const response = await open(call, pick, scope, "stream", body, options);
|
|
484
472
|
info = readRateLimit(response.headers, Date.now());
|
|
485
473
|
if (response.body === null)
|
|
486
474
|
throw new AttemptError("transient", "the provider answered no stream body");
|
|
487
475
|
const held = [];
|
|
488
|
-
for await (const event of parseSse(response.body)) {
|
|
489
|
-
if (event.
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
476
|
+
for await (const event of wireOf(pick.lane.provider).chunks(parseSse(response.body))) {
|
|
477
|
+
if (event.kind === "malformed")
|
|
478
|
+
throw new AttemptError("stream", "the provider streamed an event that is not JSON");
|
|
479
|
+
if (event.kind === "error")
|
|
480
|
+
throw failureOf(pick.lane, undefined, event.body, null, "the provider streamed an error");
|
|
481
|
+
const chunk = readChunk(event.chunk);
|
|
482
|
+
if (chunk === undefined)
|
|
493
483
|
continue;
|
|
494
|
-
|
|
495
|
-
|
|
484
|
+
const chunkUsage = readUsage(chunk.usage);
|
|
485
|
+
if (chunkUsage !== undefined)
|
|
486
|
+
usage = chunkUsage;
|
|
496
487
|
scope.arm(call.policy.timeoutMs);
|
|
497
488
|
if (firstToken) {
|
|
498
489
|
yield chunk;
|
|
@@ -516,7 +507,7 @@ export function createRouter(config) {
|
|
|
516
507
|
catch (error) {
|
|
517
508
|
outcome = "failed";
|
|
518
509
|
const failure = scope.classify(error);
|
|
519
|
-
await fail(call, pick, failure, Date.now() - started, body);
|
|
510
|
+
await fail(call, pick, failure, scope.secret, Date.now() - started, body);
|
|
520
511
|
if (failure.errorClass === "cancelled" || failure.errorClass === "deadline")
|
|
521
512
|
throw call.stopped();
|
|
522
513
|
if (firstToken) {
|
|
@@ -553,17 +544,19 @@ export function createRouter(config) {
|
|
|
553
544
|
async chat(request, options = {}) {
|
|
554
545
|
const body = { ...request };
|
|
555
546
|
delete body.stream;
|
|
556
|
-
const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy
|
|
557
|
-
const { value, routing } = await unary(start(), "
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
547
|
+
const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy));
|
|
548
|
+
const { value, routing } = await unary(start(), "chat", body, options, (json, lane) => {
|
|
549
|
+
const completion = readCompletion(wireOf(lane.provider).chatResponse(json));
|
|
550
|
+
if (completion === undefined)
|
|
551
|
+
throw new AttemptError("transient", "the provider answered a completion without choices");
|
|
552
|
+
return { value: completion, usage: completion.usage };
|
|
553
|
+
});
|
|
561
554
|
return { ...value, routing };
|
|
562
555
|
},
|
|
563
556
|
stream(request, options = {}) {
|
|
564
557
|
const body = { ...request };
|
|
565
558
|
delete body.stream;
|
|
566
|
-
const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy
|
|
559
|
+
const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy));
|
|
567
560
|
return {
|
|
568
561
|
requestId,
|
|
569
562
|
[Symbol.asyncIterator]: () => streamCall(start(), body, options),
|
|
@@ -574,11 +567,21 @@ export function createRouter(config) {
|
|
|
574
567
|
capabilities: ["embeddings", ...policy.require],
|
|
575
568
|
promptTokens: estimateTokens(typeof request.input === "string" ? request.input : request.input.join("\n")),
|
|
576
569
|
outputTokens: 0,
|
|
570
|
+
// An embeddings call never changes model space: every lane serves the first lane's model identity.
|
|
571
|
+
oneIdentity: true,
|
|
572
|
+
dimensions: request.dimensions ?? null,
|
|
577
573
|
}));
|
|
578
|
-
const { value, routing } = await unary(start(), "
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
574
|
+
const { value, routing } = await unary(start(), "embeddings", { ...request }, options, (json, lane) => {
|
|
575
|
+
const embeddings = readEmbeddings(json);
|
|
576
|
+
if (embeddings === undefined)
|
|
577
|
+
throw new AttemptError("transient", "the provider answered embeddings without a list of vectors");
|
|
578
|
+
const expected = lane.model.dimensions;
|
|
579
|
+
const wrong = expected === undefined ? undefined : embeddings.data.find((entry) => vectorLength(entry.embedding) !== expected);
|
|
580
|
+
// A vector of another length is another space: never handed to a caller whose index holds the pinned one.
|
|
581
|
+
if (wrong !== undefined)
|
|
582
|
+
throw new AttemptError("transient", `the provider answered ${String(vectorLength(wrong.embedding))}-dimensional vectors, the model is declared at ${String(expected)}`);
|
|
583
|
+
return { value: embeddings, usage: embeddings.usage };
|
|
584
|
+
});
|
|
582
585
|
return { ...value, routing };
|
|
583
586
|
},
|
|
584
587
|
catalog: () => catalog,
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { KeyConfig } from "./types.js";
|
|
2
|
+
/** Reads a key's secret now: its env variable or its resolver, trimmed; undefined when absent, empty or the resolver throws. */
|
|
3
|
+
export declare function resolveSecret(key: KeyConfig): Promise<string | undefined>;
|
|
4
|
+
/**
|
|
5
|
+
* The fingerprint the blacklist keeps of a secret, to notice a new one: 16 hex (64 bits) of a namespaced SHA-256, `""`
|
|
6
|
+
* for no secret. It identifies a change; it cannot give the secret back.
|
|
7
|
+
*/
|
|
8
|
+
export declare function fingerprintOf(secret: string | undefined): string;
|
package/dist/secrets.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
/** Reads a key's secret now: its env variable or its resolver, trimmed; undefined when absent, empty or the resolver throws. */
|
|
3
|
+
export async function resolveSecret(key) {
|
|
4
|
+
let value;
|
|
5
|
+
try {
|
|
6
|
+
value = "env" in key ? process.env[key.env] : await key.resolve();
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
const trimmed = value?.trim();
|
|
12
|
+
return trimmed === undefined || trimmed === "" ? undefined : trimmed;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The fingerprint the blacklist keeps of a secret, to notice a new one: 16 hex (64 bits) of a namespaced SHA-256, `""`
|
|
16
|
+
* for no secret. It identifies a change; it cannot give the secret back.
|
|
17
|
+
*/
|
|
18
|
+
export function fingerprintOf(secret) {
|
|
19
|
+
if (secret === undefined)
|
|
20
|
+
return "";
|
|
21
|
+
return createHash("sha256").update(`ai-router:fingerprint:${secret}`, "utf8").digest("hex").slice(0, 16);
|
|
22
|
+
}
|
package/dist/shapes.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ChatCompletionBody, ChatCompletionChunk, EmbeddingsBody, Usage } from "./types.js";
|
|
2
|
+
/** The usage of an answer, when it reports prompt tokens. */
|
|
3
|
+
export declare function readUsage(value: unknown): Usage | undefined;
|
|
4
|
+
/** A chat completion, or undefined when the body has no readable choices. */
|
|
5
|
+
export declare function readCompletion(json: Record<string, unknown>): ChatCompletionBody | undefined;
|
|
6
|
+
/** A stream chunk, or undefined when it has no list of choices. */
|
|
7
|
+
export declare function readChunk(json: Record<string, unknown>): ChatCompletionChunk | undefined;
|
|
8
|
+
/** The length of one vector: its numbers, or the float32 values of a base64 string. */
|
|
9
|
+
export declare function vectorLength(embedding: readonly number[] | string): number;
|
|
10
|
+
/** An embeddings response, or undefined when its data is not a list of vectors. */
|
|
11
|
+
export declare function readEmbeddings(json: Record<string, unknown>): EmbeddingsBody | undefined;
|
package/dist/shapes.js
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reads provider answers into the router's typed shapes, field by field. A provider's extra fields (DeepSeek's
|
|
3
|
+
* `reasoning_content`, a `refusal`, `system_fingerprint`) ride along; a body without the fields the router reads is
|
|
4
|
+
* refused (undefined), and the attempt fails as a malformed answer.
|
|
5
|
+
*/
|
|
6
|
+
const ASSISTANT = "assistant";
|
|
7
|
+
function isRecord(value) {
|
|
8
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9
|
+
}
|
|
10
|
+
function text(value, fallback = "") {
|
|
11
|
+
return typeof value === "string" ? value : fallback;
|
|
12
|
+
}
|
|
13
|
+
function number(value, fallback = 0) {
|
|
14
|
+
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
15
|
+
}
|
|
16
|
+
/** The usage of an answer, when it reports prompt tokens. */
|
|
17
|
+
export function readUsage(value) {
|
|
18
|
+
if (!isRecord(value) || typeof value.prompt_tokens !== "number")
|
|
19
|
+
return undefined;
|
|
20
|
+
const completion = typeof value.completion_tokens === "number" ? { completion_tokens: value.completion_tokens } : {};
|
|
21
|
+
return { ...value, prompt_tokens: value.prompt_tokens, ...completion, total_tokens: number(value.total_tokens, value.prompt_tokens + number(value.completion_tokens)) };
|
|
22
|
+
}
|
|
23
|
+
function toolCall(value) {
|
|
24
|
+
if (!isRecord(value) || !isRecord(value.function) || typeof value.id !== "string" || typeof value.function.name !== "string")
|
|
25
|
+
return undefined;
|
|
26
|
+
return { id: value.id, type: "function", function: { name: value.function.name, arguments: text(value.function.arguments, "{}") } };
|
|
27
|
+
}
|
|
28
|
+
function choice(value, at) {
|
|
29
|
+
if (!isRecord(value) || !isRecord(value.message))
|
|
30
|
+
return undefined;
|
|
31
|
+
const message = value.message;
|
|
32
|
+
const calls = Array.isArray(message.tool_calls) ? message.tool_calls.map(toolCall).filter((call) => call !== undefined) : [];
|
|
33
|
+
return {
|
|
34
|
+
index: number(value.index, at),
|
|
35
|
+
message: { ...message, role: "assistant", content: typeof message.content === "string" ? message.content : null, ...(calls.length > 0 ? { tool_calls: calls } : {}) },
|
|
36
|
+
finish_reason: typeof value.finish_reason === "string" ? value.finish_reason : null,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** A chat completion, or undefined when the body has no readable choices. */
|
|
40
|
+
export function readCompletion(json) {
|
|
41
|
+
if (!Array.isArray(json.choices))
|
|
42
|
+
return undefined;
|
|
43
|
+
const choices = json.choices.map(choice);
|
|
44
|
+
if (choices.some((entry) => entry === undefined))
|
|
45
|
+
return undefined;
|
|
46
|
+
const usage = readUsage(json.usage);
|
|
47
|
+
return {
|
|
48
|
+
...json,
|
|
49
|
+
id: text(json.id),
|
|
50
|
+
object: "chat.completion",
|
|
51
|
+
created: number(json.created),
|
|
52
|
+
model: text(json.model),
|
|
53
|
+
choices: choices.filter((entry) => entry !== undefined),
|
|
54
|
+
...(usage === undefined ? {} : { usage }),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
function toolDelta(value, at) {
|
|
58
|
+
if (!isRecord(value))
|
|
59
|
+
return undefined;
|
|
60
|
+
const fn = isRecord(value.function) ? value.function : undefined;
|
|
61
|
+
return {
|
|
62
|
+
index: number(value.index, at),
|
|
63
|
+
...(typeof value.id === "string" ? { id: value.id } : {}),
|
|
64
|
+
...(value.type === "function" ? { type: "function" } : {}),
|
|
65
|
+
...(fn === undefined
|
|
66
|
+
? {}
|
|
67
|
+
: { function: { ...(typeof fn.name === "string" ? { name: fn.name } : {}), ...(typeof fn.arguments === "string" ? { arguments: fn.arguments } : {}) } }),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** A stream chunk, or undefined when it has no list of choices. */
|
|
71
|
+
export function readChunk(json) {
|
|
72
|
+
if (!Array.isArray(json.choices))
|
|
73
|
+
return undefined;
|
|
74
|
+
const choices = [];
|
|
75
|
+
for (const [at, entry] of json.choices.entries()) {
|
|
76
|
+
if (!isRecord(entry))
|
|
77
|
+
return undefined;
|
|
78
|
+
const delta = isRecord(entry.delta) ? entry.delta : {};
|
|
79
|
+
const calls = Array.isArray(delta.tool_calls) ? delta.tool_calls.map(toolDelta).filter((call) => call !== undefined) : undefined;
|
|
80
|
+
const { role, content } = delta;
|
|
81
|
+
const rest = Object.fromEntries(Object.entries(delta).filter(([field]) => field !== "role" && field !== "content" && field !== "tool_calls"));
|
|
82
|
+
choices.push({
|
|
83
|
+
index: number(entry.index, at),
|
|
84
|
+
delta: {
|
|
85
|
+
...rest,
|
|
86
|
+
...(role === ASSISTANT ? { role: ASSISTANT } : {}),
|
|
87
|
+
...(typeof content === "string" || content === null ? { content } : {}),
|
|
88
|
+
...(calls === undefined ? {} : { tool_calls: calls }),
|
|
89
|
+
},
|
|
90
|
+
finish_reason: typeof entry.finish_reason === "string" ? entry.finish_reason : null,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
const usage = json.usage === null ? null : readUsage(json.usage);
|
|
94
|
+
return {
|
|
95
|
+
...json,
|
|
96
|
+
id: text(json.id),
|
|
97
|
+
object: "chat.completion.chunk",
|
|
98
|
+
created: number(json.created),
|
|
99
|
+
model: text(json.model),
|
|
100
|
+
choices,
|
|
101
|
+
...(usage === undefined ? {} : { usage }),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** The length of one vector: its numbers, or the float32 values of a base64 string. */
|
|
105
|
+
export function vectorLength(embedding) {
|
|
106
|
+
return typeof embedding === "string" ? Buffer.from(embedding, "base64").length / 4 : embedding.length;
|
|
107
|
+
}
|
|
108
|
+
/** An embeddings response, or undefined when its data is not a list of vectors. */
|
|
109
|
+
export function readEmbeddings(json) {
|
|
110
|
+
if (!Array.isArray(json.data))
|
|
111
|
+
return undefined;
|
|
112
|
+
const data = [];
|
|
113
|
+
for (const [at, entry] of json.data.entries()) {
|
|
114
|
+
if (!isRecord(entry))
|
|
115
|
+
return undefined;
|
|
116
|
+
const vector = entry.embedding;
|
|
117
|
+
const numbers = Array.isArray(vector) ? vector.filter((value) => typeof value === "number") : undefined;
|
|
118
|
+
if (typeof vector !== "string" && (numbers === undefined || !Array.isArray(vector) || numbers.length !== vector.length))
|
|
119
|
+
return undefined;
|
|
120
|
+
data.push({ object: "embedding", index: number(entry.index, at), embedding: typeof vector === "string" ? vector : (numbers ?? []) });
|
|
121
|
+
}
|
|
122
|
+
const usage = readUsage(json.usage);
|
|
123
|
+
return { object: "list", data, model: text(json.model), ...(usage === undefined ? {} : { usage }) };
|
|
124
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { HealthRecord, StateStore } from "./store.js";
|
|
2
|
+
/** The schema of the durable state store: one JSON record per key, and the half-open probe leases. */
|
|
3
|
+
export declare const STATE_STORE_SCHEMA = "\nCREATE TABLE IF NOT EXISTS ai_router_state (\n key TEXT PRIMARY KEY,\n record TEXT NOT NULL,\n updated_at INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS ai_router_leases (\n key TEXT PRIMARY KEY,\n until INTEGER NOT NULL\n);\n";
|
|
4
|
+
/** The SQLite state store: a `StateStore` whose methods answer at once, and that can be closed. */
|
|
5
|
+
export interface ClosableStateStore extends StateStore {
|
|
6
|
+
get(key: string): HealthRecord | undefined;
|
|
7
|
+
set(key: string, record: HealthRecord): void;
|
|
8
|
+
delete(key: string): void;
|
|
9
|
+
keys(prefix: string): readonly string[];
|
|
10
|
+
lease(key: string, ttlMs: number): boolean;
|
|
11
|
+
release(key: string): void;
|
|
12
|
+
close(): void;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Opens (and creates) the durable state store at a SQLite file path (`node:sqlite`, Node 22.13 or newer); `:memory:`
|
|
16
|
+
* gives a throwaway one. It may share the key store's file: the tables differ.
|
|
17
|
+
*/
|
|
18
|
+
export declare function openSqliteStateStore(path: string): Promise<ClosableStateStore>;
|
|
19
|
+
/**
|
|
20
|
+
* The few Redis commands the state store needs, so any client fits in a few lines (node-redis:
|
|
21
|
+
* `{ get: (k) => c.get(k), set: (k, v) => c.set(k, v), setIfAbsent: async (k, v, ms) => (await c.set(k, v, { NX: true, PX: ms })) === "OK", del: (k) => c.del(k), keys: (p) => c.keys(p) }`).
|
|
22
|
+
*/
|
|
23
|
+
export interface RedisLike {
|
|
24
|
+
get(key: string): Promise<string | null>;
|
|
25
|
+
set(key: string, value: string): Promise<unknown>;
|
|
26
|
+
/** `SET key value NX PX ttlMs`: true when it was set. */
|
|
27
|
+
setIfAbsent(key: string, value: string, ttlMs: number): Promise<boolean>;
|
|
28
|
+
del(key: string): Promise<unknown>;
|
|
29
|
+
/** `KEYS pattern` (or a SCAN loop): the keys matching a glob. */
|
|
30
|
+
keys(pattern: string): Promise<readonly string[]>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* A `StateStore` over Redis: one JSON string per record and `SET NX PX` leases, so every gateway process of an app sees
|
|
34
|
+
* the same breakers and blacklists. `prefix` (default `state:`) separates it from the lease keys.
|
|
35
|
+
*/
|
|
36
|
+
export declare function redisStateStore(client: RedisLike, options?: {
|
|
37
|
+
readonly prefix?: string;
|
|
38
|
+
}): StateStore;
|