@starci/ai-router 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +13 -3
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { parseHealthRecord } from "./store.js";
|
|
2
|
+
/** The schema of the durable state store: one JSON record per key, and the half-open probe leases. */
|
|
3
|
+
export const STATE_STORE_SCHEMA = `
|
|
4
|
+
CREATE TABLE IF NOT EXISTS ai_router_state (
|
|
5
|
+
key TEXT PRIMARY KEY,
|
|
6
|
+
record TEXT NOT NULL,
|
|
7
|
+
updated_at INTEGER NOT NULL
|
|
8
|
+
);
|
|
9
|
+
CREATE TABLE IF NOT EXISTS ai_router_leases (
|
|
10
|
+
key TEXT PRIMARY KEY,
|
|
11
|
+
until INTEGER NOT NULL
|
|
12
|
+
);
|
|
13
|
+
`;
|
|
14
|
+
/** The pool's state over one SQLite file: breakers, blacklists, headroom and budgets survive a restart; processes on one host may share the file. */
|
|
15
|
+
class SqliteStateStore {
|
|
16
|
+
#db;
|
|
17
|
+
constructor(db) {
|
|
18
|
+
this.#db = db;
|
|
19
|
+
}
|
|
20
|
+
get(key) {
|
|
21
|
+
const row = this.#db.prepare("SELECT record FROM ai_router_state WHERE key = ?").get(key);
|
|
22
|
+
const text = row?.record;
|
|
23
|
+
return typeof text === "string" ? parseHealthRecord(text) : undefined;
|
|
24
|
+
}
|
|
25
|
+
set(key, record) {
|
|
26
|
+
this.#db
|
|
27
|
+
.prepare("INSERT INTO ai_router_state (key, record, updated_at) VALUES (?, ?, ?) ON CONFLICT (key) DO UPDATE SET record = excluded.record, updated_at = excluded.updated_at")
|
|
28
|
+
.run(key, JSON.stringify(record), Date.now());
|
|
29
|
+
}
|
|
30
|
+
delete(key) {
|
|
31
|
+
this.#db.prepare("DELETE FROM ai_router_state WHERE key = ?").run(key);
|
|
32
|
+
}
|
|
33
|
+
keys(prefix) {
|
|
34
|
+
// A range scan, not LIKE: a prefix may hold `%` or `_`.
|
|
35
|
+
return this.#db
|
|
36
|
+
.prepare("SELECT key FROM ai_router_state WHERE key >= ? AND key < ? ORDER BY key")
|
|
37
|
+
.all(prefix, `${prefix}\u{10FFFF}`)
|
|
38
|
+
.map((row) => row.key)
|
|
39
|
+
.filter((key) => typeof key === "string");
|
|
40
|
+
}
|
|
41
|
+
lease(key, ttlMs) {
|
|
42
|
+
const now = Date.now();
|
|
43
|
+
// One statement: it inserts a free lease or takes over an expired one, and returns a row only when it did.
|
|
44
|
+
const row = this.#db
|
|
45
|
+
.prepare("INSERT INTO ai_router_leases (key, until) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET until = excluded.until WHERE ai_router_leases.until <= ? RETURNING key")
|
|
46
|
+
.get(key, now + ttlMs, now);
|
|
47
|
+
return row !== undefined;
|
|
48
|
+
}
|
|
49
|
+
release(key) {
|
|
50
|
+
this.#db.prepare("DELETE FROM ai_router_leases WHERE key = ?").run(key);
|
|
51
|
+
}
|
|
52
|
+
close() {
|
|
53
|
+
this.#db.close();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Opens (and creates) the durable state store at a SQLite file path (`node:sqlite`, Node 22.13 or newer); `:memory:`
|
|
58
|
+
* gives a throwaway one. It may share the key store's file: the tables differ.
|
|
59
|
+
*/
|
|
60
|
+
export async function openSqliteStateStore(path) {
|
|
61
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
62
|
+
const db = new DatabaseSync(path);
|
|
63
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
64
|
+
if (path !== ":memory:")
|
|
65
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
66
|
+
db.exec(STATE_STORE_SCHEMA);
|
|
67
|
+
return new SqliteStateStore(db);
|
|
68
|
+
}
|
|
69
|
+
/** Escapes a Redis glob's special characters so a prefix matches literally. */
|
|
70
|
+
function globPrefix(prefix) {
|
|
71
|
+
return `${prefix.replace(/[*?[\]\\]/g, (char) => `\\${char}`)}*`;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* A `StateStore` over Redis: one JSON string per record and `SET NX PX` leases, so every gateway process of an app sees
|
|
75
|
+
* the same breakers and blacklists. `prefix` (default `state:`) separates it from the lease keys.
|
|
76
|
+
*/
|
|
77
|
+
export function redisStateStore(client, options = {}) {
|
|
78
|
+
const records = options.prefix ?? "state:";
|
|
79
|
+
const leases = "lease:";
|
|
80
|
+
return {
|
|
81
|
+
async get(key) {
|
|
82
|
+
return parseHealthRecord(await client.get(`${records}${key}`));
|
|
83
|
+
},
|
|
84
|
+
async set(key, record) {
|
|
85
|
+
await client.set(`${records}${key}`, JSON.stringify(record));
|
|
86
|
+
},
|
|
87
|
+
async delete(key) {
|
|
88
|
+
await client.del(`${records}${key}`);
|
|
89
|
+
},
|
|
90
|
+
async keys(prefix) {
|
|
91
|
+
const found = await client.keys(globPrefix(`${records}${prefix}`));
|
|
92
|
+
return found.map((key) => key.slice(records.length)).sort();
|
|
93
|
+
},
|
|
94
|
+
lease(key, ttlMs) {
|
|
95
|
+
return client.setIfAbsent(`${leases}${key}`, "1", ttlMs);
|
|
96
|
+
},
|
|
97
|
+
async release(key) {
|
|
98
|
+
await client.del(`${leases}${key}`);
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
package/dist/store.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { MaybePromise } from "./types.js";
|
|
1
|
+
import type { DisabledReason, MaybePromise } from "./types.js";
|
|
2
2
|
/**
|
|
3
3
|
* The shared health of one key, one provider or one key and model ("lane"). Every field is a JSON number, string or
|
|
4
4
|
* null, so an adapter can store it anywhere. Times are epoch milliseconds; 0 means "not set".
|
|
@@ -11,9 +11,16 @@ export interface HealthRecord {
|
|
|
11
11
|
readonly openUntil: number;
|
|
12
12
|
/** The cooldown the breaker last opened with; the next failed probe doubles it. */
|
|
13
13
|
readonly cooldownMs: number;
|
|
14
|
-
/** A
|
|
14
|
+
/** A blacklisted key is no candidate until then; `Number.MAX_SAFE_INTEGER` means until its secret changes or `reset()`. */
|
|
15
15
|
readonly disabledUntil: number;
|
|
16
|
-
readonly disabledReason:
|
|
16
|
+
readonly disabledReason: DisabledReason | null;
|
|
17
|
+
/** The provider's error code that blacklisted the key, for health and telemetry. */
|
|
18
|
+
readonly disabledCode: string | null;
|
|
19
|
+
/**
|
|
20
|
+
* A 64-bit fingerprint (SHA-256, truncated) of the secret the key held when it was blacklisted, `""` when it had
|
|
21
|
+
* none: a different secret lifts the blacklist. Never the secret.
|
|
22
|
+
*/
|
|
23
|
+
readonly fingerprint: string | null;
|
|
17
24
|
/** The lane's last reported remaining requests, or null when the provider never said. */
|
|
18
25
|
readonly remainingRequests: number | null;
|
|
19
26
|
readonly requestsResetAt: number;
|
|
@@ -21,10 +28,15 @@ export interface HealthRecord {
|
|
|
21
28
|
readonly tokensResetAt: number;
|
|
22
29
|
/** A 429 keeps the lane out until then. */
|
|
23
30
|
readonly cooldownUntil: number;
|
|
31
|
+
/** A provider's spend this UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`), against its `budget`. */
|
|
32
|
+
readonly spendDay: string;
|
|
33
|
+
readonly spendDayCost: number;
|
|
34
|
+
readonly spendMonth: string;
|
|
35
|
+
readonly spendMonthCost: number;
|
|
24
36
|
}
|
|
25
37
|
/**
|
|
26
|
-
* Where breaker,
|
|
27
|
-
*
|
|
38
|
+
* Where breaker, blacklist, headroom and budget state lives. The in-memory store is the default; `openSqliteStateStore`
|
|
39
|
+
* and `redisStateStore` keep it across restarts and share it between processes. Methods may be synchronous or async.
|
|
28
40
|
*/
|
|
29
41
|
export interface StateStore {
|
|
30
42
|
get(key: string): MaybePromise<HealthRecord | undefined>;
|
|
@@ -36,9 +48,16 @@ export interface StateStore {
|
|
|
36
48
|
lease(key: string, ttlMs: number): MaybePromise<boolean>;
|
|
37
49
|
release(key: string): MaybePromise<void>;
|
|
38
50
|
}
|
|
39
|
-
/** A record with nothing known: a closed breaker, an enabled key, unknown headroom. */
|
|
51
|
+
/** A record with nothing known: a closed breaker, an enabled key, unknown headroom, nothing spent. */
|
|
40
52
|
export declare const EMPTY_RECORD: HealthRecord;
|
|
41
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Reads a stored record field by field, filling what an older version did not write with the empty record's value; a
|
|
55
|
+
* value that is not a record at all reads as undefined. Durable adapters parse their JSON through it.
|
|
56
|
+
*/
|
|
57
|
+
export declare function toHealthRecord(value: unknown): HealthRecord | undefined;
|
|
58
|
+
/** Parses a record a durable adapter stored as JSON; unreadable text reads as undefined. */
|
|
59
|
+
export declare function parseHealthRecord(text: string | null | undefined): HealthRecord | undefined;
|
|
60
|
+
/** The default store: one process's memory. A restart forgets every blacklist; use a durable store in production. */
|
|
42
61
|
export declare class MemoryStateStore implements StateStore {
|
|
43
62
|
#private;
|
|
44
63
|
/** Reads a record. */
|
package/dist/store.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** A record with nothing known: a closed breaker, an enabled key, unknown headroom. */
|
|
1
|
+
/** A record with nothing known: a closed breaker, an enabled key, unknown headroom, nothing spent. */
|
|
2
2
|
export const EMPTY_RECORD = Object.freeze({
|
|
3
3
|
breaker: "closed",
|
|
4
4
|
failures: 0,
|
|
@@ -6,13 +6,75 @@ export const EMPTY_RECORD = Object.freeze({
|
|
|
6
6
|
cooldownMs: 0,
|
|
7
7
|
disabledUntil: 0,
|
|
8
8
|
disabledReason: null,
|
|
9
|
+
disabledCode: null,
|
|
10
|
+
fingerprint: null,
|
|
9
11
|
remainingRequests: null,
|
|
10
12
|
requestsResetAt: 0,
|
|
11
13
|
remainingTokens: null,
|
|
12
14
|
tokensResetAt: 0,
|
|
13
15
|
cooldownUntil: 0,
|
|
16
|
+
spendDay: "",
|
|
17
|
+
spendDayCost: 0,
|
|
18
|
+
spendMonth: "",
|
|
19
|
+
spendMonthCost: 0,
|
|
14
20
|
});
|
|
15
|
-
|
|
21
|
+
function isRecord(value) {
|
|
22
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
23
|
+
}
|
|
24
|
+
function num(value, fallback) {
|
|
25
|
+
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
26
|
+
}
|
|
27
|
+
function numOrNull(value) {
|
|
28
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
29
|
+
}
|
|
30
|
+
function str(value, fallback) {
|
|
31
|
+
return typeof value === "string" ? value : fallback;
|
|
32
|
+
}
|
|
33
|
+
function strOrNull(value) {
|
|
34
|
+
return typeof value === "string" ? value : null;
|
|
35
|
+
}
|
|
36
|
+
function reason(value) {
|
|
37
|
+
return value === "auth" || value === "missing" || value === "quota" ? value : null;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Reads a stored record field by field, filling what an older version did not write with the empty record's value; a
|
|
41
|
+
* value that is not a record at all reads as undefined. Durable adapters parse their JSON through it.
|
|
42
|
+
*/
|
|
43
|
+
export function toHealthRecord(value) {
|
|
44
|
+
if (!isRecord(value))
|
|
45
|
+
return undefined;
|
|
46
|
+
return {
|
|
47
|
+
breaker: value.breaker === "open" ? "open" : "closed",
|
|
48
|
+
failures: num(value.failures, 0),
|
|
49
|
+
openUntil: num(value.openUntil, 0),
|
|
50
|
+
cooldownMs: num(value.cooldownMs, 0),
|
|
51
|
+
disabledUntil: num(value.disabledUntil, 0),
|
|
52
|
+
disabledReason: reason(value.disabledReason),
|
|
53
|
+
disabledCode: strOrNull(value.disabledCode),
|
|
54
|
+
fingerprint: strOrNull(value.fingerprint),
|
|
55
|
+
remainingRequests: numOrNull(value.remainingRequests),
|
|
56
|
+
requestsResetAt: num(value.requestsResetAt, 0),
|
|
57
|
+
remainingTokens: numOrNull(value.remainingTokens),
|
|
58
|
+
tokensResetAt: num(value.tokensResetAt, 0),
|
|
59
|
+
cooldownUntil: num(value.cooldownUntil, 0),
|
|
60
|
+
spendDay: str(value.spendDay, ""),
|
|
61
|
+
spendDayCost: num(value.spendDayCost, 0),
|
|
62
|
+
spendMonth: str(value.spendMonth, ""),
|
|
63
|
+
spendMonthCost: num(value.spendMonthCost, 0),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Parses a record a durable adapter stored as JSON; unreadable text reads as undefined. */
|
|
67
|
+
export function parseHealthRecord(text) {
|
|
68
|
+
if (text === null || text === undefined)
|
|
69
|
+
return undefined;
|
|
70
|
+
try {
|
|
71
|
+
return toHealthRecord(JSON.parse(text));
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** The default store: one process's memory. A restart forgets every blacklist; use a durable store in production. */
|
|
16
78
|
export class MemoryStateStore {
|
|
17
79
|
#records = new Map();
|
|
18
80
|
#leases = new Map();
|
package/dist/testing.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { AiRouterTestingModule } from "./ai-router-testing.module.js";
|
|
2
2
|
export type { AiRouterTestingOptions } from "./ai-router-testing.module.js";
|
|
3
|
-
export { createFakeProvider } from "./fake.js";
|
|
4
|
-
export type { FakeCall, FakeMatch, FakeProvider, FakeReply, FakeStream } from "./fake.js";
|
|
3
|
+
export { createFakeProvider, PROVIDER_ERRORS } from "./fake.js";
|
|
4
|
+
export type { FakeCall, FakeErrorCase, FakeMatch, FakeProvider, FakeReply, FakeStream, FakeToolCall } from "./fake.js";
|
package/dist/testing.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export { AiRouterTestingModule } from "./ai-router-testing.module.js";
|
|
2
|
-
export { createFakeProvider } from "./fake.js";
|
|
2
|
+
export { createFakeProvider, PROVIDER_ERRORS } from "./fake.js";
|
package/dist/types.d.ts
CHANGED
|
@@ -17,6 +17,14 @@ export interface ModelConfig {
|
|
|
17
17
|
/** Omitted means every chat capability (chat, streaming, tools, json) and no embeddings. */
|
|
18
18
|
readonly capabilities?: readonly Capability[];
|
|
19
19
|
readonly weight?: number;
|
|
20
|
+
/**
|
|
21
|
+
* The model's identity across providers, when two providers name one model differently (`text-embedding-3-small` at
|
|
22
|
+
* OpenAI, `openai/text-embedding-3-small` at OpenRouter). Default: the id. A pinned profile and every embeddings call
|
|
23
|
+
* stay on one identity.
|
|
24
|
+
*/
|
|
25
|
+
readonly identity?: string;
|
|
26
|
+
/** The length of the vectors an embeddings model answers; a response of another length fails the attempt. */
|
|
27
|
+
readonly dimensions?: number;
|
|
20
28
|
}
|
|
21
29
|
/** Returns the secret of a key at the moment of an attempt; an empty or missing answer disables the key. */
|
|
22
30
|
export type KeyResolver = () => MaybePromise<string | undefined>;
|
|
@@ -37,25 +45,47 @@ export interface ResolverKeyConfig extends KeyCommon {
|
|
|
37
45
|
}
|
|
38
46
|
/** One API key of a provider, given by reference: an environment variable name or a resolver. */
|
|
39
47
|
export type KeyConfig = EnvKeyConfig | ResolverKeyConfig;
|
|
40
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Which provider an endpoint is: it sets the wire protocol, the auth header, the default base URL, whether it serves
|
|
50
|
+
* embeddings, and how its errors are classified (docs/design.md, the provider matrix). `openai-compatible` (the default)
|
|
51
|
+
* is any endpoint that speaks the OpenAI chat-completions API with a bearer key: Ollama, vLLM, Together, LiteLLM...
|
|
52
|
+
*/
|
|
53
|
+
export type ProviderKind = "openai" | "anthropic" | "openrouter" | "qwen" | "deepseek" | "gemini" | "groq" | "openai-compatible";
|
|
54
|
+
/** A provider's own spending cap, in the unit of the models' `cost`; past it the provider is no candidate until the next UTC day or month. */
|
|
55
|
+
export interface ProviderBudget {
|
|
56
|
+
readonly costPerDay?: number;
|
|
57
|
+
readonly costPerMonth?: number;
|
|
58
|
+
}
|
|
59
|
+
/** One provider endpoint with its keys and models. */
|
|
41
60
|
export interface ProviderConfig {
|
|
42
61
|
readonly id: string;
|
|
43
|
-
/**
|
|
44
|
-
readonly
|
|
62
|
+
/** Default `openai-compatible`. */
|
|
63
|
+
readonly kind?: ProviderKind;
|
|
64
|
+
/** The base URL the API paths hang from, such as `https://api.openai.com/v1`; required for `openai-compatible`, else the kind's default. */
|
|
65
|
+
readonly baseUrl?: string;
|
|
45
66
|
readonly headers?: Readonly<Record<string, string>>;
|
|
46
67
|
readonly weight?: number;
|
|
47
68
|
/** The most requests this provider carries at once in this process, across its keys. */
|
|
48
69
|
readonly maxConcurrency?: number;
|
|
70
|
+
/** The `max_tokens` sent to a provider that requires one (Anthropic) when the request sets none. Default 4096. */
|
|
71
|
+
readonly defaultMaxTokens?: number;
|
|
72
|
+
readonly budget?: ProviderBudget;
|
|
49
73
|
readonly keys: readonly KeyConfig[];
|
|
50
74
|
readonly models: readonly ModelConfig[];
|
|
51
75
|
}
|
|
76
|
+
/** One candidate of a profile: a model by id or alias, on one provider or on any that serves it, weighted within its tier. */
|
|
77
|
+
export interface CandidateRef {
|
|
78
|
+
readonly model: string;
|
|
79
|
+
readonly provider?: string;
|
|
80
|
+
readonly weight?: number;
|
|
81
|
+
}
|
|
52
82
|
/** How one call picks and falls back between models. */
|
|
53
83
|
export interface ModelPolicy {
|
|
54
84
|
/** The preferred model (id or alias); the request's own `model` is used when this is absent. */
|
|
55
85
|
readonly model?: string;
|
|
56
86
|
/** Models tried, in order, once the preferred model has no usable candidate. */
|
|
57
87
|
readonly fallbacks?: readonly string[];
|
|
58
|
-
/** False pins the call to the preferred model; the fallbacks are ignored. Default true. */
|
|
88
|
+
/** False pins the call to the preferred model (or the first tier of `candidates`); the fallbacks are ignored. Default true. */
|
|
59
89
|
readonly allowFallback?: boolean;
|
|
60
90
|
/** The most attempts across keys, providers and models. Default 4. */
|
|
61
91
|
readonly maxAttempts?: number;
|
|
@@ -71,6 +101,16 @@ export interface ModelPolicy {
|
|
|
71
101
|
readonly providers?: readonly string[];
|
|
72
102
|
/** Capabilities required beyond those the request shape implies. */
|
|
73
103
|
readonly require?: readonly Capability[];
|
|
104
|
+
/**
|
|
105
|
+
* The ordered candidates of a profile, in place of `model` and `fallbacks`: each entry is a tier, tried in order; an
|
|
106
|
+
* entry that is a list is one tier whose candidates share the load by weight.
|
|
107
|
+
*/
|
|
108
|
+
readonly candidates?: readonly (CandidateRef | readonly CandidateRef[])[];
|
|
109
|
+
/**
|
|
110
|
+
* True pins the profile to one model identity (an embeddings space): every candidate must resolve to models of one
|
|
111
|
+
* `identity` and `dimensions`, so keys and providers rotate but the model never changes, and a call cannot name another.
|
|
112
|
+
*/
|
|
113
|
+
readonly pinned?: boolean;
|
|
74
114
|
}
|
|
75
115
|
/** The thresholds of a circuit breaker. */
|
|
76
116
|
export interface BreakerConfig {
|
|
@@ -81,7 +121,7 @@ export interface BreakerConfig {
|
|
|
81
121
|
/** The cooldown never grows past this. */
|
|
82
122
|
readonly maxCooldownMs?: number;
|
|
83
123
|
}
|
|
84
|
-
/** The retry and
|
|
124
|
+
/** The retry, breaker and blacklist behaviour of the pool. */
|
|
85
125
|
export interface ResilienceConfig {
|
|
86
126
|
/** The breaker of each key. Defaults: 3 failures, 15000 ms, 300000 ms. */
|
|
87
127
|
readonly keyBreaker?: BreakerConfig;
|
|
@@ -93,10 +133,15 @@ export interface ResilienceConfig {
|
|
|
93
133
|
readonly factor?: number;
|
|
94
134
|
readonly maxMs?: number;
|
|
95
135
|
};
|
|
96
|
-
/** How long a key that answered 401/403 stays
|
|
136
|
+
/** How long a key that answered 401/403 stays blacklisted; null keeps it out until its secret changes or `reset()`. Default null. */
|
|
97
137
|
readonly authDisableMs?: number | null;
|
|
98
138
|
/** The cooldown of a key and model after a 429 without a retry-after header. Default 10000. */
|
|
99
139
|
readonly rateLimitCooldownMs?: number;
|
|
140
|
+
/**
|
|
141
|
+
* How long a key whose quota or credit is exhausted stays blacklisted when the provider's answer names no reset
|
|
142
|
+
* window; a new secret for the key lifts it at once. Default 21600000 (6 hours).
|
|
143
|
+
*/
|
|
144
|
+
readonly quotaBlacklistMs?: number;
|
|
100
145
|
}
|
|
101
146
|
/** How candidates of equal health are ranked. */
|
|
102
147
|
export interface SelectionConfig {
|
|
@@ -129,7 +174,10 @@ export interface PoolConfig<Profiles extends string = string> {
|
|
|
129
174
|
readonly resilience?: ResilienceConfig;
|
|
130
175
|
readonly selection?: SelectionConfig;
|
|
131
176
|
readonly telemetry?: TelemetryConfig;
|
|
132
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* Where breaker, blacklist, headroom and budget state lives; in memory by default. A durable store
|
|
179
|
+
* (`openSqliteStateStore`, `redisStateStore`) keeps blacklisted keys out across a restart and shares them across processes.
|
|
180
|
+
*/
|
|
133
181
|
readonly store?: StateStore;
|
|
134
182
|
/** The prefix of every store key, so several pools can share one store. Default `ai-router`. */
|
|
135
183
|
readonly storeNamespace?: string;
|
|
@@ -160,7 +208,7 @@ export interface ChatMessage {
|
|
|
160
208
|
readonly tool_calls?: readonly ToolCall[];
|
|
161
209
|
readonly tool_call_id?: string;
|
|
162
210
|
}
|
|
163
|
-
/** An OpenAI chat-completions request; unknown fields pass through to
|
|
211
|
+
/** An OpenAI chat-completions request; unknown fields pass through to an OpenAI-compatible provider. */
|
|
164
212
|
export interface ChatCompletionRequest {
|
|
165
213
|
/** The model (id or alias); a policy's `model` takes precedence. */
|
|
166
214
|
readonly model?: string;
|
|
@@ -174,6 +222,7 @@ export interface ChatCompletionRequest {
|
|
|
174
222
|
readonly user?: string;
|
|
175
223
|
readonly tools?: readonly unknown[];
|
|
176
224
|
readonly tool_choice?: unknown;
|
|
225
|
+
readonly parallel_tool_calls?: boolean;
|
|
177
226
|
readonly response_format?: {
|
|
178
227
|
readonly type: string;
|
|
179
228
|
readonly [key: string]: unknown;
|
|
@@ -188,6 +237,7 @@ export interface Usage {
|
|
|
188
237
|
readonly prompt_tokens: number;
|
|
189
238
|
readonly completion_tokens?: number;
|
|
190
239
|
readonly total_tokens: number;
|
|
240
|
+
readonly [key: string]: unknown;
|
|
191
241
|
}
|
|
192
242
|
/** Where a successful call went and what it cost. */
|
|
193
243
|
export interface RoutingInfo {
|
|
@@ -201,26 +251,33 @@ export interface RoutingInfo {
|
|
|
201
251
|
/** Computed from usage and the model's cost; absent when either is unknown. */
|
|
202
252
|
readonly cost?: number;
|
|
203
253
|
}
|
|
204
|
-
/**
|
|
205
|
-
export interface
|
|
254
|
+
/** The assistant message of one choice. */
|
|
255
|
+
export interface ChatChoiceMessage {
|
|
256
|
+
readonly role: "assistant";
|
|
257
|
+
readonly content: string | null;
|
|
258
|
+
readonly tool_calls?: readonly ToolCall[];
|
|
259
|
+
readonly [key: string]: unknown;
|
|
260
|
+
}
|
|
261
|
+
/** One choice of a chat completion. */
|
|
262
|
+
export interface ChatChoice {
|
|
263
|
+
readonly index: number;
|
|
264
|
+
readonly message: ChatChoiceMessage;
|
|
265
|
+
readonly finish_reason: string | null;
|
|
266
|
+
}
|
|
267
|
+
/** An OpenAI chat completion as a provider answers it. */
|
|
268
|
+
export interface ChatCompletionBody {
|
|
206
269
|
readonly id: string;
|
|
207
270
|
readonly object: "chat.completion";
|
|
208
271
|
readonly created: number;
|
|
209
272
|
readonly model: string;
|
|
210
|
-
readonly choices: readonly
|
|
211
|
-
readonly index: number;
|
|
212
|
-
readonly message: {
|
|
213
|
-
readonly role: "assistant";
|
|
214
|
-
readonly content: string | null;
|
|
215
|
-
readonly tool_calls?: readonly ToolCall[];
|
|
216
|
-
readonly [key: string]: unknown;
|
|
217
|
-
};
|
|
218
|
-
readonly finish_reason: string | null;
|
|
219
|
-
}[];
|
|
273
|
+
readonly choices: readonly ChatChoice[];
|
|
220
274
|
readonly usage?: Usage;
|
|
221
|
-
readonly routing: RoutingInfo;
|
|
222
275
|
readonly [key: string]: unknown;
|
|
223
276
|
}
|
|
277
|
+
/** An OpenAI chat completion, with the routing that produced it. */
|
|
278
|
+
export interface ChatCompletion extends ChatCompletionBody {
|
|
279
|
+
readonly routing: RoutingInfo;
|
|
280
|
+
}
|
|
224
281
|
/** One tool-call fragment of a streamed delta. */
|
|
225
282
|
export interface ToolCallDelta {
|
|
226
283
|
readonly index: number;
|
|
@@ -263,16 +320,21 @@ export interface EmbeddingsRequest {
|
|
|
263
320
|
readonly user?: string;
|
|
264
321
|
readonly [key: string]: unknown;
|
|
265
322
|
}
|
|
266
|
-
/**
|
|
267
|
-
export interface
|
|
323
|
+
/** One vector of an embeddings response. */
|
|
324
|
+
export interface EmbeddingData {
|
|
325
|
+
readonly object: "embedding";
|
|
326
|
+
readonly index: number;
|
|
327
|
+
readonly embedding: readonly number[] | string;
|
|
328
|
+
}
|
|
329
|
+
/** An OpenAI embeddings response as a provider answers it. */
|
|
330
|
+
export interface EmbeddingsBody {
|
|
268
331
|
readonly object: "list";
|
|
269
|
-
readonly data: readonly
|
|
270
|
-
readonly object: "embedding";
|
|
271
|
-
readonly index: number;
|
|
272
|
-
readonly embedding: readonly number[] | string;
|
|
273
|
-
}[];
|
|
332
|
+
readonly data: readonly EmbeddingData[];
|
|
274
333
|
readonly model: string;
|
|
275
334
|
readonly usage?: Usage;
|
|
335
|
+
}
|
|
336
|
+
/** An OpenAI embeddings response, with the routing that produced it. */
|
|
337
|
+
export interface EmbeddingsResponse extends EmbeddingsBody {
|
|
276
338
|
readonly routing: RoutingInfo;
|
|
277
339
|
}
|
|
278
340
|
/** The per-call options of `chat`, `stream` and `embeddings`. */
|
|
@@ -283,11 +345,11 @@ export interface CallOptions<Profiles extends string = string> {
|
|
|
283
345
|
readonly signal?: AbortSignal;
|
|
284
346
|
/** The id sent as `x-request-id` and `idempotency-key` on every attempt; generated when absent. */
|
|
285
347
|
readonly requestId?: string;
|
|
286
|
-
/** Extra headers for this call; they cannot replace
|
|
348
|
+
/** Extra headers for this call; they cannot replace the provider's auth header. */
|
|
287
349
|
readonly headers?: Readonly<Record<string, string>>;
|
|
288
350
|
}
|
|
289
351
|
/** Why an attempt or a call failed. */
|
|
290
|
-
export type ErrorClass = "rate_limit" | "auth" | "transient" | "timeout" | "network" | "stream" | "invalid_request" | "cancelled" | "deadline" | "no_candidate" | "config";
|
|
352
|
+
export type ErrorClass = "rate_limit" | "quota" | "auth" | "transient" | "timeout" | "network" | "stream" | "invalid_request" | "cancelled" | "deadline" | "no_candidate" | "config";
|
|
291
353
|
/** The operation a call performs. */
|
|
292
354
|
export type Operation = "chat" | "stream" | "embeddings";
|
|
293
355
|
/** The state of a breaker as health reports it; `half-open` is an open breaker past its cooldown. */
|
|
@@ -321,14 +383,27 @@ export interface BreakerEvent {
|
|
|
321
383
|
readonly from: BreakerState;
|
|
322
384
|
readonly to: BreakerState;
|
|
323
385
|
}
|
|
324
|
-
/**
|
|
386
|
+
/** Why a key is blacklisted: its secret was refused (`auth`), it has none (`missing`), or its quota or credit is spent (`quota`). */
|
|
387
|
+
export type DisabledReason = "auth" | "missing" | "quota";
|
|
388
|
+
/** Emitted when a key is blacklisted. It names the key by alias and the provider's error code, never the secret. */
|
|
325
389
|
export interface KeyDisabledEvent {
|
|
326
390
|
readonly type: "key-disabled";
|
|
327
391
|
readonly provider: string;
|
|
328
392
|
readonly key: string;
|
|
329
|
-
readonly reason:
|
|
330
|
-
/** Epoch milliseconds the key returns, or null until `reset()`. */
|
|
393
|
+
readonly reason: DisabledReason;
|
|
394
|
+
/** Epoch milliseconds the key returns, or null until its secret changes or `reset()`. */
|
|
331
395
|
readonly until: number | null;
|
|
396
|
+
/** The provider's error code or type (`insufficient_quota`, `billing_error`, ...), when it sent one. */
|
|
397
|
+
readonly code?: string;
|
|
398
|
+
}
|
|
399
|
+
/** Emitted when a blacklisted key comes back because its secret changed. */
|
|
400
|
+
export interface KeyRestoredEvent {
|
|
401
|
+
readonly type: "key-restored";
|
|
402
|
+
readonly provider: string;
|
|
403
|
+
readonly key: string;
|
|
404
|
+
/** Why it was out. */
|
|
405
|
+
readonly was: DisabledReason;
|
|
406
|
+
readonly reason: "secret_changed";
|
|
332
407
|
}
|
|
333
408
|
/** Emitted once per call when it settles. */
|
|
334
409
|
export interface RequestEvent {
|
|
@@ -348,6 +423,7 @@ export interface RouterEvents {
|
|
|
348
423
|
readonly attempt: AttemptEvent;
|
|
349
424
|
readonly breaker: BreakerEvent;
|
|
350
425
|
readonly "key-disabled": KeyDisabledEvent;
|
|
426
|
+
readonly "key-restored": KeyRestoredEvent;
|
|
351
427
|
readonly request: RequestEvent;
|
|
352
428
|
}
|
|
353
429
|
/** The health of one key and model. */
|
|
@@ -365,16 +441,24 @@ export interface KeyHealth {
|
|
|
365
441
|
readonly breaker: BreakerState;
|
|
366
442
|
readonly failures: number;
|
|
367
443
|
readonly disabled: boolean;
|
|
368
|
-
readonly disabledReason:
|
|
444
|
+
readonly disabledReason: DisabledReason | null;
|
|
445
|
+
/** Epoch milliseconds the blacklist ends; null while it lasts until the secret changes, and when the key is not disabled. */
|
|
446
|
+
readonly disabledUntil: number | null;
|
|
369
447
|
readonly inflight: number;
|
|
370
448
|
readonly models: readonly LaneHealth[];
|
|
371
449
|
}
|
|
372
450
|
/** The health of one provider. */
|
|
373
451
|
export interface ProviderHealth {
|
|
374
452
|
readonly id: string;
|
|
453
|
+
readonly kind: ProviderKind;
|
|
375
454
|
readonly breaker: BreakerState;
|
|
376
455
|
readonly failures: number;
|
|
377
456
|
readonly inflight: number;
|
|
457
|
+
/** This UTC day's and month's spend against `budget`, when the provider has one. */
|
|
458
|
+
readonly spend?: {
|
|
459
|
+
readonly day: number;
|
|
460
|
+
readonly month: number;
|
|
461
|
+
};
|
|
378
462
|
readonly keys: readonly KeyHealth[];
|
|
379
463
|
}
|
|
380
464
|
/** A snapshot of the pool's health. */
|
|
@@ -415,13 +499,13 @@ export interface AiRouter<Profiles extends string = string> {
|
|
|
415
499
|
stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
|
|
416
500
|
/** Sends an embeddings request through the pool. */
|
|
417
501
|
embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
|
|
418
|
-
/** Reads every breaker,
|
|
502
|
+
/** Reads every breaker, blacklisted key, headroom and budget from the store. */
|
|
419
503
|
health(): Promise<HealthSnapshot>;
|
|
420
504
|
/** Returns this process's counters. */
|
|
421
505
|
stats(): StatsSnapshot;
|
|
422
506
|
/** Subscribes to an event; returns the unsubscribe function. */
|
|
423
507
|
on<E extends keyof RouterEvents>(event: E, handler: (payload: RouterEvents[E]) => void): () => void;
|
|
424
|
-
/** Closes breakers,
|
|
508
|
+
/** Closes breakers, lifts blacklists and clears headroom for the target (everything by default). */
|
|
425
509
|
reset(target?: ResetTarget): Promise<void>;
|
|
426
510
|
/** Lists the pool's profiles and models by name, with no key: what a caller may ask for. */
|
|
427
511
|
catalog(): RouterCatalog;
|
|
@@ -438,10 +522,14 @@ export interface CatalogModel {
|
|
|
438
522
|
/** One named profile as the catalog lists it. */
|
|
439
523
|
export interface CatalogProfile {
|
|
440
524
|
readonly name: string;
|
|
441
|
-
/** The profile's preferred model, else the defaults'; absent when neither names one. */
|
|
525
|
+
/** The profile's preferred model (its first candidate), else the defaults'; absent when neither names one. */
|
|
442
526
|
readonly model?: string;
|
|
443
|
-
/** The fallbacks the profile tries (its own, else the defaults'); empty when fallback is off. */
|
|
527
|
+
/** The fallbacks the profile tries (its later candidates, or its own fallbacks, else the defaults'); empty when fallback is off. */
|
|
444
528
|
readonly fallbacks: readonly string[];
|
|
529
|
+
/** True when the profile is pinned to one model identity. */
|
|
530
|
+
readonly pinned: boolean;
|
|
531
|
+
/** The operations its first model serves: `chat`, `embeddings` or both. */
|
|
532
|
+
readonly serves: readonly ("chat" | "embeddings")[];
|
|
445
533
|
}
|
|
446
534
|
/** The names a caller may route by: the defaults' fallbacks, every profile and every model. */
|
|
447
535
|
export interface RouterCatalog {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ProviderConfig } from "../types.js";
|
|
2
|
+
import type { Wire } from "./wire.js";
|
|
3
|
+
/**
|
|
4
|
+
* The Anthropic Messages API (`POST /v1/messages`, read 2026-10-31 at platform.claude.com/docs/en/api/messages and
|
|
5
|
+
* /build-with-claude/streaming) behind the OpenAI chat-completions shape. docs/design.md, "The Anthropic adapter", maps
|
|
6
|
+
* every field; what has no counterpart is refused (`n` above 1) or dropped (seed, penalties, logprobs).
|
|
7
|
+
*/
|
|
8
|
+
/** The `anthropic-version` sent unless the provider's headers set one. */
|
|
9
|
+
export declare const ANTHROPIC_VERSION = "2023-06-01";
|
|
10
|
+
/** The `max_tokens` sent when neither the request nor the provider's `defaultMaxTokens` gives one: Anthropic requires it. */
|
|
11
|
+
export declare const ANTHROPIC_DEFAULT_MAX_TOKENS = 4096;
|
|
12
|
+
/** Builds the Anthropic Messages body of an OpenAI chat request. */
|
|
13
|
+
export declare function toAnthropicRequest(body: Record<string, unknown>, model: string, stream: boolean, provider: ProviderConfig): Record<string, unknown>;
|
|
14
|
+
/** Anthropic's stop reason as OpenAI's finish reason. */
|
|
15
|
+
export declare function finishReason(reason: unknown): string | null;
|
|
16
|
+
/** Anthropic usage as OpenAI usage: the prompt counts the cache reads and writes, which Anthropic lists apart. */
|
|
17
|
+
export declare function usageOf(usage: unknown, outputOverride?: number): Record<string, unknown> | undefined;
|
|
18
|
+
/** An Anthropic message as an OpenAI `chat.completion`. */
|
|
19
|
+
export declare function fromAnthropicMessage(json: Record<string, unknown>): Record<string, unknown>;
|
|
20
|
+
/** The Anthropic Messages API behind the OpenAI shape: `x-api-key` auth, `/messages`, and its SSE events as OpenAI chunks. */
|
|
21
|
+
export declare const anthropicWire: Wire;
|