@starci/ai-router 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +12 -3
package/dist/classify.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
function isRecord(value) {
|
|
2
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3
|
+
}
|
|
4
|
+
function text(value) {
|
|
5
|
+
if (typeof value === "string")
|
|
6
|
+
return value;
|
|
7
|
+
if (typeof value === "number")
|
|
8
|
+
return String(value);
|
|
9
|
+
return "";
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Reads a provider error body into its fields. It knows the OpenAI `{ error: { message, type, code, param } }`, the
|
|
13
|
+
* Anthropic `{ type: "error", error: { type, message, details } }`, OpenRouter's `{ error: { code: <status>, message,
|
|
14
|
+
* metadata } }`, Gemini's object or one-element list `[{ error: { code, message, status } }]` and DashScope's top-level
|
|
15
|
+
* `{ code, message }`. Anything else leaves the fields empty and the raw text as the message.
|
|
16
|
+
*/
|
|
17
|
+
export function readProviderError(body, status) {
|
|
18
|
+
const root = Array.isArray(body) ? body[0] : body;
|
|
19
|
+
const error = isRecord(root) ? (isRecord(root.error) ? root.error : typeof root.code === "string" ? root : undefined) : undefined;
|
|
20
|
+
if (error === undefined)
|
|
21
|
+
return { status, type: "", code: "", detail: "", message: typeof body === "string" ? body : "" };
|
|
22
|
+
const details = isRecord(error.details) ? error.details : {};
|
|
23
|
+
const metadata = isRecord(error.metadata) ? error.metadata : {};
|
|
24
|
+
const numeric = typeof error.code === "number" ? error.code : undefined;
|
|
25
|
+
return {
|
|
26
|
+
status: status ?? numeric,
|
|
27
|
+
type: text(error.type) || text(error.status),
|
|
28
|
+
code: numeric === undefined ? text(error.code) : "",
|
|
29
|
+
detail: text(details.error_code) || text(metadata.limit_source) || text(metadata.error_type),
|
|
30
|
+
message: text(error.message),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/** Codes and types that mean the key's quota, credit or billing is spent: another key may serve, this one will not soon. */
|
|
34
|
+
const QUOTA_CODES = new Set([
|
|
35
|
+
"insufficient_quota",
|
|
36
|
+
"credit_balance_exhausted",
|
|
37
|
+
"organization_spend_limit_exceeded",
|
|
38
|
+
"project_spend_limit_exceeded",
|
|
39
|
+
"organization_usage_limit_exceeded",
|
|
40
|
+
"billing_error",
|
|
41
|
+
"billing_not_active",
|
|
42
|
+
"payment_required",
|
|
43
|
+
"quota_exceeded",
|
|
44
|
+
"insufficient_balance",
|
|
45
|
+
"arrearage",
|
|
46
|
+
"allocationquota.freetieronly",
|
|
47
|
+
"enforced_spend_limit_reached",
|
|
48
|
+
"openrouter_key_limit",
|
|
49
|
+
"openrouter_credits",
|
|
50
|
+
]);
|
|
51
|
+
/** Messages that mean the same where a provider sends no code (DeepSeek, Anthropic's own usage limits, OpenRouter's free tier). */
|
|
52
|
+
const QUOTA_MESSAGE = /credit balance is too low|insufficient (?:credits|balance)|reached your specified (?:workspace )?api usage limits|exceeded your current quota|credit balance is depleted|free-models-per-day|requests per day|per day limit|perday/i;
|
|
53
|
+
/** Codes and types that mean the secret itself is refused: wrong, revoked, expired or leaked. */
|
|
54
|
+
const AUTH_CODES = new Set(["invalid_api_key", "authentication_error", "authentication", "invalidapikey", "unauthenticated", "permission_error", "permission_denied", "accessdenied", "model.accessdenied", "invalid_authentication"]);
|
|
55
|
+
const AUTH_MESSAGE = /reported as leaked|incorrect api key|invalid api key|api key (?:not valid|is invalid|expired)|invalid authentication/i;
|
|
56
|
+
/** Codes that mean the request was refused for what it is, not for the key: no retry. */
|
|
57
|
+
const INVALID_CODES = new Set(["context_length_exceeded", "invalid_request", "invalid_request_error", "invalidparameter", "datainspectionfailed", "modelnotfound", "model_not_found", "not_found", "not_found_error", "parameter_unknown", "request_too_large"]);
|
|
58
|
+
/** Codes of a rate limit: a short cooldown of the key and model. */
|
|
59
|
+
const RATE_CODES = new Set(["rate_limit_exceeded", "rate_limit_error", "too_many_requests", "slow_down", "throttling", "throttling.ratequota", "throttling.allocationquota", "throttling.burstrate", "openrouter_in_flight_budget", "resource_exhausted"]);
|
|
60
|
+
/** Codes of a provider that is overloaded or failing: transient. */
|
|
61
|
+
const TRANSIENT_CODES = new Set(["overloaded_error", "api_error", "server_error", "service_unavailable", "service_unavailable_error", "server_is_overloaded", "internalerror", "timeout_error", "deadline_exceeded"]);
|
|
62
|
+
/** The quota window a provider's answer implies, beyond the default TTL (docs/design.md, the blacklist lifecycle). */
|
|
63
|
+
function windowOf(kind, info, codes) {
|
|
64
|
+
const has = (code) => codes.includes(code);
|
|
65
|
+
// Anthropic's tier spend cap and OpenAI's monthly usage and spend limits reset at the start of the next UTC month.
|
|
66
|
+
if (has("enforced_spend_limit_reached") || has("organization_usage_limit_exceeded") || has("organization_spend_limit_exceeded") || has("project_spend_limit_exceeded"))
|
|
67
|
+
return { kind: "month-utc" };
|
|
68
|
+
// Gemini's requests-per-day reset at midnight Pacific time.
|
|
69
|
+
if (kind === "gemini" && (has("quota_exceeded") || /per ?day/i.test(info.message)))
|
|
70
|
+
return { kind: "day", timeZone: "America/Los_Angeles" };
|
|
71
|
+
// OpenRouter's free-model daily cap resets at 00:00 UTC.
|
|
72
|
+
if (kind === "openrouter" && /free-models-per-day|per day/i.test(info.message))
|
|
73
|
+
return { kind: "day", timeZone: "UTC" };
|
|
74
|
+
// Groq's request limit is per day (x-ratelimit-*-requests); its reset header carries the exact time instead.
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Classifies a failed attempt from its status and error body, per provider kind (docs/design.md, the classification
|
|
79
|
+
* table). Order: a refused secret, then a spent quota (which some providers send as 429 or 400), then a rate limit,
|
|
80
|
+
* then the status alone. Classes: `auth` and `quota` blacklist the key, `rate_limit` cools it down, `transient` feeds
|
|
81
|
+
* the breakers, `invalid_request` is the caller's and is not retried.
|
|
82
|
+
*/
|
|
83
|
+
export function classifyFailure(kind, status, body) {
|
|
84
|
+
const info = readProviderError(body, status);
|
|
85
|
+
const codes = [info.detail, info.code, info.type].map((code) => code.toLowerCase()).filter((code) => code !== "");
|
|
86
|
+
const code = [info.detail, info.code, info.type].find((value) => value !== "");
|
|
87
|
+
const has = (set) => codes.some((value) => set.has(value));
|
|
88
|
+
const effective = info.status;
|
|
89
|
+
const out = (errorClass, window) => ({ errorClass, code, ...(window === undefined ? {} : { window }) });
|
|
90
|
+
// OpenRouter's in-flight budget is a 402 that clears by itself: a short wait, not a spent quota.
|
|
91
|
+
if (codes.includes("openrouter_in_flight_budget"))
|
|
92
|
+
return out("rate_limit");
|
|
93
|
+
if (effective === 401 || has(AUTH_CODES) || AUTH_MESSAGE.test(info.message)) {
|
|
94
|
+
// OpenRouter answers 403 for a moderation or guardrail block: the request's fault, not the key's.
|
|
95
|
+
if (kind === "openrouter" && effective === 403)
|
|
96
|
+
return out("invalid_request");
|
|
97
|
+
return out("auth");
|
|
98
|
+
}
|
|
99
|
+
// Gemini words its per-minute 429 like a spent quota; only a daily limit is one (the project's RPD, until midnight Pacific).
|
|
100
|
+
if (kind === "gemini" && effective === 429 && !codes.includes("quota_exceeded") && !/per ?day/i.test(info.message))
|
|
101
|
+
return out("rate_limit");
|
|
102
|
+
const billingPrecondition = codes.includes("failed_precondition") && /billing/i.test(info.message);
|
|
103
|
+
if (effective === 402 || has(QUOTA_CODES) || QUOTA_MESSAGE.test(info.message) || billingPrecondition)
|
|
104
|
+
return out("quota", windowOf(kind, info, codes));
|
|
105
|
+
if (effective === 403)
|
|
106
|
+
return out(kind === "openrouter" ? "invalid_request" : "auth");
|
|
107
|
+
if (effective === 429 || has(RATE_CODES))
|
|
108
|
+
return out("rate_limit");
|
|
109
|
+
if (effective !== undefined) {
|
|
110
|
+
// Anthropic 529 overloaded, Groq 498 flex capacity, 408/409/425 and every 5xx are worth another lane.
|
|
111
|
+
if (effective === 408 || effective === 409 || effective === 425 || effective === 498 || effective >= 500)
|
|
112
|
+
return out("transient");
|
|
113
|
+
if (effective >= 400)
|
|
114
|
+
return out("invalid_request");
|
|
115
|
+
}
|
|
116
|
+
if (has(TRANSIENT_CODES))
|
|
117
|
+
return out("transient");
|
|
118
|
+
if (has(INVALID_CODES) || codes.some((value) => /invalid|context_length|not_found/.test(value)))
|
|
119
|
+
return out("invalid_request");
|
|
120
|
+
return out("transient");
|
|
121
|
+
}
|
|
122
|
+
/** The parts of a wall-clock time in a time zone. */
|
|
123
|
+
function wallClock(time, timeZone) {
|
|
124
|
+
const parts = new Intl.DateTimeFormat("en-US", { timeZone, hourCycle: "h23", hour: "numeric", minute: "numeric", second: "numeric" }).formatToParts(new Date(time));
|
|
125
|
+
const part = (type) => Number(parts.find((entry) => entry.type === type)?.value ?? 0);
|
|
126
|
+
return { hour: part("hour"), minute: part("minute"), second: part("second") };
|
|
127
|
+
}
|
|
128
|
+
/** The next midnight in a time zone after `now` (to the second; a DST day is corrected once). */
|
|
129
|
+
export function nextMidnight(now, timeZone) {
|
|
130
|
+
const elapsed = (clock) => ((clock.hour * 60 + clock.minute) * 60 + clock.second) * 1000;
|
|
131
|
+
let candidate = now - (now % 1000) + 86_400_000 - elapsed(wallClock(now, timeZone));
|
|
132
|
+
const off = elapsed(wallClock(candidate, timeZone));
|
|
133
|
+
// On a DST change the candidate lands an hour off midnight: move it back (23:00) or forward (01:00).
|
|
134
|
+
if (off !== 0)
|
|
135
|
+
candidate += off > 43_200_000 ? 86_400_000 - off : -off;
|
|
136
|
+
return candidate;
|
|
137
|
+
}
|
|
138
|
+
/** When a quota window ends after `now`. */
|
|
139
|
+
export function windowEnd(window, now) {
|
|
140
|
+
if (window.kind === "month-utc") {
|
|
141
|
+
const date = new Date(now);
|
|
142
|
+
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1);
|
|
143
|
+
}
|
|
144
|
+
return nextMidnight(now, window.timeZone);
|
|
145
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CallOptions, Capability, ModelConfig, PoolConfig, ProviderConfig, RouterCatalog } from "./types.js";
|
|
2
2
|
/** The capabilities of a model that declares none. */
|
|
3
3
|
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
|
+
/** How long a spent quota keeps its key out when the provider names no window: 6 hours. */
|
|
5
|
+
export declare const DEFAULT_QUOTA_BLACKLIST_MS: number;
|
|
4
6
|
/** A breaker's thresholds with every default applied. */
|
|
5
7
|
export interface BreakerSettings {
|
|
6
8
|
readonly failureThreshold: number;
|
|
@@ -18,6 +20,7 @@ export interface Settings {
|
|
|
18
20
|
};
|
|
19
21
|
readonly authDisableMs: number | null;
|
|
20
22
|
readonly rateLimitCooldownMs: number;
|
|
23
|
+
readonly quotaBlacklistMs: number;
|
|
21
24
|
readonly latencyTolerance: number;
|
|
22
25
|
readonly latencySlackMs: number;
|
|
23
26
|
readonly ewmaAlpha: number;
|
|
@@ -25,10 +28,19 @@ export interface Settings {
|
|
|
25
28
|
readonly includeContent: boolean;
|
|
26
29
|
readonly namespace: string;
|
|
27
30
|
}
|
|
28
|
-
/** One
|
|
31
|
+
/** One candidate of a resolved policy: a model name, the provider it is held to (null: any), and its weight in its tier. */
|
|
32
|
+
export interface ResolvedRef {
|
|
33
|
+
readonly model: string;
|
|
34
|
+
readonly provider: string | null;
|
|
35
|
+
readonly weight: number;
|
|
36
|
+
}
|
|
37
|
+
/** One call's policy with every default applied and its candidates expanded into tiers. */
|
|
29
38
|
export interface ResolvedPolicy {
|
|
30
|
-
/** The
|
|
39
|
+
/** The tiers, tried in order; the candidates of one tier share the load by weight. */
|
|
40
|
+
readonly tiers: readonly (readonly ResolvedRef[])[];
|
|
41
|
+
/** Every model name of the tiers, in order and without repeats: what a message names. */
|
|
31
42
|
readonly models: readonly string[];
|
|
43
|
+
readonly pinned: boolean;
|
|
32
44
|
readonly maxAttempts: number;
|
|
33
45
|
readonly timeoutMs: number;
|
|
34
46
|
readonly deadlineMs: number;
|
|
@@ -38,11 +50,24 @@ export interface ResolvedPolicy {
|
|
|
38
50
|
}
|
|
39
51
|
/** True when the name is the model's id or one of its aliases. */
|
|
40
52
|
export declare function modelMatches(model: ModelConfig, name: string): boolean;
|
|
41
|
-
/**
|
|
53
|
+
/** The identity of a model: its declared `identity`, else its id. */
|
|
54
|
+
export declare function identityOf(model: ModelConfig): string;
|
|
55
|
+
/** Every (provider, model) a candidate resolves to. */
|
|
56
|
+
export declare function matchesOf(config: PoolConfig, ref: {
|
|
57
|
+
readonly model: string;
|
|
58
|
+
readonly provider?: string | null;
|
|
59
|
+
}): {
|
|
60
|
+
provider: ProviderConfig;
|
|
61
|
+
model: ModelConfig;
|
|
62
|
+
}[];
|
|
63
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL, kind or number. */
|
|
42
64
|
export declare function validateConfig(config: PoolConfig): void;
|
|
43
65
|
/** Applies the defaults of every pool-wide setting. */
|
|
44
66
|
export declare function resolveSettings(config: PoolConfig): Settings;
|
|
45
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the candidates
|
|
69
|
+
* into tiers. A call's own `model` wins over the profile's candidates; a pinned profile refuses another model.
|
|
70
|
+
*/
|
|
46
71
|
export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
|
|
47
72
|
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
48
73
|
export declare function buildCatalog(config: PoolConfig): RouterCatalog;
|
package/dist/config.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { AiRouterError } from "./errors.js";
|
|
2
|
+
import { baseUrlOf, presetOf, PROVIDER_PRESETS } from "./providers.js";
|
|
2
3
|
/** The capabilities of a model that declares none. */
|
|
3
4
|
export const DEFAULT_CAPABILITIES = ["chat", "streaming", "tools", "json"];
|
|
5
|
+
/** How long a spent quota keeps its key out when the provider names no window: 6 hours. */
|
|
6
|
+
export const DEFAULT_QUOTA_BLACKLIST_MS = 6 * 3_600_000;
|
|
4
7
|
function configError(message) {
|
|
5
8
|
return new AiRouterError({ errorClass: "config", message });
|
|
6
9
|
}
|
|
@@ -14,14 +17,57 @@ function positive(value, name, allowZero = false) {
|
|
|
14
17
|
export function modelMatches(model, name) {
|
|
15
18
|
return model.id === name || (model.aliases?.includes(name) ?? false);
|
|
16
19
|
}
|
|
20
|
+
/** The identity of a model: its declared `identity`, else its id. */
|
|
21
|
+
export function identityOf(model) {
|
|
22
|
+
return model.identity ?? model.id;
|
|
23
|
+
}
|
|
24
|
+
/** Every (provider, model) a candidate resolves to. */
|
|
25
|
+
export function matchesOf(config, ref) {
|
|
26
|
+
const out = [];
|
|
27
|
+
for (const provider of config.providers) {
|
|
28
|
+
if (ref.provider !== undefined && ref.provider !== null && provider.id !== ref.provider)
|
|
29
|
+
continue;
|
|
30
|
+
for (const model of provider.models)
|
|
31
|
+
if (modelMatches(model, ref.model))
|
|
32
|
+
out.push({ provider, model });
|
|
33
|
+
}
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
17
36
|
function knownModel(config, name) {
|
|
18
|
-
return config
|
|
37
|
+
return matchesOf(config, { model: name }).length > 0;
|
|
38
|
+
}
|
|
39
|
+
/** The candidates of a policy as tiers of refs, or null when it lists none. */
|
|
40
|
+
function candidateTiers(policy) {
|
|
41
|
+
if (policy.candidates === undefined)
|
|
42
|
+
return null;
|
|
43
|
+
return policy.candidates.map((entry) => (isRefList(entry) ? [...entry] : [entry]));
|
|
44
|
+
}
|
|
45
|
+
function isRefList(entry) {
|
|
46
|
+
return Array.isArray(entry);
|
|
19
47
|
}
|
|
20
48
|
function validatePolicy(config, policy, where) {
|
|
21
49
|
for (const name of [policy.model, ...(policy.fallbacks ?? [])]) {
|
|
22
50
|
if (name !== undefined && !knownModel(config, name))
|
|
23
51
|
throw configError(`${where} names model "${name}", which no provider serves`);
|
|
24
52
|
}
|
|
53
|
+
const tiers = candidateTiers(policy);
|
|
54
|
+
if (tiers !== null) {
|
|
55
|
+
if (policy.model !== undefined || policy.fallbacks !== undefined)
|
|
56
|
+
throw configError(`${where} sets candidates and model or fallbacks: give one or the other`);
|
|
57
|
+
if (tiers.length === 0)
|
|
58
|
+
throw configError(`${where}.candidates is empty`);
|
|
59
|
+
tiers.forEach((tier, index) => {
|
|
60
|
+
if (tier.length === 0)
|
|
61
|
+
throw configError(`${where}.candidates[${String(index)}] is an empty tier`);
|
|
62
|
+
for (const ref of tier) {
|
|
63
|
+
if (ref.provider !== undefined && !config.providers.some((provider) => provider.id === ref.provider))
|
|
64
|
+
throw configError(`${where} names provider "${ref.provider}", which is not configured`);
|
|
65
|
+
if (matchesOf(config, ref).length === 0)
|
|
66
|
+
throw configError(`${where} names model "${ref.model}"${ref.provider === undefined ? "" : ` on provider "${ref.provider}"`}, which it does not serve`);
|
|
67
|
+
positive(ref.weight, `${where} candidate "${ref.model}" weight`);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
25
71
|
positive(policy.maxAttempts, `${where}.maxAttempts`);
|
|
26
72
|
positive(policy.timeoutMs, `${where}.timeoutMs`);
|
|
27
73
|
positive(policy.deadlineMs, `${where}.deadlineMs`);
|
|
@@ -31,7 +77,32 @@ function validatePolicy(config, policy, where) {
|
|
|
31
77
|
throw configError(`${where} names provider "${id}", which is not configured`);
|
|
32
78
|
}
|
|
33
79
|
}
|
|
34
|
-
/** Refuses a
|
|
80
|
+
/** Refuses a pinned profile whose candidates resolve to more than one model identity or vector length. */
|
|
81
|
+
function validatePinned(config, policy, where) {
|
|
82
|
+
const refs = refsOf(policy);
|
|
83
|
+
if (refs.length === 0)
|
|
84
|
+
throw configError(`${where} is pinned but names no model`);
|
|
85
|
+
const identities = new Set();
|
|
86
|
+
const dimensions = new Set();
|
|
87
|
+
for (const ref of refs) {
|
|
88
|
+
for (const { model } of matchesOf(config, ref)) {
|
|
89
|
+
identities.add(identityOf(model));
|
|
90
|
+
dimensions.add(String(model.dimensions ?? "unset"));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (identities.size > 1)
|
|
94
|
+
throw configError(`${where} is pinned but its candidates are different models (${[...identities].join(", ")}): give them one identity, or split the profile`);
|
|
95
|
+
if (dimensions.size > 1)
|
|
96
|
+
throw configError(`${where} is pinned but its candidates answer vectors of different lengths (${[...dimensions].join(", ")})`);
|
|
97
|
+
}
|
|
98
|
+
/** Every candidate a policy names, in order: its candidates, else its model and fallbacks. */
|
|
99
|
+
function refsOf(policy) {
|
|
100
|
+
const tiers = candidateTiers(policy);
|
|
101
|
+
if (tiers !== null)
|
|
102
|
+
return tiers.flat();
|
|
103
|
+
return [policy.model, ...(policy.fallbacks ?? [])].filter((name) => name !== undefined).map((model) => ({ model }));
|
|
104
|
+
}
|
|
105
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL, kind or number. */
|
|
35
106
|
export function validateConfig(config) {
|
|
36
107
|
if (config.providers.length === 0)
|
|
37
108
|
throw configError("the pool has no providers");
|
|
@@ -42,9 +113,14 @@ export function validateConfig(config) {
|
|
|
42
113
|
if (ids.has(provider.id))
|
|
43
114
|
throw configError(`provider "${provider.id}" is declared twice`);
|
|
44
115
|
ids.add(provider.id);
|
|
116
|
+
if (provider.kind !== undefined && !Object.hasOwn(PROVIDER_PRESETS, provider.kind))
|
|
117
|
+
throw configError(`provider "${provider.id}" has an unknown kind "${provider.kind}"`);
|
|
118
|
+
const preset = presetOf(provider);
|
|
119
|
+
if (provider.baseUrl === undefined && preset.baseUrl === null)
|
|
120
|
+
throw configError(`provider "${provider.id}" needs a baseUrl: kind ${preset.kind} has no default`);
|
|
45
121
|
let url;
|
|
46
122
|
try {
|
|
47
|
-
url = new URL(provider
|
|
123
|
+
url = new URL(baseUrlOf(provider));
|
|
48
124
|
}
|
|
49
125
|
catch {
|
|
50
126
|
throw configError(`provider "${provider.id}" has an invalid baseUrl`);
|
|
@@ -53,6 +129,9 @@ export function validateConfig(config) {
|
|
|
53
129
|
throw configError(`provider "${provider.id}" baseUrl must be http or https`);
|
|
54
130
|
positive(provider.weight, `provider "${provider.id}" weight`);
|
|
55
131
|
positive(provider.maxConcurrency, `provider "${provider.id}" maxConcurrency`);
|
|
132
|
+
positive(provider.defaultMaxTokens, `provider "${provider.id}" defaultMaxTokens`);
|
|
133
|
+
positive(provider.budget?.costPerDay, `provider "${provider.id}" budget.costPerDay`, true);
|
|
134
|
+
positive(provider.budget?.costPerMonth, `provider "${provider.id}" budget.costPerMonth`, true);
|
|
56
135
|
if (provider.keys.length === 0)
|
|
57
136
|
throw configError(`provider "${provider.id}" has no keys`);
|
|
58
137
|
if (provider.models.length === 0)
|
|
@@ -79,12 +158,23 @@ export function validateConfig(config) {
|
|
|
79
158
|
models.add(model.id);
|
|
80
159
|
positive(model.weight, `model "${model.id}" weight`);
|
|
81
160
|
positive(model.contextWindow, `model "${model.id}" contextWindow`);
|
|
161
|
+
if (model.dimensions !== undefined && (!Number.isInteger(model.dimensions) || model.dimensions < 1))
|
|
162
|
+
throw configError(`model "${model.id}" dimensions must be a positive integer`);
|
|
163
|
+
if (!preset.embeddings && (model.capabilities ?? []).includes("embeddings"))
|
|
164
|
+
throw configError(`provider "${provider.id}" (${preset.kind}) has no embeddings API, so model "${model.id}" cannot declare embeddings`);
|
|
82
165
|
}
|
|
83
166
|
}
|
|
84
|
-
|
|
167
|
+
positive(config.resilience?.quotaBlacklistMs, "resilience.quotaBlacklistMs");
|
|
168
|
+
if (config.defaults !== undefined) {
|
|
85
169
|
validatePolicy(config, config.defaults, "defaults");
|
|
86
|
-
|
|
170
|
+
if (config.defaults.pinned === true)
|
|
171
|
+
throw configError("defaults cannot be pinned: pin a profile");
|
|
172
|
+
}
|
|
173
|
+
for (const [name, policy] of Object.entries(config.profiles ?? {})) {
|
|
87
174
|
validatePolicy(config, policy, `profile "${name}"`);
|
|
175
|
+
if (policy.pinned === true)
|
|
176
|
+
validatePinned(config, policy, `profile "${name}"`);
|
|
177
|
+
}
|
|
88
178
|
}
|
|
89
179
|
/** Applies the defaults of every pool-wide setting. */
|
|
90
180
|
export function resolveSettings(config) {
|
|
@@ -104,6 +194,7 @@ export function resolveSettings(config) {
|
|
|
104
194
|
backoff: { baseMs: r.backoff?.baseMs ?? 100, factor: r.backoff?.factor ?? 2, maxMs: r.backoff?.maxMs ?? 5_000 },
|
|
105
195
|
authDisableMs: r.authDisableMs === undefined ? null : r.authDisableMs,
|
|
106
196
|
rateLimitCooldownMs: r.rateLimitCooldownMs ?? 10_000,
|
|
197
|
+
quotaBlacklistMs: r.quotaBlacklistMs ?? DEFAULT_QUOTA_BLACKLIST_MS,
|
|
107
198
|
latencyTolerance: s.latencyTolerance ?? 1.5,
|
|
108
199
|
latencySlackMs: s.latencySlackMs ?? 50,
|
|
109
200
|
ewmaAlpha: s.ewmaAlpha ?? 0.3,
|
|
@@ -112,7 +203,32 @@ export function resolveSettings(config) {
|
|
|
112
203
|
namespace: config.storeNamespace ?? "ai-router",
|
|
113
204
|
};
|
|
114
205
|
}
|
|
115
|
-
/**
|
|
206
|
+
/**
|
|
207
|
+
* Where a call's models come from, the first that names any: the call's model, the call's candidates, the profile's
|
|
208
|
+
* candidates, the profile's model, the request's model, the defaults' candidates, the defaults' model.
|
|
209
|
+
*/
|
|
210
|
+
function sourceOf(defaults, profile, call, requestModel) {
|
|
211
|
+
if (call.model !== undefined)
|
|
212
|
+
return { model: call.model };
|
|
213
|
+
for (const policy of [call, profile]) {
|
|
214
|
+
const tiers = candidateTiers(policy);
|
|
215
|
+
if (tiers !== null)
|
|
216
|
+
return { candidates: tiers };
|
|
217
|
+
if (policy === profile && profile.model !== undefined)
|
|
218
|
+
return { model: profile.model };
|
|
219
|
+
}
|
|
220
|
+
if (requestModel !== undefined)
|
|
221
|
+
return { model: requestModel };
|
|
222
|
+
const tiers = candidateTiers(defaults);
|
|
223
|
+
return tiers === null ? { model: defaults.model } : { candidates: tiers };
|
|
224
|
+
}
|
|
225
|
+
function single(model) {
|
|
226
|
+
return [{ model, provider: null, weight: 1 }];
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the candidates
|
|
230
|
+
* into tiers. A call's own `model` wins over the profile's candidates; a pinned profile refuses another model.
|
|
231
|
+
*/
|
|
116
232
|
export function resolvePolicy(config, options, requestModel) {
|
|
117
233
|
let profile = {};
|
|
118
234
|
if (options.profile !== undefined) {
|
|
@@ -121,23 +237,41 @@ export function resolvePolicy(config, options, requestModel) {
|
|
|
121
237
|
throw configError(`profile "${options.profile}" is not configured`);
|
|
122
238
|
profile = found;
|
|
123
239
|
}
|
|
124
|
-
const
|
|
125
|
-
if (
|
|
126
|
-
validatePolicy(config,
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const
|
|
134
|
-
if (
|
|
240
|
+
const call = options.policy;
|
|
241
|
+
if (call !== undefined)
|
|
242
|
+
validatePolicy(config, call, "policy");
|
|
243
|
+
const pinned = profile.pinned === true;
|
|
244
|
+
if (pinned && call !== undefined && (call.model !== undefined || call.fallbacks !== undefined || call.candidates !== undefined)) {
|
|
245
|
+
throw configError(`profile "${options.profile ?? ""}" is pinned to one model: a call cannot name another`);
|
|
246
|
+
}
|
|
247
|
+
const merged = { ...config.defaults, ...profile, ...call };
|
|
248
|
+
let tiers;
|
|
249
|
+
const source = sourceOf(config.defaults ?? {}, profile, call ?? {}, requestModel);
|
|
250
|
+
if ("candidates" in source) {
|
|
251
|
+
tiers = source.candidates.map((tier) => tier.map((ref) => ({ model: ref.model, provider: ref.provider ?? null, weight: ref.weight ?? 1 })));
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
const preferred = source.model;
|
|
255
|
+
if (preferred === undefined)
|
|
256
|
+
throw configError("no model: set request.model, a policy model or a profile model");
|
|
257
|
+
if (!knownModel(config, preferred))
|
|
258
|
+
throw configError(`model "${preferred}" is not served by any provider`);
|
|
259
|
+
const names = [preferred];
|
|
135
260
|
for (const name of merged.fallbacks ?? [])
|
|
136
|
-
if (!
|
|
137
|
-
|
|
261
|
+
if (!names.includes(name))
|
|
262
|
+
names.push(name);
|
|
263
|
+
tiers = names.map(single);
|
|
138
264
|
}
|
|
265
|
+
if (merged.allowFallback === false)
|
|
266
|
+
tiers = tiers.slice(0, 1);
|
|
267
|
+
const models = [];
|
|
268
|
+
for (const ref of tiers.flat())
|
|
269
|
+
if (!models.includes(ref.model))
|
|
270
|
+
models.push(ref.model);
|
|
139
271
|
return {
|
|
272
|
+
tiers,
|
|
140
273
|
models,
|
|
274
|
+
pinned,
|
|
141
275
|
maxAttempts: merged.maxAttempts ?? 4,
|
|
142
276
|
timeoutMs: merged.timeoutMs ?? 60_000,
|
|
143
277
|
deadlineMs: merged.deadlineMs ?? 120_000,
|
|
@@ -146,9 +280,29 @@ export function resolvePolicy(config, options, requestModel) {
|
|
|
146
280
|
require: merged.require ?? [],
|
|
147
281
|
};
|
|
148
282
|
}
|
|
149
|
-
|
|
150
|
-
|
|
283
|
+
/** The model names a profile (or the defaults) tries, first then the rest; the rest is empty when fallback is off. */
|
|
284
|
+
function namesOf(defaults, profile) {
|
|
285
|
+
const source = sourceOf(defaults, profile, {}, undefined);
|
|
286
|
+
const merged = { ...defaults, ...profile };
|
|
287
|
+
const names = [];
|
|
288
|
+
const add = (name) => {
|
|
289
|
+
if (!names.includes(name))
|
|
290
|
+
names.push(name);
|
|
291
|
+
};
|
|
292
|
+
if ("candidates" in source) {
|
|
293
|
+
for (const ref of source.candidates.flat())
|
|
294
|
+
add(ref.model);
|
|
295
|
+
const [first, ...rest] = names;
|
|
296
|
+
return { ...(first === undefined ? {} : { model: first }), fallbacks: merged.allowFallback === false ? [] : rest };
|
|
297
|
+
}
|
|
298
|
+
if (source.model !== undefined)
|
|
299
|
+
add(source.model);
|
|
300
|
+
for (const name of merged.fallbacks ?? [])
|
|
301
|
+
add(name);
|
|
302
|
+
const rest = source.model === undefined ? names : names.slice(1);
|
|
303
|
+
return { ...(source.model === undefined ? {} : { model: source.model }), fallbacks: merged.allowFallback === false ? [] : rest };
|
|
151
304
|
}
|
|
305
|
+
const OPERATIONS = ["chat", "embeddings"];
|
|
152
306
|
/** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
|
|
153
307
|
export function buildCatalog(config) {
|
|
154
308
|
const models = new Map();
|
|
@@ -171,11 +325,17 @@ export function buildCatalog(config) {
|
|
|
171
325
|
}
|
|
172
326
|
}
|
|
173
327
|
const defaults = config.defaults ?? {};
|
|
328
|
+
const serves = (name) => {
|
|
329
|
+
if (name === undefined)
|
|
330
|
+
return [];
|
|
331
|
+
const capabilities = new Set(matchesOf(config, { model: name }).flatMap(({ model }) => model.capabilities ?? DEFAULT_CAPABILITIES));
|
|
332
|
+
return OPERATIONS.filter((operation) => capabilities.has(operation));
|
|
333
|
+
};
|
|
174
334
|
return {
|
|
175
|
-
defaults:
|
|
335
|
+
defaults: namesOf({}, defaults),
|
|
176
336
|
profiles: Object.entries(config.profiles ?? {}).map(([name, policy]) => {
|
|
177
|
-
const
|
|
178
|
-
return { name, ...
|
|
337
|
+
const named = namesOf(defaults, policy);
|
|
338
|
+
return { name, ...named, pinned: policy.pinned === true, serves: serves(named.model) };
|
|
179
339
|
}),
|
|
180
340
|
models: [...models].map(([id, entry]) => ({
|
|
181
341
|
id,
|
package/dist/env.d.ts
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { KeyQuota, MountedKeySpec } from "./gateway/keys.js";
|
|
2
|
+
import type { KeyConfig, ModelConfig, PoolConfig, ProviderBudget, ProviderConfig, ProviderKind } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* The canonical env naming of upstream keys (docs/design.md, "Config from env"). For a provider whose prefix is `P`:
|
|
5
|
+
*
|
|
6
|
+
* P_API_KEY, P_API_KEY_2, P_API_KEY_3, ... one key each, aliases `<id>-1`, `<id>-2`, ... (gaps allowed, no upper bound)
|
|
7
|
+
* P_API_KEY_FILE, P_API_KEY_2_FILE, ... the same key mounted as a file (a Docker or Kubernetes secret)
|
|
8
|
+
* P_API_KEYS a list in one variable, split at commas and whitespace, aliases `<id>-list-1`, ...
|
|
9
|
+
* P_BASE_URL moves the endpoint (a Qwen region or workspace host, an Ollama host)
|
|
10
|
+
*
|
|
11
|
+
* The prefixes are ANTHROPIC, OPENROUTER, DASHSCOPE (Qwen), DEEPSEEK, OPENAI, GEMINI and GROQ; an `openai-compatible`
|
|
12
|
+
* provider names its own. Every key is read at each attempt, so a rotated secret needs no restart and lifts its blacklist.
|
|
13
|
+
*/
|
|
14
|
+
/** Env variables are read through this; the default is `process.env`. */
|
|
15
|
+
export type EnvSource = Readonly<Record<string, string | undefined>>;
|
|
16
|
+
/** One provider as an app declares it from env: its kind, models, and optionally its id, prefix and limits. */
|
|
17
|
+
export interface EnvProviderSpec {
|
|
18
|
+
readonly kind: ProviderKind;
|
|
19
|
+
/** Default: the kind (`openai-compatible` needs one). */
|
|
20
|
+
readonly id?: string;
|
|
21
|
+
/** Default: the kind's prefix (`openai-compatible` needs one). */
|
|
22
|
+
readonly envPrefix?: string;
|
|
23
|
+
/** Default: `<PREFIX>_BASE_URL`, then the kind's base URL. */
|
|
24
|
+
readonly baseUrl?: string;
|
|
25
|
+
readonly models: readonly ModelConfig[];
|
|
26
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
27
|
+
readonly weight?: number;
|
|
28
|
+
readonly maxConcurrency?: number;
|
|
29
|
+
/** The concurrency cap of each of its keys. */
|
|
30
|
+
readonly keyMaxConcurrency?: number;
|
|
31
|
+
readonly defaultMaxTokens?: number;
|
|
32
|
+
readonly budget?: ProviderBudget;
|
|
33
|
+
}
|
|
34
|
+
/** A pool declared from env: providers as `EnvProviderSpec`, everything else as `PoolConfig`. */
|
|
35
|
+
export interface EnvPoolSpec<Profiles extends string = string> extends Omit<PoolConfig<Profiles>, "providers"> {
|
|
36
|
+
readonly providers: readonly EnvProviderSpec[];
|
|
37
|
+
}
|
|
38
|
+
/** The prefix a spec reads its keys under. */
|
|
39
|
+
export declare function envPrefixOf(spec: Pick<EnvProviderSpec, "kind" | "envPrefix">): string;
|
|
40
|
+
/**
|
|
41
|
+
* The keys of one provider found in env under its prefix, as references read at each attempt: never a copied value.
|
|
42
|
+
* A key given both as a variable and as a `_FILE` twin is refused, since which one wins would be a guess.
|
|
43
|
+
*/
|
|
44
|
+
export declare function keysFromEnv(prefix: string, id: string, env?: EnvSource, maxConcurrency?: number): KeyConfig[];
|
|
45
|
+
/** The providers of the specs that have at least one key in env; a provider with none is left out, not refused. */
|
|
46
|
+
export declare function providersFromEnv(specs: readonly EnvProviderSpec[], env?: EnvSource): ProviderConfig[];
|
|
47
|
+
/**
|
|
48
|
+
* A pool from env: the providers that have keys, and every profile cut down to the candidates they serve, so an app
|
|
49
|
+
* declares every provider it may use and deploys with the keys it has. A profile left with nothing is refused, naming
|
|
50
|
+
* the variables that would give it a candidate.
|
|
51
|
+
*/
|
|
52
|
+
export declare function poolFromEnv<P extends string = string>(spec: EnvPoolSpec<P>, env?: EnvSource): PoolConfig<P>;
|
|
53
|
+
/** What `virtualKeyFromEnv` binds the app's own gateway key to. */
|
|
54
|
+
export interface VirtualKeyEnvSpec {
|
|
55
|
+
/** The label in the secret, `sk-starci-<label>-...`. */
|
|
56
|
+
readonly label: string;
|
|
57
|
+
/** Default `["work", "embed"]`. */
|
|
58
|
+
readonly profiles?: readonly string[];
|
|
59
|
+
/** Exact catalog model ids the key may name besides its profiles. Default none. */
|
|
60
|
+
readonly models?: readonly string[];
|
|
61
|
+
readonly quota?: KeyQuota;
|
|
62
|
+
readonly owner?: string;
|
|
63
|
+
/** Default `STARCI_AI_KEY`; `<name>_FILE` is read when the variable itself is unset. */
|
|
64
|
+
readonly variable?: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* The app's virtual key as a mounted key, from `STARCI_AI_KEY` (or its `_FILE` twin, a sealed secret mounted as a
|
|
68
|
+
* file): the one key every client of the app (its LangChain integration, its agent runtime) sends to the gateway.
|
|
69
|
+
*/
|
|
70
|
+
export declare function virtualKeyFromEnv(spec: VirtualKeyEnvSpec, env?: EnvSource): Promise<MountedKeySpec>;
|