@starci/ai-router 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +133 -40
  3. package/dist/ai-router.service.d.ts +2 -2
  4. package/dist/ai-router.service.js +4 -2
  5. package/dist/attempt.d.ts +26 -0
  6. package/dist/attempt.js +29 -0
  7. package/dist/classify.d.ts +40 -0
  8. package/dist/classify.js +145 -0
  9. package/dist/config.d.ts +30 -5
  10. package/dist/config.js +184 -24
  11. package/dist/env.d.ts +70 -0
  12. package/dist/env.js +152 -0
  13. package/dist/errors.d.ts +1 -3
  14. package/dist/errors.js +4 -23
  15. package/dist/fake.d.ts +624 -5
  16. package/dist/fake.js +177 -21
  17. package/dist/gateway/config-file.d.ts +8 -3
  18. package/dist/gateway/config-file.js +53 -8
  19. package/dist/gateway/gateway.d.ts +2 -0
  20. package/dist/gateway/gateway.js +19 -3
  21. package/dist/gateway/index.d.ts +10 -1
  22. package/dist/gateway/index.js +5 -0
  23. package/dist/gateway/serve.js +7 -2
  24. package/dist/headers.d.ts +1 -1
  25. package/dist/headers.js +12 -6
  26. package/dist/pool.d.ts +37 -16
  27. package/dist/pool.js +173 -48
  28. package/dist/providers.d.ts +34 -0
  29. package/dist/providers.js +22 -0
  30. package/dist/router.js +108 -105
  31. package/dist/secrets.d.ts +8 -0
  32. package/dist/secrets.js +22 -0
  33. package/dist/shapes.d.ts +11 -0
  34. package/dist/shapes.js +124 -0
  35. package/dist/state-stores.d.ts +38 -0
  36. package/dist/state-stores.js +101 -0
  37. package/dist/store.d.ts +26 -7
  38. package/dist/store.js +64 -2
  39. package/dist/testing.d.ts +2 -2
  40. package/dist/testing.js +1 -1
  41. package/dist/types.d.ts +126 -38
  42. package/dist/wire/anthropic.d.ts +21 -0
  43. package/dist/wire/anthropic.js +322 -0
  44. package/dist/wire/index.d.ts +5 -0
  45. package/dist/wire/index.js +7 -0
  46. package/dist/wire/wire.d.ts +42 -0
  47. package/dist/wire/wire.js +35 -0
  48. package/docs/design.md +279 -0
  49. package/package.json +13 -3
@@ -0,0 +1,145 @@
1
+ function isRecord(value) {
2
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3
+ }
4
+ function text(value) {
5
+ if (typeof value === "string")
6
+ return value;
7
+ if (typeof value === "number")
8
+ return String(value);
9
+ return "";
10
+ }
11
+ /**
12
+ * Reads a provider error body into its fields. It knows the OpenAI `{ error: { message, type, code, param } }`, the
13
+ * Anthropic `{ type: "error", error: { type, message, details } }`, OpenRouter's `{ error: { code: <status>, message,
14
+ * metadata } }`, Gemini's object or one-element list `[{ error: { code, message, status } }]` and DashScope's top-level
15
+ * `{ code, message }`. Anything else leaves the fields empty and the raw text as the message.
16
+ */
17
+ export function readProviderError(body, status) {
18
+ const root = Array.isArray(body) ? body[0] : body;
19
+ const error = isRecord(root) ? (isRecord(root.error) ? root.error : typeof root.code === "string" ? root : undefined) : undefined;
20
+ if (error === undefined)
21
+ return { status, type: "", code: "", detail: "", message: typeof body === "string" ? body : "" };
22
+ const details = isRecord(error.details) ? error.details : {};
23
+ const metadata = isRecord(error.metadata) ? error.metadata : {};
24
+ const numeric = typeof error.code === "number" ? error.code : undefined;
25
+ return {
26
+ status: status ?? numeric,
27
+ type: text(error.type) || text(error.status),
28
+ code: numeric === undefined ? text(error.code) : "",
29
+ detail: text(details.error_code) || text(metadata.limit_source) || text(metadata.error_type),
30
+ message: text(error.message),
31
+ };
32
+ }
33
+ /** Codes and types that mean the key's quota, credit or billing is spent: another key may serve, this one will not soon. */
34
+ const QUOTA_CODES = new Set([
35
+ "insufficient_quota",
36
+ "credit_balance_exhausted",
37
+ "organization_spend_limit_exceeded",
38
+ "project_spend_limit_exceeded",
39
+ "organization_usage_limit_exceeded",
40
+ "billing_error",
41
+ "billing_not_active",
42
+ "payment_required",
43
+ "quota_exceeded",
44
+ "insufficient_balance",
45
+ "arrearage",
46
+ "allocationquota.freetieronly",
47
+ "enforced_spend_limit_reached",
48
+ "openrouter_key_limit",
49
+ "openrouter_credits",
50
+ ]);
51
+ /** Messages that mean the same where a provider sends no code (DeepSeek, Anthropic's own usage limits, OpenRouter's free tier). */
52
+ const QUOTA_MESSAGE = /credit balance is too low|insufficient (?:credits|balance)|reached your specified (?:workspace )?api usage limits|exceeded your current quota|credit balance is depleted|free-models-per-day|requests per day|per day limit|perday/i;
53
+ /** Codes and types that mean the secret itself is refused: wrong, revoked, expired or leaked. */
54
+ const AUTH_CODES = new Set(["invalid_api_key", "authentication_error", "authentication", "invalidapikey", "unauthenticated", "permission_error", "permission_denied", "accessdenied", "model.accessdenied", "invalid_authentication"]);
55
+ const AUTH_MESSAGE = /reported as leaked|incorrect api key|invalid api key|api key (?:not valid|is invalid|expired)|invalid authentication/i;
56
+ /** Codes that mean the request was refused for what it is, not for the key: no retry. */
57
+ const INVALID_CODES = new Set(["context_length_exceeded", "invalid_request", "invalid_request_error", "invalidparameter", "datainspectionfailed", "modelnotfound", "model_not_found", "not_found", "not_found_error", "parameter_unknown", "request_too_large"]);
58
+ /** Codes of a rate limit: a short cooldown of the key and model. */
59
+ const RATE_CODES = new Set(["rate_limit_exceeded", "rate_limit_error", "too_many_requests", "slow_down", "throttling", "throttling.ratequota", "throttling.allocationquota", "throttling.burstrate", "openrouter_in_flight_budget", "resource_exhausted"]);
60
+ /** Codes of a provider that is overloaded or failing: transient. */
61
+ const TRANSIENT_CODES = new Set(["overloaded_error", "api_error", "server_error", "service_unavailable", "service_unavailable_error", "server_is_overloaded", "internalerror", "timeout_error", "deadline_exceeded"]);
62
+ /** The quota window a provider's answer implies, beyond the default TTL (docs/design.md, the blacklist lifecycle). */
63
+ function windowOf(kind, info, codes) {
64
+ const has = (code) => codes.includes(code);
65
+ // Anthropic's tier spend cap and OpenAI's monthly usage and spend limits reset at the start of the next UTC month.
66
+ if (has("enforced_spend_limit_reached") || has("organization_usage_limit_exceeded") || has("organization_spend_limit_exceeded") || has("project_spend_limit_exceeded"))
67
+ return { kind: "month-utc" };
68
+ // Gemini's requests-per-day reset at midnight Pacific time.
69
+ if (kind === "gemini" && (has("quota_exceeded") || /per ?day/i.test(info.message)))
70
+ return { kind: "day", timeZone: "America/Los_Angeles" };
71
+ // OpenRouter's free-model daily cap resets at 00:00 UTC.
72
+ if (kind === "openrouter" && /free-models-per-day|per day/i.test(info.message))
73
+ return { kind: "day", timeZone: "UTC" };
74
+ // Groq's request limit is per day (x-ratelimit-*-requests); its reset header carries the exact time instead.
75
+ return undefined;
76
+ }
77
+ /**
78
+ * Classifies a failed attempt from its status and error body, per provider kind (docs/design.md, the classification
79
+ * table). Order: a refused secret, then a spent quota (which some providers send as 429 or 400), then a rate limit,
80
+ * then the status alone. Classes: `auth` and `quota` blacklist the key, `rate_limit` cools it down, `transient` feeds
81
+ * the breakers, `invalid_request` is the caller's and is not retried.
82
+ */
83
+ export function classifyFailure(kind, status, body) {
84
+ const info = readProviderError(body, status);
85
+ const codes = [info.detail, info.code, info.type].map((code) => code.toLowerCase()).filter((code) => code !== "");
86
+ const code = [info.detail, info.code, info.type].find((value) => value !== "");
87
+ const has = (set) => codes.some((value) => set.has(value));
88
+ const effective = info.status;
89
+ const out = (errorClass, window) => ({ errorClass, code, ...(window === undefined ? {} : { window }) });
90
+ // OpenRouter's in-flight budget is a 402 that clears by itself: a short wait, not a spent quota.
91
+ if (codes.includes("openrouter_in_flight_budget"))
92
+ return out("rate_limit");
93
+ if (effective === 401 || has(AUTH_CODES) || AUTH_MESSAGE.test(info.message)) {
94
+ // OpenRouter answers 403 for a moderation or guardrail block: the request's fault, not the key's.
95
+ if (kind === "openrouter" && effective === 403)
96
+ return out("invalid_request");
97
+ return out("auth");
98
+ }
99
+ // Gemini words its per-minute 429 like a spent quota; only a daily limit is one (the project's RPD, until midnight Pacific).
100
+ if (kind === "gemini" && effective === 429 && !codes.includes("quota_exceeded") && !/per ?day/i.test(info.message))
101
+ return out("rate_limit");
102
+ const billingPrecondition = codes.includes("failed_precondition") && /billing/i.test(info.message);
103
+ if (effective === 402 || has(QUOTA_CODES) || QUOTA_MESSAGE.test(info.message) || billingPrecondition)
104
+ return out("quota", windowOf(kind, info, codes));
105
+ if (effective === 403)
106
+ return out(kind === "openrouter" ? "invalid_request" : "auth");
107
+ if (effective === 429 || has(RATE_CODES))
108
+ return out("rate_limit");
109
+ if (effective !== undefined) {
110
+ // Anthropic 529 overloaded, Groq 498 flex capacity, 408/409/425 and every 5xx are worth another lane.
111
+ if (effective === 408 || effective === 409 || effective === 425 || effective === 498 || effective >= 500)
112
+ return out("transient");
113
+ if (effective >= 400)
114
+ return out("invalid_request");
115
+ }
116
+ if (has(TRANSIENT_CODES))
117
+ return out("transient");
118
+ if (has(INVALID_CODES) || codes.some((value) => /invalid|context_length|not_found/.test(value)))
119
+ return out("invalid_request");
120
+ return out("transient");
121
+ }
122
+ /** The parts of a wall-clock time in a time zone. */
123
+ function wallClock(time, timeZone) {
124
+ const parts = new Intl.DateTimeFormat("en-US", { timeZone, hourCycle: "h23", hour: "numeric", minute: "numeric", second: "numeric" }).formatToParts(new Date(time));
125
+ const part = (type) => Number(parts.find((entry) => entry.type === type)?.value ?? 0);
126
+ return { hour: part("hour"), minute: part("minute"), second: part("second") };
127
+ }
128
+ /** The next midnight in a time zone after `now` (to the second; a DST day is corrected once). */
129
+ export function nextMidnight(now, timeZone) {
130
+ const elapsed = (clock) => ((clock.hour * 60 + clock.minute) * 60 + clock.second) * 1000;
131
+ let candidate = now - (now % 1000) + 86_400_000 - elapsed(wallClock(now, timeZone));
132
+ const off = elapsed(wallClock(candidate, timeZone));
133
+ // On a DST change the candidate lands an hour off midnight: move it back (23:00) or forward (01:00).
134
+ if (off !== 0)
135
+ candidate += off > 43_200_000 ? 86_400_000 - off : -off;
136
+ return candidate;
137
+ }
138
+ /** When a quota window ends after `now`. */
139
+ export function windowEnd(window, now) {
140
+ if (window.kind === "month-utc") {
141
+ const date = new Date(now);
142
+ return Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1);
143
+ }
144
+ return nextMidnight(now, window.timeZone);
145
+ }
package/dist/config.d.ts CHANGED
@@ -1,6 +1,8 @@
1
- import type { Capability, CallOptions, ModelConfig, PoolConfig, RouterCatalog } from "./types.js";
1
+ import type { CallOptions, Capability, ModelConfig, PoolConfig, ProviderConfig, RouterCatalog } from "./types.js";
2
2
  /** The capabilities of a model that declares none. */
3
3
  export declare const DEFAULT_CAPABILITIES: readonly Capability[];
4
+ /** How long a spent quota keeps its key out when the provider names no window: 6 hours. */
5
+ export declare const DEFAULT_QUOTA_BLACKLIST_MS: number;
4
6
  /** A breaker's thresholds with every default applied. */
5
7
  export interface BreakerSettings {
6
8
  readonly failureThreshold: number;
@@ -18,6 +20,7 @@ export interface Settings {
18
20
  };
19
21
  readonly authDisableMs: number | null;
20
22
  readonly rateLimitCooldownMs: number;
23
+ readonly quotaBlacklistMs: number;
21
24
  readonly latencyTolerance: number;
22
25
  readonly latencySlackMs: number;
23
26
  readonly ewmaAlpha: number;
@@ -25,10 +28,19 @@ export interface Settings {
25
28
  readonly includeContent: boolean;
26
29
  readonly namespace: string;
27
30
  }
28
- /** One call's policy with every default applied and its model list expanded. */
31
+ /** One candidate of a resolved policy: a model name, the provider it is held to (null: any), and its weight in its tier. */
32
+ export interface ResolvedRef {
33
+ readonly model: string;
34
+ readonly provider: string | null;
35
+ readonly weight: number;
36
+ }
37
+ /** One call's policy with every default applied and its candidates expanded into tiers. */
29
38
  export interface ResolvedPolicy {
30
- /** The preferred model first, then the fallbacks when fallback is allowed, without repeats. */
39
+ /** The tiers, tried in order; the candidates of one tier share the load by weight. */
40
+ readonly tiers: readonly (readonly ResolvedRef[])[];
41
+ /** Every model name of the tiers, in order and without repeats: what a message names. */
31
42
  readonly models: readonly string[];
43
+ readonly pinned: boolean;
32
44
  readonly maxAttempts: number;
33
45
  readonly timeoutMs: number;
34
46
  readonly deadlineMs: number;
@@ -38,11 +50,24 @@ export interface ResolvedPolicy {
38
50
  }
39
51
  /** True when the name is the model's id or one of its aliases. */
40
52
  export declare function modelMatches(model: ModelConfig, name: string): boolean;
41
- /** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
53
+ /** The identity of a model: its declared `identity`, else its id. */
54
+ export declare function identityOf(model: ModelConfig): string;
55
+ /** Every (provider, model) a candidate resolves to. */
56
+ export declare function matchesOf(config: PoolConfig, ref: {
57
+ readonly model: string;
58
+ readonly provider?: string | null;
59
+ }): {
60
+ provider: ProviderConfig;
61
+ model: ModelConfig;
62
+ }[];
63
+ /** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL, kind or number. */
42
64
  export declare function validateConfig(config: PoolConfig): void;
43
65
  /** Applies the defaults of every pool-wide setting. */
44
66
  export declare function resolveSettings(config: PoolConfig): Settings;
45
- /** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
67
+ /**
68
+ * Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the candidates
69
+ * into tiers. A call's own `model` wins over the profile's candidates; a pinned profile refuses another model.
70
+ */
46
71
  export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
47
72
  /** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
48
73
  export declare function buildCatalog(config: PoolConfig): RouterCatalog;
package/dist/config.js CHANGED
@@ -1,6 +1,9 @@
1
1
  import { AiRouterError } from "./errors.js";
2
+ import { baseUrlOf, presetOf, PROVIDER_PRESETS } from "./providers.js";
2
3
  /** The capabilities of a model that declares none. */
3
4
  export const DEFAULT_CAPABILITIES = ["chat", "streaming", "tools", "json"];
5
+ /** How long a spent quota keeps its key out when the provider names no window: 6 hours. */
6
+ export const DEFAULT_QUOTA_BLACKLIST_MS = 6 * 3_600_000;
4
7
  function configError(message) {
5
8
  return new AiRouterError({ errorClass: "config", message });
6
9
  }
@@ -14,14 +17,57 @@ function positive(value, name, allowZero = false) {
14
17
  export function modelMatches(model, name) {
15
18
  return model.id === name || (model.aliases?.includes(name) ?? false);
16
19
  }
20
+ /** The identity of a model: its declared `identity`, else its id. */
21
+ export function identityOf(model) {
22
+ return model.identity ?? model.id;
23
+ }
24
+ /** Every (provider, model) a candidate resolves to. */
25
+ export function matchesOf(config, ref) {
26
+ const out = [];
27
+ for (const provider of config.providers) {
28
+ if (ref.provider !== undefined && ref.provider !== null && provider.id !== ref.provider)
29
+ continue;
30
+ for (const model of provider.models)
31
+ if (modelMatches(model, ref.model))
32
+ out.push({ provider, model });
33
+ }
34
+ return out;
35
+ }
17
36
  function knownModel(config, name) {
18
- return config.providers.some((provider) => provider.models.some((model) => modelMatches(model, name)));
37
+ return matchesOf(config, { model: name }).length > 0;
38
+ }
39
+ /** The candidates of a policy as tiers of refs, or null when it lists none. */
40
+ function candidateTiers(policy) {
41
+ if (policy.candidates === undefined)
42
+ return null;
43
+ return policy.candidates.map((entry) => (isRefList(entry) ? [...entry] : [entry]));
44
+ }
45
+ function isRefList(entry) {
46
+ return Array.isArray(entry);
19
47
  }
20
48
  function validatePolicy(config, policy, where) {
21
49
  for (const name of [policy.model, ...(policy.fallbacks ?? [])]) {
22
50
  if (name !== undefined && !knownModel(config, name))
23
51
  throw configError(`${where} names model "${name}", which no provider serves`);
24
52
  }
53
+ const tiers = candidateTiers(policy);
54
+ if (tiers !== null) {
55
+ if (policy.model !== undefined || policy.fallbacks !== undefined)
56
+ throw configError(`${where} sets candidates and model or fallbacks: give one or the other`);
57
+ if (tiers.length === 0)
58
+ throw configError(`${where}.candidates is empty`);
59
+ tiers.forEach((tier, index) => {
60
+ if (tier.length === 0)
61
+ throw configError(`${where}.candidates[${String(index)}] is an empty tier`);
62
+ for (const ref of tier) {
63
+ if (ref.provider !== undefined && !config.providers.some((provider) => provider.id === ref.provider))
64
+ throw configError(`${where} names provider "${ref.provider}", which is not configured`);
65
+ if (matchesOf(config, ref).length === 0)
66
+ throw configError(`${where} names model "${ref.model}"${ref.provider === undefined ? "" : ` on provider "${ref.provider}"`}, which it does not serve`);
67
+ positive(ref.weight, `${where} candidate "${ref.model}" weight`);
68
+ }
69
+ });
70
+ }
25
71
  positive(policy.maxAttempts, `${where}.maxAttempts`);
26
72
  positive(policy.timeoutMs, `${where}.timeoutMs`);
27
73
  positive(policy.deadlineMs, `${where}.deadlineMs`);
@@ -31,7 +77,32 @@ function validatePolicy(config, policy, where) {
31
77
  throw configError(`${where} names provider "${id}", which is not configured`);
32
78
  }
33
79
  }
34
- /** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
80
+ /** Refuses a pinned profile whose candidates resolve to more than one model identity or vector length. */
81
+ function validatePinned(config, policy, where) {
82
+ const refs = refsOf(policy);
83
+ if (refs.length === 0)
84
+ throw configError(`${where} is pinned but names no model`);
85
+ const identities = new Set();
86
+ const dimensions = new Set();
87
+ for (const ref of refs) {
88
+ for (const { model } of matchesOf(config, ref)) {
89
+ identities.add(identityOf(model));
90
+ dimensions.add(String(model.dimensions ?? "unset"));
91
+ }
92
+ }
93
+ if (identities.size > 1)
94
+ throw configError(`${where} is pinned but its candidates are different models (${[...identities].join(", ")}): give them one identity, or split the profile`);
95
+ if (dimensions.size > 1)
96
+ throw configError(`${where} is pinned but its candidates answer vectors of different lengths (${[...dimensions].join(", ")})`);
97
+ }
98
+ /** Every candidate a policy names, in order: its candidates, else its model and fallbacks. */
99
+ function refsOf(policy) {
100
+ const tiers = candidateTiers(policy);
101
+ if (tiers !== null)
102
+ return tiers.flat();
103
+ return [policy.model, ...(policy.fallbacks ?? [])].filter((name) => name !== undefined).map((model) => ({ model }));
104
+ }
105
+ /** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL, kind or number. */
35
106
  export function validateConfig(config) {
36
107
  if (config.providers.length === 0)
37
108
  throw configError("the pool has no providers");
@@ -42,9 +113,14 @@ export function validateConfig(config) {
42
113
  if (ids.has(provider.id))
43
114
  throw configError(`provider "${provider.id}" is declared twice`);
44
115
  ids.add(provider.id);
116
+ if (provider.kind !== undefined && !Object.hasOwn(PROVIDER_PRESETS, provider.kind))
117
+ throw configError(`provider "${provider.id}" has an unknown kind "${provider.kind}"`);
118
+ const preset = presetOf(provider);
119
+ if (provider.baseUrl === undefined && preset.baseUrl === null)
120
+ throw configError(`provider "${provider.id}" needs a baseUrl: kind ${preset.kind} has no default`);
45
121
  let url;
46
122
  try {
47
- url = new URL(provider.baseUrl);
123
+ url = new URL(baseUrlOf(provider));
48
124
  }
49
125
  catch {
50
126
  throw configError(`provider "${provider.id}" has an invalid baseUrl`);
@@ -53,6 +129,9 @@ export function validateConfig(config) {
53
129
  throw configError(`provider "${provider.id}" baseUrl must be http or https`);
54
130
  positive(provider.weight, `provider "${provider.id}" weight`);
55
131
  positive(provider.maxConcurrency, `provider "${provider.id}" maxConcurrency`);
132
+ positive(provider.defaultMaxTokens, `provider "${provider.id}" defaultMaxTokens`);
133
+ positive(provider.budget?.costPerDay, `provider "${provider.id}" budget.costPerDay`, true);
134
+ positive(provider.budget?.costPerMonth, `provider "${provider.id}" budget.costPerMonth`, true);
56
135
  if (provider.keys.length === 0)
57
136
  throw configError(`provider "${provider.id}" has no keys`);
58
137
  if (provider.models.length === 0)
@@ -79,12 +158,23 @@ export function validateConfig(config) {
79
158
  models.add(model.id);
80
159
  positive(model.weight, `model "${model.id}" weight`);
81
160
  positive(model.contextWindow, `model "${model.id}" contextWindow`);
161
+ if (model.dimensions !== undefined && (!Number.isInteger(model.dimensions) || model.dimensions < 1))
162
+ throw configError(`model "${model.id}" dimensions must be a positive integer`);
163
+ if (!preset.embeddings && (model.capabilities ?? []).includes("embeddings"))
164
+ throw configError(`provider "${provider.id}" (${preset.kind}) has no embeddings API, so model "${model.id}" cannot declare embeddings`);
82
165
  }
83
166
  }
84
- if (config.defaults !== undefined)
167
+ positive(config.resilience?.quotaBlacklistMs, "resilience.quotaBlacklistMs");
168
+ if (config.defaults !== undefined) {
85
169
  validatePolicy(config, config.defaults, "defaults");
86
- for (const [name, policy] of Object.entries(config.profiles ?? {}))
170
+ if (config.defaults.pinned === true)
171
+ throw configError("defaults cannot be pinned: pin a profile");
172
+ }
173
+ for (const [name, policy] of Object.entries(config.profiles ?? {})) {
87
174
  validatePolicy(config, policy, `profile "${name}"`);
175
+ if (policy.pinned === true)
176
+ validatePinned(config, policy, `profile "${name}"`);
177
+ }
88
178
  }
89
179
  /** Applies the defaults of every pool-wide setting. */
90
180
  export function resolveSettings(config) {
@@ -104,6 +194,7 @@ export function resolveSettings(config) {
104
194
  backoff: { baseMs: r.backoff?.baseMs ?? 100, factor: r.backoff?.factor ?? 2, maxMs: r.backoff?.maxMs ?? 5_000 },
105
195
  authDisableMs: r.authDisableMs === undefined ? null : r.authDisableMs,
106
196
  rateLimitCooldownMs: r.rateLimitCooldownMs ?? 10_000,
197
+ quotaBlacklistMs: r.quotaBlacklistMs ?? DEFAULT_QUOTA_BLACKLIST_MS,
107
198
  latencyTolerance: s.latencyTolerance ?? 1.5,
108
199
  latencySlackMs: s.latencySlackMs ?? 50,
109
200
  ewmaAlpha: s.ewmaAlpha ?? 0.3,
@@ -112,7 +203,32 @@ export function resolveSettings(config) {
112
203
  namespace: config.storeNamespace ?? "ai-router",
113
204
  };
114
205
  }
115
- /** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
206
+ /**
207
+ * Where a call's models come from, the first that names any: the call's model, the call's candidates, the profile's
208
+ * candidates, the profile's model, the request's model, the defaults' candidates, the defaults' model.
209
+ */
210
+ function sourceOf(defaults, profile, call, requestModel) {
211
+ if (call.model !== undefined)
212
+ return { model: call.model };
213
+ for (const policy of [call, profile]) {
214
+ const tiers = candidateTiers(policy);
215
+ if (tiers !== null)
216
+ return { candidates: tiers };
217
+ if (policy === profile && profile.model !== undefined)
218
+ return { model: profile.model };
219
+ }
220
+ if (requestModel !== undefined)
221
+ return { model: requestModel };
222
+ const tiers = candidateTiers(defaults);
223
+ return tiers === null ? { model: defaults.model } : { candidates: tiers };
224
+ }
225
+ function single(model) {
226
+ return [{ model, provider: null, weight: 1 }];
227
+ }
228
+ /**
229
+ * Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the candidates
230
+ * into tiers. A call's own `model` wins over the profile's candidates; a pinned profile refuses another model.
231
+ */
116
232
  export function resolvePolicy(config, options, requestModel) {
117
233
  let profile = {};
118
234
  if (options.profile !== undefined) {
@@ -121,23 +237,41 @@ export function resolvePolicy(config, options, requestModel) {
121
237
  throw configError(`profile "${options.profile}" is not configured`);
122
238
  profile = found;
123
239
  }
124
- const merged = { ...config.defaults, ...profile, ...options.policy };
125
- if (options.policy !== undefined)
126
- validatePolicy(config, options.policy, "policy");
127
- // A call's policy, then its chosen profile, then the request's own model, then the defaults.
128
- const preferred = options.policy?.model ?? profile.model ?? requestModel ?? config.defaults?.model;
129
- if (preferred === undefined)
130
- throw configError("no model: set request.model, a policy model or a profile model");
131
- if (!knownModel(config, preferred))
132
- throw configError(`model "${preferred}" is not served by any provider`);
133
- const models = [preferred];
134
- if (merged.allowFallback !== false) {
240
+ const call = options.policy;
241
+ if (call !== undefined)
242
+ validatePolicy(config, call, "policy");
243
+ const pinned = profile.pinned === true;
244
+ if (pinned && call !== undefined && (call.model !== undefined || call.fallbacks !== undefined || call.candidates !== undefined)) {
245
+ throw configError(`profile "${options.profile ?? ""}" is pinned to one model: a call cannot name another`);
246
+ }
247
+ const merged = { ...config.defaults, ...profile, ...call };
248
+ let tiers;
249
+ const source = sourceOf(config.defaults ?? {}, profile, call ?? {}, requestModel);
250
+ if ("candidates" in source) {
251
+ tiers = source.candidates.map((tier) => tier.map((ref) => ({ model: ref.model, provider: ref.provider ?? null, weight: ref.weight ?? 1 })));
252
+ }
253
+ else {
254
+ const preferred = source.model;
255
+ if (preferred === undefined)
256
+ throw configError("no model: set request.model, a policy model or a profile model");
257
+ if (!knownModel(config, preferred))
258
+ throw configError(`model "${preferred}" is not served by any provider`);
259
+ const names = [preferred];
135
260
  for (const name of merged.fallbacks ?? [])
136
- if (!models.includes(name))
137
- models.push(name);
261
+ if (!names.includes(name))
262
+ names.push(name);
263
+ tiers = names.map(single);
138
264
  }
265
+ if (merged.allowFallback === false)
266
+ tiers = tiers.slice(0, 1);
267
+ const models = [];
268
+ for (const ref of tiers.flat())
269
+ if (!models.includes(ref.model))
270
+ models.push(ref.model);
139
271
  return {
272
+ tiers,
140
273
  models,
274
+ pinned,
141
275
  maxAttempts: merged.maxAttempts ?? 4,
142
276
  timeoutMs: merged.timeoutMs ?? 60_000,
143
277
  deadlineMs: merged.deadlineMs ?? 120_000,
@@ -146,9 +280,29 @@ export function resolvePolicy(config, options, requestModel) {
146
280
  require: merged.require ?? [],
147
281
  };
148
282
  }
149
- function fallbacksOf(policy) {
150
- return policy.allowFallback === false ? [] : (policy.fallbacks ?? []);
283
+ /** The model names a profile (or the defaults) tries, first then the rest; the rest is empty when fallback is off. */
284
+ function namesOf(defaults, profile) {
285
+ const source = sourceOf(defaults, profile, {}, undefined);
286
+ const merged = { ...defaults, ...profile };
287
+ const names = [];
288
+ const add = (name) => {
289
+ if (!names.includes(name))
290
+ names.push(name);
291
+ };
292
+ if ("candidates" in source) {
293
+ for (const ref of source.candidates.flat())
294
+ add(ref.model);
295
+ const [first, ...rest] = names;
296
+ return { ...(first === undefined ? {} : { model: first }), fallbacks: merged.allowFallback === false ? [] : rest };
297
+ }
298
+ if (source.model !== undefined)
299
+ add(source.model);
300
+ for (const name of merged.fallbacks ?? [])
301
+ add(name);
302
+ const rest = source.model === undefined ? names : names.slice(1);
303
+ return { ...(source.model === undefined ? {} : { model: source.model }), fallbacks: merged.allowFallback === false ? [] : rest };
151
304
  }
305
+ const OPERATIONS = ["chat", "embeddings"];
152
306
  /** Lists the profiles and models of a validated pool; a model id served by several providers is one entry. */
153
307
  export function buildCatalog(config) {
154
308
  const models = new Map();
@@ -171,11 +325,17 @@ export function buildCatalog(config) {
171
325
  }
172
326
  }
173
327
  const defaults = config.defaults ?? {};
328
+ const serves = (name) => {
329
+ if (name === undefined)
330
+ return [];
331
+ const capabilities = new Set(matchesOf(config, { model: name }).flatMap(({ model }) => model.capabilities ?? DEFAULT_CAPABILITIES));
332
+ return OPERATIONS.filter((operation) => capabilities.has(operation));
333
+ };
174
334
  return {
175
- defaults: { ...(defaults.model === undefined ? {} : { model: defaults.model }), fallbacks: fallbacksOf(defaults) },
335
+ defaults: namesOf({}, defaults),
176
336
  profiles: Object.entries(config.profiles ?? {}).map(([name, policy]) => {
177
- const merged = { ...defaults, ...policy };
178
- return { name, ...(merged.model === undefined ? {} : { model: merged.model }), fallbacks: fallbacksOf(merged) };
337
+ const named = namesOf(defaults, policy);
338
+ return { name, ...named, pinned: policy.pinned === true, serves: serves(named.model) };
179
339
  }),
180
340
  models: [...models].map(([id, entry]) => ({
181
341
  id,
package/dist/env.d.ts ADDED
@@ -0,0 +1,70 @@
1
+ import type { KeyQuota, MountedKeySpec } from "./gateway/keys.js";
2
+ import type { KeyConfig, ModelConfig, PoolConfig, ProviderBudget, ProviderConfig, ProviderKind } from "./types.js";
3
+ /**
4
+ * The canonical env naming of upstream keys (docs/design.md, "Config from env"). For a provider whose prefix is `P`:
5
+ *
6
+ * P_API_KEY, P_API_KEY_2, P_API_KEY_3, ... one key each, aliases `<id>-1`, `<id>-2`, ... (gaps allowed, no upper bound)
7
+ * P_API_KEY_FILE, P_API_KEY_2_FILE, ... the same key mounted as a file (a Docker or Kubernetes secret)
8
+ * P_API_KEYS a list in one variable, split at commas and whitespace, aliases `<id>-list-1`, ...
9
+ * P_BASE_URL moves the endpoint (a Qwen region or workspace host, an Ollama host)
10
+ *
11
+ * The prefixes are ANTHROPIC, OPENROUTER, DASHSCOPE (Qwen), DEEPSEEK, OPENAI, GEMINI and GROQ; an `openai-compatible`
12
+ * provider names its own. Every key is read at each attempt, so a rotated secret needs no restart and lifts its blacklist.
13
+ */
14
+ /** Env variables are read through this; the default is `process.env`. */
15
+ export type EnvSource = Readonly<Record<string, string | undefined>>;
16
+ /** One provider as an app declares it from env: its kind, models, and optionally its id, prefix and limits. */
17
+ export interface EnvProviderSpec {
18
+ readonly kind: ProviderKind;
19
+ /** Default: the kind (`openai-compatible` needs one). */
20
+ readonly id?: string;
21
+ /** Default: the kind's prefix (`openai-compatible` needs one). */
22
+ readonly envPrefix?: string;
23
+ /** Default: `<PREFIX>_BASE_URL`, then the kind's base URL. */
24
+ readonly baseUrl?: string;
25
+ readonly models: readonly ModelConfig[];
26
+ readonly headers?: Readonly<Record<string, string>>;
27
+ readonly weight?: number;
28
+ readonly maxConcurrency?: number;
29
+ /** The concurrency cap of each of its keys. */
30
+ readonly keyMaxConcurrency?: number;
31
+ readonly defaultMaxTokens?: number;
32
+ readonly budget?: ProviderBudget;
33
+ }
34
+ /** A pool declared from env: providers as `EnvProviderSpec`, everything else as `PoolConfig`. */
35
+ export interface EnvPoolSpec<Profiles extends string = string> extends Omit<PoolConfig<Profiles>, "providers"> {
36
+ readonly providers: readonly EnvProviderSpec[];
37
+ }
38
+ /** The prefix a spec reads its keys under. */
39
+ export declare function envPrefixOf(spec: Pick<EnvProviderSpec, "kind" | "envPrefix">): string;
40
+ /**
41
+ * The keys of one provider found in env under its prefix, as references read at each attempt: never a copied value.
42
+ * A key given both as a variable and as a `_FILE` twin is refused, since which one wins would be a guess.
43
+ */
44
+ export declare function keysFromEnv(prefix: string, id: string, env?: EnvSource, maxConcurrency?: number): KeyConfig[];
45
+ /** The providers of the specs that have at least one key in env; a provider with none is left out, not refused. */
46
+ export declare function providersFromEnv(specs: readonly EnvProviderSpec[], env?: EnvSource): ProviderConfig[];
47
+ /**
48
+ * A pool from env: the providers that have keys, and every profile cut down to the candidates they serve, so an app
49
+ * declares every provider it may use and deploys with the keys it has. A profile left with nothing is refused, naming
50
+ * the variables that would give it a candidate.
51
+ */
52
+ export declare function poolFromEnv<P extends string = string>(spec: EnvPoolSpec<P>, env?: EnvSource): PoolConfig<P>;
53
+ /** What `virtualKeyFromEnv` binds the app's own gateway key to. */
54
+ export interface VirtualKeyEnvSpec {
55
+ /** The label in the secret, `sk-starci-<label>-...`. */
56
+ readonly label: string;
57
+ /** Default `["work", "embed"]`. */
58
+ readonly profiles?: readonly string[];
59
+ /** Exact catalog model ids the key may name besides its profiles. Default none. */
60
+ readonly models?: readonly string[];
61
+ readonly quota?: KeyQuota;
62
+ readonly owner?: string;
63
+ /** Default `STARCI_AI_KEY`; `<name>_FILE` is read when the variable itself is unset. */
64
+ readonly variable?: string;
65
+ }
66
+ /**
67
+ * The app's virtual key as a mounted key, from `STARCI_AI_KEY` (or its `_FILE` twin, a sealed secret mounted as a
68
+ * file): the one key every client of the app (its LangChain integration, its agent runtime) sends to the gateway.
69
+ */
70
+ export declare function virtualKeyFromEnv(spec: VirtualKeyEnvSpec, env?: EnvSource): Promise<MountedKeySpec>;