@starci/ai-router 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +133 -40
  3. package/dist/ai-router.service.d.ts +2 -2
  4. package/dist/ai-router.service.js +4 -2
  5. package/dist/attempt.d.ts +26 -0
  6. package/dist/attempt.js +29 -0
  7. package/dist/classify.d.ts +40 -0
  8. package/dist/classify.js +145 -0
  9. package/dist/config.d.ts +30 -5
  10. package/dist/config.js +184 -24
  11. package/dist/env.d.ts +70 -0
  12. package/dist/env.js +152 -0
  13. package/dist/errors.d.ts +1 -3
  14. package/dist/errors.js +4 -23
  15. package/dist/fake.d.ts +624 -5
  16. package/dist/fake.js +177 -21
  17. package/dist/gateway/config-file.d.ts +8 -3
  18. package/dist/gateway/config-file.js +53 -8
  19. package/dist/gateway/gateway.d.ts +2 -0
  20. package/dist/gateway/gateway.js +19 -3
  21. package/dist/gateway/index.d.ts +10 -1
  22. package/dist/gateway/index.js +5 -0
  23. package/dist/gateway/serve.js +7 -2
  24. package/dist/headers.d.ts +1 -1
  25. package/dist/headers.js +12 -6
  26. package/dist/pool.d.ts +37 -16
  27. package/dist/pool.js +173 -48
  28. package/dist/providers.d.ts +34 -0
  29. package/dist/providers.js +22 -0
  30. package/dist/router.js +108 -105
  31. package/dist/secrets.d.ts +8 -0
  32. package/dist/secrets.js +22 -0
  33. package/dist/shapes.d.ts +11 -0
  34. package/dist/shapes.js +124 -0
  35. package/dist/state-stores.d.ts +38 -0
  36. package/dist/state-stores.js +101 -0
  37. package/dist/store.d.ts +26 -7
  38. package/dist/store.js +64 -2
  39. package/dist/testing.d.ts +2 -2
  40. package/dist/testing.js +1 -1
  41. package/dist/types.d.ts +126 -38
  42. package/dist/wire/anthropic.d.ts +21 -0
  43. package/dist/wire/anthropic.js +322 -0
  44. package/dist/wire/index.d.ts +5 -0
  45. package/dist/wire/index.js +7 -0
  46. package/dist/wire/wire.d.ts +42 -0
  47. package/dist/wire/wire.js +35 -0
  48. package/docs/design.md +279 -0
  49. package/package.json +13 -3
package/dist/fake.js CHANGED
@@ -1,7 +1,87 @@
1
+ /**
2
+ * Each provider's real error answers (status, body, headers) as its docs showed them on 2026-10-31 (docs/design.md,
3
+ * the classification table), for a spec to script with `fake.on(match, PROVIDER_ERRORS.anthropic.quota_exhausted)`.
4
+ * Extra cases name a provider's second shape of one class (Anthropic's monthly spend cap, OpenRouter's free daily cap).
5
+ */
6
+ export const PROVIDER_ERRORS = Object.freeze({
7
+ openai: {
8
+ rate_limited: { status: 429, headers: { "retry-after": "2", "x-ratelimit-remaining-requests": "0", "x-ratelimit-reset-requests": "2s" }, body: { error: { message: "Rate limit reached for requests", type: "requests", param: null, code: "rate_limit_exceeded" } } },
9
+ quota_exhausted: { status: 429, body: { error: { message: "You exceeded your current quota, please check your plan and billing details.", type: "insufficient_quota", param: null, code: "insufficient_quota" } } },
10
+ credit_exhausted: { status: 429, body: { error: { message: "Your credit balance is exhausted.", type: "insufficient_quota", param: null, code: "credit_balance_exhausted" } } },
11
+ usage_limit: { status: 429, body: { error: { message: "Your organization reached its monthly usage limit.", type: "insufficient_quota", param: null, code: "organization_usage_limit_exceeded" } } },
12
+ invalid_key: { status: 401, body: { error: { message: "Incorrect API key provided: sk-proj-****abcd.", type: "invalid_request_error", param: null, code: "invalid_api_key" } } },
13
+ overloaded: { status: 503, headers: { "retry-after": "1" }, body: { error: { message: "The server is overloaded.", type: "server_error", param: null, code: "server_is_overloaded" } } },
14
+ bad_request: { status: 400, body: { error: { message: "This model's maximum context length is 128000 tokens.", type: "invalid_request_error", param: "messages", code: "context_length_exceeded" } } },
15
+ },
16
+ anthropic: {
17
+ rate_limited: { status: 429, headers: { "retry-after": "3", "anthropic-ratelimit-requests-remaining": "0", "anthropic-ratelimit-requests-reset": "2026-10-31T12:00:03Z" }, body: { type: "error", error: { type: "rate_limit_error", message: "Number of request tokens has exceeded your per-minute rate limit" }, request_id: "req_fake" } },
18
+ quota_exhausted: { status: 402, body: { type: "error", error: { type: "billing_error", message: "Your credit balance is too low to access the Anthropic API." }, request_id: "req_fake" } },
19
+ spend_cap: { status: 429, body: { type: "error", error: { type: "rate_limit_error", message: "You have reached your monthly spend limit.", details: { error_code: "enforced_spend_limit_reached" } }, request_id: "req_fake" } },
20
+ usage_limit: { status: 400, body: { type: "error", error: { type: "invalid_request_error", message: "You have reached your specified API usage limits. You will regain access on 2026-11-01 at 00:00 UTC." }, request_id: "req_fake" } },
21
+ invalid_key: { status: 401, body: { type: "error", error: { type: "authentication_error", message: "invalid x-api-key" }, request_id: "req_fake" } },
22
+ overloaded: { status: 529, body: { type: "error", error: { type: "overloaded_error", message: "Overloaded" }, request_id: "req_fake" } },
23
+ bad_request: { status: 400, body: { type: "error", error: { type: "invalid_request_error", message: "prompt is too long: 1000001 tokens > 1000000 maximum" }, request_id: "req_fake" } },
24
+ },
25
+ openrouter: {
26
+ rate_limited: { status: 429, headers: { "X-RateLimit-Limit": "20", "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1961913600000" }, body: { error: { code: 429, message: "Rate limit exceeded", metadata: {} } } },
27
+ quota_exhausted: { status: 402, body: { error: { code: 402, message: "Insufficient credits. Add more using https://openrouter.ai/credits", metadata: { limit_source: "openrouter_credits" } } } },
28
+ key_limit: { status: 402, body: { error: { code: 402, message: "Key limit exceeded", metadata: { limit_source: "openrouter_key_limit" } } } },
29
+ in_flight_budget: { status: 402, headers: { "retry-after": "1" }, body: { error: { code: 402, message: "In-flight budget exceeded", metadata: { limit_source: "openrouter_in_flight_budget" } } } },
30
+ free_daily: { status: 429, body: { error: { code: 429, message: "Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day", metadata: {} } } },
31
+ invalid_key: { status: 401, body: { error: { code: 401, message: "User not found.", metadata: {} } } },
32
+ overloaded: { status: 502, body: { error: { code: 502, message: "Provider returned error", metadata: { provider_name: "upstream", raw: "overloaded" } } } },
33
+ moderation: { status: 403, body: { error: { code: 403, message: "Input was flagged by moderation", metadata: { reasons: ["harassment"] } } } },
34
+ bad_request: { status: 400, body: { error: { code: 400, message: "Invalid request: messages is required", metadata: {} } } },
35
+ in_body: { status: 200, body: { error: { code: 502, message: "Upstream provider failed", metadata: { error_type: "provider_error" } } } },
36
+ },
37
+ qwen: {
38
+ rate_limited: { status: 429, body: { error: { code: "Throttling.RateQuota", message: "Requests rate limit exceeded, please try again later.", type: "Throttling.RateQuota" } } },
39
+ token_rate: { status: 429, body: { error: { code: "Throttling.AllocationQuota", message: "Allocated quota exceeded, please increase your quota limit.", type: "Throttling.AllocationQuota" } } },
40
+ quota_exhausted: { status: 400, body: { error: { code: "Arrearage", message: "Access denied, please make sure your account is in good standing.", type: "Arrearage" } } },
41
+ free_tier: { status: 403, body: { error: { code: "AllocationQuota.FreeTierOnly", message: "The free tier of the model has been exhausted.", type: "AllocationQuota.FreeTierOnly" } } },
42
+ invalid_key: { status: 401, body: { error: { code: "invalid_api_key", message: "Incorrect API key provided.", type: "invalid_request_error" } } },
43
+ overloaded: { status: 500, body: { error: { code: "InternalError", message: "An internal error has occured, please try again later.", type: "InternalError" } } },
44
+ bad_request: { status: 400, body: { error: { code: "InvalidParameter", message: "Range of input length should be [1, 30720]", type: "InvalidParameter" } } },
45
+ },
46
+ deepseek: {
47
+ rate_limited: { status: 429, body: { error: { message: "Rate Limit Reached", type: "rate_limit_reached_error", param: null, code: "rate_limit_reached" } } },
48
+ quota_exhausted: { status: 402, body: { error: { message: "Insufficient Balance", type: "unknown_error", param: null, code: "invalid_request_error" } } },
49
+ invalid_key: { status: 401, body: { error: { message: "Authentication Fails, Your api key: ****abcd is invalid", type: "authentication_error", param: null, code: "invalid_request_error" } } },
50
+ overloaded: { status: 503, body: { error: { message: "Server Overloaded", type: "server_error", param: null, code: "server_overloaded" } } },
51
+ bad_request: { status: 422, body: { error: { message: "Invalid Parameters", type: "invalid_request_error", param: null, code: "invalid_request_error" } } },
52
+ },
53
+ gemini: {
54
+ rate_limited: { status: 429, body: [{ error: { code: 429, message: "You exceeded your current quota, please check your plan and billing details. Quota exceeded for metric: generate_content_free_tier_requests, limit: 15 per minute", status: "RESOURCE_EXHAUSTED" } }] },
55
+ quota_exhausted: { status: 429, body: { error: { code: "quota_exceeded", message: "Quota exceeded for metric: generate_content_free_tier_requests, limit: 1500 per day" } } },
56
+ prepay: { status: 402, body: { error: { code: "payment_required", message: "Your Prepay credit balance is depleted." } } },
57
+ invalid_key: { status: 400, body: [{ error: { code: 400, message: "API key not valid. Please pass a valid API key.", status: "INVALID_ARGUMENT" } }] },
58
+ leaked_key: { status: 403, body: { error: { code: "permission_denied", message: "Your API key was reported as leaked. Please use another API key." } } },
59
+ overloaded: { status: 503, body: { error: { code: "service_unavailable", message: "The model is overloaded. Please try again later." } } },
60
+ bad_request: { status: 400, body: { error: { code: "invalid_request", message: "Request contains an invalid argument." } } },
61
+ },
62
+ groq: {
63
+ rate_limited: { status: 429, headers: { "retry-after": "2", "x-ratelimit-remaining-tokens": "0", "x-ratelimit-reset-tokens": "7.66s" }, body: { error: { message: "Rate limit reached for model on tokens per minute (TPM)", type: "tokens", code: "rate_limit_exceeded" } } },
64
+ quota_exhausted: { status: 429, headers: { "x-ratelimit-remaining-requests": "0", "x-ratelimit-reset-requests": "2m59.56s" }, body: { error: { message: "Rate limit reached for model on requests per day (RPD)", type: "requests", code: "rate_limit_exceeded" } } },
65
+ invalid_key: { status: 401, body: { error: { message: "Invalid API Key", type: "invalid_request_error", code: "invalid_api_key" } } },
66
+ overloaded: { status: 498, body: { error: { message: "Flex tier capacity exceeded", type: "capacity_exceeded" } } },
67
+ bad_request: { status: 400, body: { error: { message: "'messages.0.name' : property 'name' is unsupported", type: "invalid_request_error" } } },
68
+ },
69
+ "openai-compatible": {
70
+ rate_limited: { status: 429, body: { error: { message: "too many requests", type: "rate_limit_error" } } },
71
+ quota_exhausted: { status: 402, body: { error: { message: "payment required", type: "payment_required" } } },
72
+ invalid_key: { status: 401, body: { error: { message: "invalid api key", type: "authentication_error" } } },
73
+ overloaded: { status: 503, body: { error: { message: "no backend available", type: "server_error" } } },
74
+ bad_request: { status: 400, body: { error: { message: "bad request", type: "invalid_request_error" } } },
75
+ },
76
+ });
77
+ function isRecord(value) {
78
+ return typeof value === "object" && value !== null && !Array.isArray(value);
79
+ }
1
80
  function wait(ms, signal) {
2
81
  return new Promise((resolve, reject) => {
82
+ const aborted = () => (signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
3
83
  if (signal?.aborted === true) {
4
- reject(signal.reason);
84
+ reject(aborted());
5
85
  return;
6
86
  }
7
87
  const timer = setTimeout(() => {
@@ -10,7 +90,7 @@ function wait(ms, signal) {
10
90
  }, ms);
11
91
  const stop = () => {
12
92
  clearTimeout(timer);
13
- reject(signal?.reason);
93
+ reject(aborted());
14
94
  };
15
95
  signal?.addEventListener("abort", stop, { once: true });
16
96
  });
@@ -28,27 +108,73 @@ function headerRecord(init) {
28
108
  });
29
109
  return out;
30
110
  }
31
- function streamBody(reply, model, signal) {
111
+ function openAiFrames(reply, model, pieces) {
32
112
  const options = reply.stream ?? {};
33
- const pieces = options.chunks ?? (reply.text ?? "ok").split(/(?<= )/);
34
- const encoder = new TextEncoder();
35
113
  const base = { id: "fake-stream", object: "chat.completion.chunk", created: 0, model };
36
- const frames = [JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })];
114
+ const frame = (delta, finish = null) => ({ data: JSON.stringify({ ...base, choices: [{ index: 0, delta, finish_reason: finish }] }) });
115
+ const frames = [frame({ role: "assistant" })];
37
116
  pieces.forEach((piece, index) => {
38
117
  if (options.stallAfter === index)
39
118
  frames.push("stall");
40
119
  if (options.failAfter === index)
41
120
  frames.push("fail");
42
- frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: piece }, finish_reason: null }] }));
121
+ frames.push(frame({ content: piece }));
43
122
  });
44
123
  if (options.failAfter !== undefined && options.failAfter >= pieces.length)
45
124
  frames.push("fail");
46
- frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }));
125
+ (reply.toolCalls ?? []).forEach((call, index) => {
126
+ frames.push(frame({ tool_calls: [{ index, id: call.id, type: "function", function: { name: call.name, arguments: "" } }] }));
127
+ for (const part of call.argumentChunks ?? [call.arguments])
128
+ frames.push(frame({ tool_calls: [{ index, function: { arguments: part } }] }));
129
+ });
130
+ frames.push(frame({}, reply.toolCalls === undefined ? "stop" : "tool_calls"));
47
131
  if (options.includeUsage !== false) {
48
132
  const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length };
49
- frames.push(JSON.stringify({ ...base, choices: [], usage }));
133
+ frames.push({ data: JSON.stringify({ ...base, choices: [], usage }) });
134
+ }
135
+ frames.push({ data: "[DONE]" });
136
+ return frames;
137
+ }
138
+ function anthropicFrames(reply, model, pieces) {
139
+ const options = reply.stream ?? {};
140
+ const event = (name, data) => ({ event: name, data: JSON.stringify({ type: name, ...data }) });
141
+ const input = reply.usage?.prompt_tokens ?? 10;
142
+ const frames = [event("message_start", { message: { id: "msg_fake", type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { input_tokens: input, output_tokens: 1 } } }), event("ping", {})];
143
+ let block = 0;
144
+ if (pieces.length > 0) {
145
+ frames.push(event("content_block_start", { index: block, content_block: { type: "text", text: "" } }));
146
+ pieces.forEach((piece, index) => {
147
+ if (options.stallAfter === index)
148
+ frames.push("stall");
149
+ if (options.failAfter === index)
150
+ frames.push("fail");
151
+ frames.push(event("content_block_delta", { index: block, delta: { type: "text_delta", text: piece } }));
152
+ });
153
+ frames.push(event("content_block_stop", { index: block }));
154
+ block++;
155
+ }
156
+ if (options.failAfter !== undefined && options.failAfter >= pieces.length)
157
+ frames.push("fail");
158
+ for (const call of reply.toolCalls ?? []) {
159
+ frames.push(event("content_block_start", { index: block, content_block: { type: "tool_use", id: call.id, name: call.name, input: {} } }));
160
+ for (const part of call.argumentChunks ?? [call.arguments])
161
+ frames.push(event("content_block_delta", { index: block, delta: { type: "input_json_delta", partial_json: part } }));
162
+ frames.push(event("content_block_stop", { index: block }));
163
+ block++;
50
164
  }
51
- frames.push("[DONE]");
165
+ const output = reply.usage?.completion_tokens ?? pieces.length;
166
+ frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: { output_tokens: output } }));
167
+ frames.push(event("message_stop", {}));
168
+ return frames;
169
+ }
170
+ function streamBody(reply, call, signal) {
171
+ const options = reply.stream ?? {};
172
+ const pieces = options.chunks ?? (reply.toolCalls !== undefined && reply.text === undefined ? [] : (reply.text ?? "ok").split(/(?<= )/));
173
+ const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call.model, pieces) : openAiFrames(reply, call.model, pieces);
174
+ const failure = call.protocol === "anthropic"
175
+ ? `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })}\n\n`
176
+ : `data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`;
177
+ const encoder = new TextEncoder();
52
178
  let index = 0;
53
179
  return new ReadableStream({
54
180
  async pull(controller) {
@@ -69,12 +195,12 @@ function streamBody(reply, model, signal) {
69
195
  controller.close();
70
196
  return;
71
197
  }
72
- if (frame !== "fail") {
73
- controller.enqueue(encoder.encode(`data: ${frame}\n\n`));
198
+ if (frame !== "fail" && frame !== "stall") {
199
+ controller.enqueue(encoder.encode(`${frame.event === undefined ? "" : `event: ${frame.event}\n`}data: ${frame.data}\n\n`));
74
200
  return;
75
201
  }
76
202
  if (options.failWith === "error-event") {
77
- controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`));
203
+ controller.enqueue(encoder.encode(failure));
78
204
  index = frames.length;
79
205
  return;
80
206
  }
@@ -82,12 +208,22 @@ function streamBody(reply, model, signal) {
82
208
  },
83
209
  });
84
210
  }
211
+ function errorBody(status, protocol) {
212
+ const type = status === 429 ? "rate_limit_exceeded" : "fake_error";
213
+ if (protocol === "anthropic")
214
+ return { type: "error", error: { type: status === 429 ? "rate_limit_error" : type, message: `fake ${String(status)}` } };
215
+ return { error: { message: `fake ${String(status)}`, type } };
216
+ }
217
+ function toolUseBlock(tool) {
218
+ const input = JSON.parse(tool.arguments);
219
+ return { type: "tool_use", id: tool.id, name: tool.name, input };
220
+ }
85
221
  function jsonBody(reply, call) {
86
222
  if (reply.body !== undefined)
87
223
  return reply.body;
88
224
  const status = reply.status ?? 200;
89
225
  if (status >= 400)
90
- return { error: { message: `fake ${String(status)}`, type: status === 429 ? "rate_limit_exceeded" : "fake_error" } };
226
+ return errorBody(status, call.protocol);
91
227
  if (call.path.endsWith("/embeddings")) {
92
228
  const input = call.body.input;
93
229
  const count = Array.isArray(input) ? input.length : 1;
@@ -98,17 +234,35 @@ function jsonBody(reply, call) {
98
234
  usage: reply.usage ?? { prompt_tokens: 4 * count, total_tokens: 4 * count },
99
235
  };
100
236
  }
101
- const text = reply.text ?? "ok";
237
+ const text = reply.text ?? (reply.toolCalls === undefined ? "ok" : "");
238
+ const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 };
239
+ if (call.protocol === "anthropic") {
240
+ const content = [
241
+ ...(text === "" ? [] : [{ type: "text", text }]),
242
+ ...(reply.toolCalls ?? []).map(toolUseBlock),
243
+ ];
244
+ return {
245
+ id: "msg_fake",
246
+ type: "message",
247
+ role: "assistant",
248
+ model: call.model,
249
+ content,
250
+ stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use",
251
+ stop_sequence: null,
252
+ usage: { input_tokens: usage.prompt_tokens, output_tokens: usage.completion_tokens ?? 0 },
253
+ };
254
+ }
255
+ const calls = (reply.toolCalls ?? []).map((tool) => ({ id: tool.id, type: "function", function: { name: tool.name, arguments: tool.arguments } }));
102
256
  return {
103
257
  id: `fake-${String(Date.now())}`,
104
258
  object: "chat.completion",
105
259
  created: 0,
106
260
  model: call.model,
107
- choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }],
108
- usage: reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 },
261
+ choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
262
+ usage,
109
263
  };
110
264
  }
111
- /** Creates a fake OpenAI-compatible provider: deterministic replies, latency, errors, 429s and streams by script. */
265
+ /** Creates a fake provider: deterministic replies, latency, errors, 429s and streams by script, in either protocol. */
112
266
  export function createFakeProvider() {
113
267
  let scripts = [];
114
268
  const calls = [];
@@ -125,13 +279,15 @@ export function createFakeProvider() {
125
279
  async fetch(input, init) {
126
280
  const url = new URL(input);
127
281
  const headers = headerRecord(init);
128
- const body = (typeof init.body === "string" ? JSON.parse(init.body) : {});
282
+ const parsed = typeof init.body === "string" ? JSON.parse(init.body) : {};
283
+ const body = isRecord(parsed) ? parsed : {};
129
284
  const call = {
130
285
  url: input,
131
286
  path: url.pathname,
132
- key: (headers.authorization ?? "").replace(/^Bearer /, ""),
287
+ key: headers["x-api-key"] ?? (headers.authorization ?? "").replace(/^Bearer /, ""),
133
288
  model: typeof body.model === "string" ? body.model : "",
134
289
  stream: body.stream === true,
290
+ protocol: url.pathname.endsWith("/messages") ? "anthropic" : "openai",
135
291
  headers,
136
292
  body,
137
293
  };
@@ -144,7 +300,7 @@ export function createFakeProvider() {
144
300
  throw new TypeError("fetch failed");
145
301
  const status = reply.status ?? 200;
146
302
  if (call.stream && status < 400 && reply.body === undefined) {
147
- return new Response(streamBody(reply, call.model, init.signal), { status, headers: { "content-type": "text/event-stream", ...reply.headers } });
303
+ return new Response(streamBody(reply, call, init.signal), { status, headers: { "content-type": "text/event-stream", ...reply.headers } });
148
304
  }
149
305
  return new Response(JSON.stringify(jsonBody(reply, call)), { status, headers: { "content-type": "application/json", ...reply.headers } });
150
306
  },
@@ -12,6 +12,8 @@ export type KeyStoreConfig = {
12
12
  readonly type: "sqlite";
13
13
  readonly path: string;
14
14
  };
15
+ /** Where the pool keeps breakers, blacklists, headroom and budgets: in memory (a restart forgets them) or in a SQLite file. */
16
+ export type StateStoreConfig = KeyStoreConfig;
15
17
  /** The `gateway` section of a config file, with every default applied and paths resolved from the file's folder. */
16
18
  export interface GatewaySettings {
17
19
  readonly port: number;
@@ -24,14 +26,17 @@ export interface GatewaySettings {
24
26
  export interface GatewayFileConfig {
25
27
  readonly pool: PoolConfig;
26
28
  readonly gateway: GatewaySettings;
29
+ /** The pool's state store; `serveGateway` opens it and hands it to the router. */
30
+ readonly stateStore: StateStoreConfig;
27
31
  }
28
32
  /** Reads a referenced secret now: the variable's value or the file's first line, trimmed; undefined when absent or empty. */
29
33
  export declare function readSecret(ref: SecretRef, env?: NodeJS.ProcessEnv): Promise<string | undefined>;
30
34
  /**
31
35
  * Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
32
36
  * keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
33
- * no restart). A field that would hold a secret's value, or a provider header that would carry one, is refused.
37
+ * no restart); `keysFromEnv: PREFIX` finds the canonical `PREFIX_API_KEY`, `_2`, ... variables in `env` (src/env.ts).
38
+ * A field that would hold a secret's value, or a provider header that would carry one, is refused.
34
39
  */
35
- export declare function parseGatewayConfig(raw: unknown, base?: string): GatewayFileConfig;
40
+ export declare function parseGatewayConfig(raw: unknown, base?: string, env?: NodeJS.ProcessEnv): GatewayFileConfig;
36
41
  /** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
37
- export declare function loadGatewayConfig(path: string): Promise<GatewayFileConfig>;
42
+ export declare function loadGatewayConfig(path: string, env?: NodeJS.ProcessEnv): Promise<GatewayFileConfig>;
@@ -2,8 +2,11 @@ import { readFile } from "node:fs/promises";
2
2
  import { dirname, extname, resolve } from "node:path";
3
3
  import { parse as parseYaml } from "yaml";
4
4
  import { AiRouterError } from "../errors.js";
5
+ import { keysFromEnv } from "../env.js";
6
+ import { PROVIDER_PRESETS } from "../providers.js";
5
7
  const CAPABILITIES = { list: { enum: ["chat", "streaming", "tools", "json", "embeddings", "vision"] } };
6
8
  const BREAKER = { fields: { failureThreshold: "number", cooldownMs: "number", maxCooldownMs: "number" } };
9
+ const CANDIDATE = { fields: { model: "string", provider: "string", weight: "number" }, required: ["model"] };
7
10
  const POLICY = {
8
11
  fields: {
9
12
  model: "string",
@@ -15,9 +18,12 @@ const POLICY = {
15
18
  budget: { fields: { maxCost: "number" } },
16
19
  providers: { list: "string" },
17
20
  require: CAPABILITIES,
21
+ candidates: { list: { anyOf: [CANDIDATE, { list: CANDIDATE }] } },
22
+ pinned: "boolean",
18
23
  },
19
24
  };
20
25
  const SECRET = { fields: { env: "string", file: "string" } };
26
+ const STORE = { fields: { type: { enum: ["memory", "sqlite"] }, path: "string" }, required: ["type"] };
21
27
  const FILE = {
22
28
  fields: {
23
29
  gateway: {
@@ -28,14 +34,18 @@ const FILE = {
28
34
  host: "string",
29
35
  maxBodyBytes: "number",
30
36
  adminToken: SECRET,
31
- keyStore: { fields: { type: { enum: ["memory", "sqlite"] }, path: "string" }, required: ["type"] },
37
+ keyStore: STORE,
32
38
  },
33
39
  },
34
40
  providers: {
35
41
  list: {
36
42
  fields: {
37
43
  id: "string",
44
+ kind: { enum: Object.keys(PROVIDER_PRESETS) },
38
45
  baseUrl: "string",
46
+ defaultMaxTokens: "number",
47
+ budget: { fields: { costPerDay: "number", costPerMonth: "number" } },
48
+ keysFromEnv: "string",
39
49
  headers: { map: "string" },
40
50
  weight: "number",
41
51
  maxConcurrency: "number",
@@ -49,12 +59,14 @@ const FILE = {
49
59
  cost: { fields: { input: "number", output: "number" }, required: ["input", "output"] },
50
60
  capabilities: CAPABILITIES,
51
61
  weight: "number",
62
+ identity: "string",
63
+ dimensions: "number",
52
64
  },
53
65
  required: ["id"],
54
66
  },
55
67
  },
56
68
  },
57
- required: ["id", "baseUrl", "keys", "models"],
69
+ required: ["id", "models"],
58
70
  },
59
71
  },
60
72
  defaults: POLICY,
@@ -66,11 +78,13 @@ const FILE = {
66
78
  backoff: { fields: { baseMs: "number", factor: "number", maxMs: "number" } },
67
79
  authDisableMs: "number|null",
68
80
  rateLimitCooldownMs: "number",
81
+ quotaBlacklistMs: "number",
69
82
  },
70
83
  },
71
84
  selection: { fields: { latencyTolerance: "number", latencySlackMs: "number", ewmaAlpha: "number", lowHeadroom: "number" } },
72
85
  telemetry: { fields: { includeContent: "boolean" } },
73
86
  storeNamespace: "string",
87
+ stateStore: STORE,
74
88
  },
75
89
  required: ["providers"],
76
90
  };
@@ -101,6 +115,19 @@ function check(value, schema, where) {
101
115
  throw configError(`${where} must be true or false`);
102
116
  return;
103
117
  }
118
+ if ("anyOf" in schema) {
119
+ const errors = [];
120
+ for (const option of schema.anyOf) {
121
+ try {
122
+ check(value, option, where);
123
+ return;
124
+ }
125
+ catch (error) {
126
+ errors.push(error instanceof Error ? error.message : String(error));
127
+ }
128
+ }
129
+ throw configError(errors[0] ?? `${where} matches no allowed shape`);
130
+ }
104
131
  if ("enum" in schema) {
105
132
  if (typeof value !== "string" || !schema.enum.includes(value))
106
133
  throw configError(`${where} must be one of ${schema.enum.join(", ")}`);
@@ -162,9 +189,10 @@ export async function readSecret(ref, env = process.env) {
162
189
  /**
163
190
  * Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
164
191
  * keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
165
- * no restart). A field that would hold a secret's value, or a provider header that would carry one, is refused.
192
+ * no restart); `keysFromEnv: PREFIX` finds the canonical `PREFIX_API_KEY`, `_2`, ... variables in `env` (src/env.ts).
193
+ * A field that would hold a secret's value, or a provider header that would carry one, is refused.
166
194
  */
167
- export function parseGatewayConfig(raw, base = process.cwd()) {
195
+ export function parseGatewayConfig(raw, base = process.cwd(), env = process.env) {
168
196
  check(raw, FILE, "config");
169
197
  const file = raw;
170
198
  const providers = file.providers.map((provider, index) => {
@@ -173,7 +201,12 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
173
201
  if (SECRET_HEADER.test(name))
174
202
  throw configError(`${where}.headers.${name} would carry a secret in the config file; give keys under keys[] by reference`);
175
203
  }
176
- const keys = provider.keys.map((key, at) => {
204
+ const listed = Array.isArray(provider.keys) ? provider.keys.filter(isRecord) : [];
205
+ const prefix = typeof provider.keysFromEnv === "string" ? provider.keysFromEnv : undefined;
206
+ if (prefix === undefined && listed.length === 0)
207
+ throw configError(`${where} needs keys or keysFromEnv`);
208
+ const fromEnv = prefix === undefined ? [] : keysFromEnv(prefix, String(provider.id), env);
209
+ const declared = listed.map((key, at) => {
177
210
  const ref = secretRef({ env: key.env, file: key.file }, `${where}.keys[${String(at)}]`, base);
178
211
  const common = {
179
212
  alias: key.alias,
@@ -182,7 +215,8 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
182
215
  };
183
216
  return "env" in ref ? { ...common, env: ref.env } : { ...common, resolve: () => readSecret(ref) };
184
217
  });
185
- return { ...provider, keys };
218
+ const rest = Object.fromEntries(Object.entries(provider).filter(([field]) => field !== "keysFromEnv"));
219
+ return { ...rest, keys: [...declared, ...fromEnv] };
186
220
  });
187
221
  const gateway = file.gateway ?? {};
188
222
  const store = (gateway.keyStore ?? { type: "memory" });
@@ -206,7 +240,10 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
206
240
  throw configError("config.gateway.maxBodyBytes must be a positive integer");
207
241
  const pool = { ...file, providers };
208
242
  delete pool.gateway;
243
+ delete pool.stateStore;
244
+ const state = storeOf(file.stateStore, "config.stateStore", base);
209
245
  return {
246
+ stateStore: state,
210
247
  pool: pool,
211
248
  gateway: {
212
249
  port,
@@ -217,8 +254,16 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
217
254
  },
218
255
  };
219
256
  }
257
+ /** A store setting with its path resolved from the config file's folder; memory when absent. */
258
+ function storeOf(value, where, base) {
259
+ if (!isRecord(value) || value.type !== "sqlite")
260
+ return { type: "memory" };
261
+ if (typeof value.path !== "string")
262
+ throw configError(`${where}.path is required for a sqlite store`);
263
+ return { type: "sqlite", path: value.path === ":memory:" ? ":memory:" : resolve(base, value.path) };
264
+ }
220
265
  /** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
221
- export async function loadGatewayConfig(path) {
266
+ export async function loadGatewayConfig(path, env = process.env) {
222
267
  const full = resolve(path);
223
268
  let text;
224
269
  try {
@@ -234,5 +279,5 @@ export async function loadGatewayConfig(path) {
234
279
  catch (error) {
235
280
  throw configError(`the config file ${full} does not parse: ${error instanceof Error ? error.message : "unknown error"}`);
236
281
  }
237
- return parseGatewayConfig(raw, dirname(full));
282
+ return parseGatewayConfig(raw, dirname(full), env);
238
283
  }
@@ -43,5 +43,7 @@ export interface AiRouterGateway {
43
43
  /** Stops listening, ends open connections and releases the router subscription; the key store stays open. */
44
44
  close(): Promise<void>;
45
45
  }
46
+ /** The Retry-After of a pool whose keys are all blacklisted: until the earliest returns, else this many seconds. */
47
+ export declare const BLACKLISTED_RETRY_AFTER_S = 300;
46
48
  /** Creates the gateway over a router; nothing listens until `listen`. */
47
49
  export declare function createGateway(router: AiRouter, options: GatewayOptions): AiRouterGateway;
@@ -39,6 +39,11 @@ function sendError(response, error, headers = {}) {
39
39
  function secondsUntil(at, now) {
40
40
  return Math.max(1, Math.ceil((at - now) / 1000));
41
41
  }
42
+ /** The Retry-After of a pool whose keys are all blacklisted: until the earliest returns, else this many seconds. */
43
+ export const BLACKLISTED_RETRY_AFTER_S = 300;
44
+ function retryAfterOf(error, now) {
45
+ return error.retryAt === undefined ? BLACKLISTED_RETRY_AFTER_S : secondsUntil(error.retryAt, now);
46
+ }
42
47
  /** Maps a router failure to the status a client can act on; the message is the router's, which never holds a key value. */
43
48
  function fromRouterError(error, now) {
44
49
  if (!(error instanceof AiRouterError))
@@ -55,12 +60,16 @@ function fromRouterError(error, now) {
55
60
  return new GatewayError(503, "pool_unavailable", `no model of the pool can serve this request: ${error.message}`, retry);
56
61
  case "rate_limit":
57
62
  return new GatewayError(503, "pool_rate_limited", `every key of the pool is rate limited: ${error.message}`, retry);
63
+ case "quota":
64
+ // Every key that could serve is blacklisted for a spent quota or budget: come back when the first one returns.
65
+ return new GatewayError(503, "pool_quota_exhausted", `every upstream key for this request has exhausted its quota or credit: ${error.message}`, { retryAfter: retryAfterOf(error, now) });
58
66
  case "deadline":
59
67
  return new GatewayError(504, "pool_timeout", error.message);
60
68
  case "cancelled":
61
69
  return new GatewayError(499, "cancelled", error.message);
62
70
  case "auth":
63
- return new GatewayError(502, "pool_auth_failed", "no upstream key of the pool was accepted");
71
+ // Every key is refused or unset, until an operator seals a new secret: the client can only come back later.
72
+ return new GatewayError(503, "pool_auth_failed", "no upstream key of the pool is accepted: each is blacklisted until its secret changes", { retryAfter: retryAfterOf(error, now) });
64
73
  case "transient":
65
74
  case "timeout":
66
75
  case "network":
@@ -186,19 +195,26 @@ export function createGateway(router, options) {
186
195
  if (typeof name !== "string" || name === "")
187
196
  throw new GatewayError(400, "missing_model", "model is required: a profile name or a model id");
188
197
  const pinned = key.allowFallback ? {} : { allowFallback: false };
189
- if (catalog.profiles.some((profile) => profile.name === name)) {
198
+ const needs = operation === "embeddings" ? "embeddings" : "chat";
199
+ const profile = catalog.profiles.find((entry) => entry.name === name);
200
+ if (profile !== undefined) {
190
201
  if (!allows(key.profiles, name))
191
202
  throw new GatewayError(403, "model_not_allowed", `this key may not call profile "${name}"`);
203
+ if (profile.serves.length > 0 && !profile.serves.includes(needs))
204
+ throw new GatewayError(400, "model_not_supported", `the profile "${name}" does not serve ${operation}`);
192
205
  return { options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true };
193
206
  }
194
207
  const model = modelNamed(catalog, name);
195
208
  if (model === undefined)
196
209
  throw new GatewayError(404, "model_not_found", `the model "${name}" does not exist`);
210
+ // A model the key's allowlist does not name is refused, never remapped to a profile.
197
211
  if (!modelAllowed(key, model))
198
212
  throw new GatewayError(403, "model_not_allowed", `this key may not call model "${name}"`);
199
- const needs = operation === "embeddings" ? "embeddings" : "chat";
200
213
  if (!model.capabilities.includes(needs))
201
214
  throw new GatewayError(400, "model_not_supported", `the model "${name}" does not serve ${operation}`);
215
+ // An embeddings call never falls back: another model is another vector space.
216
+ if (operation === "embeddings")
217
+ return { options: { policy: { model: name, allowFallback: false } }, dropModel: false };
202
218
  // A direct call falls back only to the pool's default fallbacks this key may call itself.
203
219
  const fallbacks = catalog.defaults.fallbacks.filter((fallback) => {
204
220
  const entry = modelNamed(catalog, fallback);
@@ -1,10 +1,19 @@
1
1
  export { AiRouterError } from "../errors.js";
2
2
  export type { AttemptFailure } from "../errors.js";
3
3
  export { createRouter } from "../router.js";
4
+ export { classifyFailure, readProviderError } from "../classify.js";
5
+ export type { Classified, ProviderErrorInfo } from "../classify.js";
6
+ export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
7
+ export type { EnvPoolSpec, EnvProviderSpec, EnvSource, VirtualKeyEnvSpec } from "../env.js";
8
+ export { PROVIDER_PRESETS } from "../providers.js";
9
+ export type { Protocol, ProviderPreset, QuotaWindow } from "../providers.js";
10
+ export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
11
+ export type { ClosableStateStore, RedisLike } from "../state-stores.js";
12
+ export { EMPTY_RECORD, MemoryStateStore, parseHealthRecord, toHealthRecord } from "../store.js";
4
13
  export type { HealthRecord, StateStore } from "../store.js";
5
14
  export type * from "../types.js";
6
15
  export { loadGatewayConfig, parseGatewayConfig } from "./config-file.js";
7
- export type { GatewayFileConfig, GatewaySettings, KeyStoreConfig, SecretRef } from "./config-file.js";
16
+ export type { GatewayFileConfig, GatewaySettings, KeyStoreConfig, SecretRef, StateStoreConfig } from "./config-file.js";
8
17
  export { createGateway } from "./gateway.js";
9
18
  export type { AiRouterGateway, GatewayOptions } from "./gateway.js";
10
19
  export { KeyAdmin, KeyAdminError, MemoryKeyStore, MountedKeyError } from "./keys.js";
@@ -1,5 +1,10 @@
1
1
  export { AiRouterError } from "../errors.js";
2
2
  export { createRouter } from "../router.js";
3
+ export { classifyFailure, readProviderError } from "../classify.js";
4
+ export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
5
+ export { PROVIDER_PRESETS } from "../providers.js";
6
+ export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
7
+ export { EMPTY_RECORD, MemoryStateStore, parseHealthRecord, toHealthRecord } from "../store.js";
3
8
  export { loadGatewayConfig, parseGatewayConfig } from "./config-file.js";
4
9
  export { createGateway } from "./gateway.js";
5
10
  export { KeyAdmin, KeyAdminError, MemoryKeyStore, MountedKeyError } from "./keys.js";
@@ -1,4 +1,5 @@
1
1
  import { createRouter } from "../router.js";
2
+ import { openSqliteStateStore } from "../state-stores.js";
2
3
  import { loadGatewayConfig, readSecret } from "./config-file.js";
3
4
  import { createGateway } from "./gateway.js";
4
5
  import { MemoryKeyStore } from "./keys.js";
@@ -9,8 +10,10 @@ export async function openKeyStore(config) {
9
10
  }
10
11
  /** Loads a config file, builds the router and the key store, and listens: the whole `ai-router serve`. */
11
12
  export async function serveGateway(options) {
12
- const file = await loadGatewayConfig(options.config);
13
- const router = createRouter(options.fetch === undefined ? file.pool : { ...file.pool, fetch: options.fetch });
13
+ const file = await loadGatewayConfig(options.config, options.env);
14
+ // A durable state store keeps blacklisted keys out across a restart; memory is the router's own default.
15
+ const state = file.stateStore.type === "sqlite" ? await openSqliteStateStore(file.stateStore.path) : undefined;
16
+ const router = createRouter({ ...file.pool, ...(state === undefined ? {} : { store: state }), ...(options.fetch === undefined ? {} : { fetch: options.fetch }) });
14
17
  const keys = await openKeyStore(file.gateway.keyStore);
15
18
  const admin = file.gateway.adminToken;
16
19
  const gateway = createGateway(router, {
@@ -27,12 +30,14 @@ export async function serveGateway(options) {
27
30
  async close() {
28
31
  await gateway.close();
29
32
  await keys.close?.();
33
+ state?.close();
30
34
  },
31
35
  };
32
36
  }
33
37
  catch (error) {
34
38
  await gateway.close();
35
39
  await keys.close?.();
40
+ state?.close();
36
41
  throw error;
37
42
  }
38
43
  }
package/dist/headers.d.ts CHANGED
@@ -11,5 +11,5 @@ export interface RateLimitInfo {
11
11
  export declare function parseDuration(value: string): number | null;
12
12
  /** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
13
13
  export declare function retryAt(headers: Headers, now: number): number;
14
- /** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
14
+ /** Reads the `x-ratelimit-*` (OpenAI, Groq, OpenRouter), `anthropic-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
15
15
  export declare function readRateLimit(headers: Headers, now: number): RateLimitInfo;