@starci/ai-router 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +12 -3
package/dist/fake.js
CHANGED
|
@@ -1,7 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Each provider's real error answers (status, body, headers) as its docs showed them on 2026-10-31 (docs/design.md,
|
|
3
|
+
* the classification table), for a spec to script with `fake.on(match, PROVIDER_ERRORS.anthropic.quota_exhausted)`.
|
|
4
|
+
* Extra cases name a provider's second shape of one class (Anthropic's monthly spend cap, OpenRouter's free daily cap).
|
|
5
|
+
*/
|
|
6
|
+
export const PROVIDER_ERRORS = Object.freeze({
|
|
7
|
+
openai: {
|
|
8
|
+
rate_limited: { status: 429, headers: { "retry-after": "2", "x-ratelimit-remaining-requests": "0", "x-ratelimit-reset-requests": "2s" }, body: { error: { message: "Rate limit reached for requests", type: "requests", param: null, code: "rate_limit_exceeded" } } },
|
|
9
|
+
quota_exhausted: { status: 429, body: { error: { message: "You exceeded your current quota, please check your plan and billing details.", type: "insufficient_quota", param: null, code: "insufficient_quota" } } },
|
|
10
|
+
credit_exhausted: { status: 429, body: { error: { message: "Your credit balance is exhausted.", type: "insufficient_quota", param: null, code: "credit_balance_exhausted" } } },
|
|
11
|
+
usage_limit: { status: 429, body: { error: { message: "Your organization reached its monthly usage limit.", type: "insufficient_quota", param: null, code: "organization_usage_limit_exceeded" } } },
|
|
12
|
+
invalid_key: { status: 401, body: { error: { message: "Incorrect API key provided: sk-proj-****abcd.", type: "invalid_request_error", param: null, code: "invalid_api_key" } } },
|
|
13
|
+
overloaded: { status: 503, headers: { "retry-after": "1" }, body: { error: { message: "The server is overloaded.", type: "server_error", param: null, code: "server_is_overloaded" } } },
|
|
14
|
+
bad_request: { status: 400, body: { error: { message: "This model's maximum context length is 128000 tokens.", type: "invalid_request_error", param: "messages", code: "context_length_exceeded" } } },
|
|
15
|
+
},
|
|
16
|
+
anthropic: {
|
|
17
|
+
rate_limited: { status: 429, headers: { "retry-after": "3", "anthropic-ratelimit-requests-remaining": "0", "anthropic-ratelimit-requests-reset": "2026-10-31T12:00:03Z" }, body: { type: "error", error: { type: "rate_limit_error", message: "Number of request tokens has exceeded your per-minute rate limit" }, request_id: "req_fake" } },
|
|
18
|
+
quota_exhausted: { status: 402, body: { type: "error", error: { type: "billing_error", message: "Your credit balance is too low to access the Anthropic API." }, request_id: "req_fake" } },
|
|
19
|
+
spend_cap: { status: 429, body: { type: "error", error: { type: "rate_limit_error", message: "You have reached your monthly spend limit.", details: { error_code: "enforced_spend_limit_reached" } }, request_id: "req_fake" } },
|
|
20
|
+
usage_limit: { status: 400, body: { type: "error", error: { type: "invalid_request_error", message: "You have reached your specified API usage limits. You will regain access on 2026-11-01 at 00:00 UTC." }, request_id: "req_fake" } },
|
|
21
|
+
invalid_key: { status: 401, body: { type: "error", error: { type: "authentication_error", message: "invalid x-api-key" }, request_id: "req_fake" } },
|
|
22
|
+
overloaded: { status: 529, body: { type: "error", error: { type: "overloaded_error", message: "Overloaded" }, request_id: "req_fake" } },
|
|
23
|
+
bad_request: { status: 400, body: { type: "error", error: { type: "invalid_request_error", message: "prompt is too long: 1000001 tokens > 1000000 maximum" }, request_id: "req_fake" } },
|
|
24
|
+
},
|
|
25
|
+
openrouter: {
|
|
26
|
+
rate_limited: { status: 429, headers: { "X-RateLimit-Limit": "20", "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1961913600000" }, body: { error: { code: 429, message: "Rate limit exceeded", metadata: {} } } },
|
|
27
|
+
quota_exhausted: { status: 402, body: { error: { code: 402, message: "Insufficient credits. Add more using https://openrouter.ai/credits", metadata: { limit_source: "openrouter_credits" } } } },
|
|
28
|
+
key_limit: { status: 402, body: { error: { code: 402, message: "Key limit exceeded", metadata: { limit_source: "openrouter_key_limit" } } } },
|
|
29
|
+
in_flight_budget: { status: 402, headers: { "retry-after": "1" }, body: { error: { code: 402, message: "In-flight budget exceeded", metadata: { limit_source: "openrouter_in_flight_budget" } } } },
|
|
30
|
+
free_daily: { status: 429, body: { error: { code: 429, message: "Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day", metadata: {} } } },
|
|
31
|
+
invalid_key: { status: 401, body: { error: { code: 401, message: "User not found.", metadata: {} } } },
|
|
32
|
+
overloaded: { status: 502, body: { error: { code: 502, message: "Provider returned error", metadata: { provider_name: "upstream", raw: "overloaded" } } } },
|
|
33
|
+
moderation: { status: 403, body: { error: { code: 403, message: "Input was flagged by moderation", metadata: { reasons: ["harassment"] } } } },
|
|
34
|
+
bad_request: { status: 400, body: { error: { code: 400, message: "Invalid request: messages is required", metadata: {} } } },
|
|
35
|
+
in_body: { status: 200, body: { error: { code: 502, message: "Upstream provider failed", metadata: { error_type: "provider_error" } } } },
|
|
36
|
+
},
|
|
37
|
+
qwen: {
|
|
38
|
+
rate_limited: { status: 429, body: { error: { code: "Throttling.RateQuota", message: "Requests rate limit exceeded, please try again later.", type: "Throttling.RateQuota" } } },
|
|
39
|
+
token_rate: { status: 429, body: { error: { code: "Throttling.AllocationQuota", message: "Allocated quota exceeded, please increase your quota limit.", type: "Throttling.AllocationQuota" } } },
|
|
40
|
+
quota_exhausted: { status: 400, body: { error: { code: "Arrearage", message: "Access denied, please make sure your account is in good standing.", type: "Arrearage" } } },
|
|
41
|
+
free_tier: { status: 403, body: { error: { code: "AllocationQuota.FreeTierOnly", message: "The free tier of the model has been exhausted.", type: "AllocationQuota.FreeTierOnly" } } },
|
|
42
|
+
invalid_key: { status: 401, body: { error: { code: "invalid_api_key", message: "Incorrect API key provided.", type: "invalid_request_error" } } },
|
|
43
|
+
overloaded: { status: 500, body: { error: { code: "InternalError", message: "An internal error has occured, please try again later.", type: "InternalError" } } },
|
|
44
|
+
bad_request: { status: 400, body: { error: { code: "InvalidParameter", message: "Range of input length should be [1, 30720]", type: "InvalidParameter" } } },
|
|
45
|
+
},
|
|
46
|
+
deepseek: {
|
|
47
|
+
rate_limited: { status: 429, body: { error: { message: "Rate Limit Reached", type: "rate_limit_reached_error", param: null, code: "rate_limit_reached" } } },
|
|
48
|
+
quota_exhausted: { status: 402, body: { error: { message: "Insufficient Balance", type: "unknown_error", param: null, code: "invalid_request_error" } } },
|
|
49
|
+
invalid_key: { status: 401, body: { error: { message: "Authentication Fails, Your api key: ****abcd is invalid", type: "authentication_error", param: null, code: "invalid_request_error" } } },
|
|
50
|
+
overloaded: { status: 503, body: { error: { message: "Server Overloaded", type: "server_error", param: null, code: "server_overloaded" } } },
|
|
51
|
+
bad_request: { status: 422, body: { error: { message: "Invalid Parameters", type: "invalid_request_error", param: null, code: "invalid_request_error" } } },
|
|
52
|
+
},
|
|
53
|
+
gemini: {
|
|
54
|
+
rate_limited: { status: 429, body: [{ error: { code: 429, message: "You exceeded your current quota, please check your plan and billing details. Quota exceeded for metric: generate_content_free_tier_requests, limit: 15 per minute", status: "RESOURCE_EXHAUSTED" } }] },
|
|
55
|
+
quota_exhausted: { status: 429, body: { error: { code: "quota_exceeded", message: "Quota exceeded for metric: generate_content_free_tier_requests, limit: 1500 per day" } } },
|
|
56
|
+
prepay: { status: 402, body: { error: { code: "payment_required", message: "Your Prepay credit balance is depleted." } } },
|
|
57
|
+
invalid_key: { status: 400, body: [{ error: { code: 400, message: "API key not valid. Please pass a valid API key.", status: "INVALID_ARGUMENT" } }] },
|
|
58
|
+
leaked_key: { status: 403, body: { error: { code: "permission_denied", message: "Your API key was reported as leaked. Please use another API key." } } },
|
|
59
|
+
overloaded: { status: 503, body: { error: { code: "service_unavailable", message: "The model is overloaded. Please try again later." } } },
|
|
60
|
+
bad_request: { status: 400, body: { error: { code: "invalid_request", message: "Request contains an invalid argument." } } },
|
|
61
|
+
},
|
|
62
|
+
groq: {
|
|
63
|
+
rate_limited: { status: 429, headers: { "retry-after": "2", "x-ratelimit-remaining-tokens": "0", "x-ratelimit-reset-tokens": "7.66s" }, body: { error: { message: "Rate limit reached for model on tokens per minute (TPM)", type: "tokens", code: "rate_limit_exceeded" } } },
|
|
64
|
+
quota_exhausted: { status: 429, headers: { "x-ratelimit-remaining-requests": "0", "x-ratelimit-reset-requests": "2m59.56s" }, body: { error: { message: "Rate limit reached for model on requests per day (RPD)", type: "requests", code: "rate_limit_exceeded" } } },
|
|
65
|
+
invalid_key: { status: 401, body: { error: { message: "Invalid API Key", type: "invalid_request_error", code: "invalid_api_key" } } },
|
|
66
|
+
overloaded: { status: 498, body: { error: { message: "Flex tier capacity exceeded", type: "capacity_exceeded" } } },
|
|
67
|
+
bad_request: { status: 400, body: { error: { message: "'messages.0.name' : property 'name' is unsupported", type: "invalid_request_error" } } },
|
|
68
|
+
},
|
|
69
|
+
"openai-compatible": {
|
|
70
|
+
rate_limited: { status: 429, body: { error: { message: "too many requests", type: "rate_limit_error" } } },
|
|
71
|
+
quota_exhausted: { status: 402, body: { error: { message: "payment required", type: "payment_required" } } },
|
|
72
|
+
invalid_key: { status: 401, body: { error: { message: "invalid api key", type: "authentication_error" } } },
|
|
73
|
+
overloaded: { status: 503, body: { error: { message: "no backend available", type: "server_error" } } },
|
|
74
|
+
bad_request: { status: 400, body: { error: { message: "bad request", type: "invalid_request_error" } } },
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
function isRecord(value) {
|
|
78
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
79
|
+
}
|
|
1
80
|
function wait(ms, signal) {
|
|
2
81
|
return new Promise((resolve, reject) => {
|
|
82
|
+
const aborted = () => (signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
|
|
3
83
|
if (signal?.aborted === true) {
|
|
4
|
-
reject(
|
|
84
|
+
reject(aborted());
|
|
5
85
|
return;
|
|
6
86
|
}
|
|
7
87
|
const timer = setTimeout(() => {
|
|
@@ -10,7 +90,7 @@ function wait(ms, signal) {
|
|
|
10
90
|
}, ms);
|
|
11
91
|
const stop = () => {
|
|
12
92
|
clearTimeout(timer);
|
|
13
|
-
reject(
|
|
93
|
+
reject(aborted());
|
|
14
94
|
};
|
|
15
95
|
signal?.addEventListener("abort", stop, { once: true });
|
|
16
96
|
});
|
|
@@ -28,27 +108,73 @@ function headerRecord(init) {
|
|
|
28
108
|
});
|
|
29
109
|
return out;
|
|
30
110
|
}
|
|
31
|
-
function
|
|
111
|
+
function openAiFrames(reply, model, pieces) {
|
|
32
112
|
const options = reply.stream ?? {};
|
|
33
|
-
const pieces = options.chunks ?? (reply.text ?? "ok").split(/(?<= )/);
|
|
34
|
-
const encoder = new TextEncoder();
|
|
35
113
|
const base = { id: "fake-stream", object: "chat.completion.chunk", created: 0, model };
|
|
36
|
-
const
|
|
114
|
+
const frame = (delta, finish = null) => ({ data: JSON.stringify({ ...base, choices: [{ index: 0, delta, finish_reason: finish }] }) });
|
|
115
|
+
const frames = [frame({ role: "assistant" })];
|
|
37
116
|
pieces.forEach((piece, index) => {
|
|
38
117
|
if (options.stallAfter === index)
|
|
39
118
|
frames.push("stall");
|
|
40
119
|
if (options.failAfter === index)
|
|
41
120
|
frames.push("fail");
|
|
42
|
-
frames.push(
|
|
121
|
+
frames.push(frame({ content: piece }));
|
|
43
122
|
});
|
|
44
123
|
if (options.failAfter !== undefined && options.failAfter >= pieces.length)
|
|
45
124
|
frames.push("fail");
|
|
46
|
-
|
|
125
|
+
(reply.toolCalls ?? []).forEach((call, index) => {
|
|
126
|
+
frames.push(frame({ tool_calls: [{ index, id: call.id, type: "function", function: { name: call.name, arguments: "" } }] }));
|
|
127
|
+
for (const part of call.argumentChunks ?? [call.arguments])
|
|
128
|
+
frames.push(frame({ tool_calls: [{ index, function: { arguments: part } }] }));
|
|
129
|
+
});
|
|
130
|
+
frames.push(frame({}, reply.toolCalls === undefined ? "stop" : "tool_calls"));
|
|
47
131
|
if (options.includeUsage !== false) {
|
|
48
132
|
const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length };
|
|
49
|
-
frames.push(JSON.stringify({ ...base, choices: [], usage }));
|
|
133
|
+
frames.push({ data: JSON.stringify({ ...base, choices: [], usage }) });
|
|
134
|
+
}
|
|
135
|
+
frames.push({ data: "[DONE]" });
|
|
136
|
+
return frames;
|
|
137
|
+
}
|
|
138
|
+
function anthropicFrames(reply, model, pieces) {
|
|
139
|
+
const options = reply.stream ?? {};
|
|
140
|
+
const event = (name, data) => ({ event: name, data: JSON.stringify({ type: name, ...data }) });
|
|
141
|
+
const input = reply.usage?.prompt_tokens ?? 10;
|
|
142
|
+
const frames = [event("message_start", { message: { id: "msg_fake", type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { input_tokens: input, output_tokens: 1 } } }), event("ping", {})];
|
|
143
|
+
let block = 0;
|
|
144
|
+
if (pieces.length > 0) {
|
|
145
|
+
frames.push(event("content_block_start", { index: block, content_block: { type: "text", text: "" } }));
|
|
146
|
+
pieces.forEach((piece, index) => {
|
|
147
|
+
if (options.stallAfter === index)
|
|
148
|
+
frames.push("stall");
|
|
149
|
+
if (options.failAfter === index)
|
|
150
|
+
frames.push("fail");
|
|
151
|
+
frames.push(event("content_block_delta", { index: block, delta: { type: "text_delta", text: piece } }));
|
|
152
|
+
});
|
|
153
|
+
frames.push(event("content_block_stop", { index: block }));
|
|
154
|
+
block++;
|
|
155
|
+
}
|
|
156
|
+
if (options.failAfter !== undefined && options.failAfter >= pieces.length)
|
|
157
|
+
frames.push("fail");
|
|
158
|
+
for (const call of reply.toolCalls ?? []) {
|
|
159
|
+
frames.push(event("content_block_start", { index: block, content_block: { type: "tool_use", id: call.id, name: call.name, input: {} } }));
|
|
160
|
+
for (const part of call.argumentChunks ?? [call.arguments])
|
|
161
|
+
frames.push(event("content_block_delta", { index: block, delta: { type: "input_json_delta", partial_json: part } }));
|
|
162
|
+
frames.push(event("content_block_stop", { index: block }));
|
|
163
|
+
block++;
|
|
50
164
|
}
|
|
51
|
-
|
|
165
|
+
const output = reply.usage?.completion_tokens ?? pieces.length;
|
|
166
|
+
frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: { output_tokens: output } }));
|
|
167
|
+
frames.push(event("message_stop", {}));
|
|
168
|
+
return frames;
|
|
169
|
+
}
|
|
170
|
+
function streamBody(reply, call, signal) {
|
|
171
|
+
const options = reply.stream ?? {};
|
|
172
|
+
const pieces = options.chunks ?? (reply.toolCalls !== undefined && reply.text === undefined ? [] : (reply.text ?? "ok").split(/(?<= )/));
|
|
173
|
+
const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call.model, pieces) : openAiFrames(reply, call.model, pieces);
|
|
174
|
+
const failure = call.protocol === "anthropic"
|
|
175
|
+
? `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })}\n\n`
|
|
176
|
+
: `data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`;
|
|
177
|
+
const encoder = new TextEncoder();
|
|
52
178
|
let index = 0;
|
|
53
179
|
return new ReadableStream({
|
|
54
180
|
async pull(controller) {
|
|
@@ -69,12 +195,12 @@ function streamBody(reply, model, signal) {
|
|
|
69
195
|
controller.close();
|
|
70
196
|
return;
|
|
71
197
|
}
|
|
72
|
-
if (frame !== "fail") {
|
|
73
|
-
controller.enqueue(encoder.encode(`data: ${frame}\n\n`));
|
|
198
|
+
if (frame !== "fail" && frame !== "stall") {
|
|
199
|
+
controller.enqueue(encoder.encode(`${frame.event === undefined ? "" : `event: ${frame.event}\n`}data: ${frame.data}\n\n`));
|
|
74
200
|
return;
|
|
75
201
|
}
|
|
76
202
|
if (options.failWith === "error-event") {
|
|
77
|
-
controller.enqueue(encoder.encode(
|
|
203
|
+
controller.enqueue(encoder.encode(failure));
|
|
78
204
|
index = frames.length;
|
|
79
205
|
return;
|
|
80
206
|
}
|
|
@@ -82,12 +208,22 @@ function streamBody(reply, model, signal) {
|
|
|
82
208
|
},
|
|
83
209
|
});
|
|
84
210
|
}
|
|
211
|
+
function errorBody(status, protocol) {
|
|
212
|
+
const type = status === 429 ? "rate_limit_exceeded" : "fake_error";
|
|
213
|
+
if (protocol === "anthropic")
|
|
214
|
+
return { type: "error", error: { type: status === 429 ? "rate_limit_error" : type, message: `fake ${String(status)}` } };
|
|
215
|
+
return { error: { message: `fake ${String(status)}`, type } };
|
|
216
|
+
}
|
|
217
|
+
function toolUseBlock(tool) {
|
|
218
|
+
const input = JSON.parse(tool.arguments);
|
|
219
|
+
return { type: "tool_use", id: tool.id, name: tool.name, input };
|
|
220
|
+
}
|
|
85
221
|
function jsonBody(reply, call) {
|
|
86
222
|
if (reply.body !== undefined)
|
|
87
223
|
return reply.body;
|
|
88
224
|
const status = reply.status ?? 200;
|
|
89
225
|
if (status >= 400)
|
|
90
|
-
return
|
|
226
|
+
return errorBody(status, call.protocol);
|
|
91
227
|
if (call.path.endsWith("/embeddings")) {
|
|
92
228
|
const input = call.body.input;
|
|
93
229
|
const count = Array.isArray(input) ? input.length : 1;
|
|
@@ -98,17 +234,35 @@ function jsonBody(reply, call) {
|
|
|
98
234
|
usage: reply.usage ?? { prompt_tokens: 4 * count, total_tokens: 4 * count },
|
|
99
235
|
};
|
|
100
236
|
}
|
|
101
|
-
const text = reply.text ?? "ok";
|
|
237
|
+
const text = reply.text ?? (reply.toolCalls === undefined ? "ok" : "");
|
|
238
|
+
const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 };
|
|
239
|
+
if (call.protocol === "anthropic") {
|
|
240
|
+
const content = [
|
|
241
|
+
...(text === "" ? [] : [{ type: "text", text }]),
|
|
242
|
+
...(reply.toolCalls ?? []).map(toolUseBlock),
|
|
243
|
+
];
|
|
244
|
+
return {
|
|
245
|
+
id: "msg_fake",
|
|
246
|
+
type: "message",
|
|
247
|
+
role: "assistant",
|
|
248
|
+
model: call.model,
|
|
249
|
+
content,
|
|
250
|
+
stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use",
|
|
251
|
+
stop_sequence: null,
|
|
252
|
+
usage: { input_tokens: usage.prompt_tokens, output_tokens: usage.completion_tokens ?? 0 },
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
const calls = (reply.toolCalls ?? []).map((tool) => ({ id: tool.id, type: "function", function: { name: tool.name, arguments: tool.arguments } }));
|
|
102
256
|
return {
|
|
103
257
|
id: `fake-${String(Date.now())}`,
|
|
104
258
|
object: "chat.completion",
|
|
105
259
|
created: 0,
|
|
106
260
|
model: call.model,
|
|
107
|
-
choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }],
|
|
108
|
-
usage
|
|
261
|
+
choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
|
|
262
|
+
usage,
|
|
109
263
|
};
|
|
110
264
|
}
|
|
111
|
-
/** Creates a fake
|
|
265
|
+
/** Creates a fake provider: deterministic replies, latency, errors, 429s and streams by script, in either protocol. */
|
|
112
266
|
export function createFakeProvider() {
|
|
113
267
|
let scripts = [];
|
|
114
268
|
const calls = [];
|
|
@@ -125,13 +279,15 @@ export function createFakeProvider() {
|
|
|
125
279
|
async fetch(input, init) {
|
|
126
280
|
const url = new URL(input);
|
|
127
281
|
const headers = headerRecord(init);
|
|
128
|
-
const
|
|
282
|
+
const parsed = typeof init.body === "string" ? JSON.parse(init.body) : {};
|
|
283
|
+
const body = isRecord(parsed) ? parsed : {};
|
|
129
284
|
const call = {
|
|
130
285
|
url: input,
|
|
131
286
|
path: url.pathname,
|
|
132
|
-
key: (headers.authorization ?? "").replace(/^Bearer /, ""),
|
|
287
|
+
key: headers["x-api-key"] ?? (headers.authorization ?? "").replace(/^Bearer /, ""),
|
|
133
288
|
model: typeof body.model === "string" ? body.model : "",
|
|
134
289
|
stream: body.stream === true,
|
|
290
|
+
protocol: url.pathname.endsWith("/messages") ? "anthropic" : "openai",
|
|
135
291
|
headers,
|
|
136
292
|
body,
|
|
137
293
|
};
|
|
@@ -144,7 +300,7 @@ export function createFakeProvider() {
|
|
|
144
300
|
throw new TypeError("fetch failed");
|
|
145
301
|
const status = reply.status ?? 200;
|
|
146
302
|
if (call.stream && status < 400 && reply.body === undefined) {
|
|
147
|
-
return new Response(streamBody(reply, call
|
|
303
|
+
return new Response(streamBody(reply, call, init.signal), { status, headers: { "content-type": "text/event-stream", ...reply.headers } });
|
|
148
304
|
}
|
|
149
305
|
return new Response(JSON.stringify(jsonBody(reply, call)), { status, headers: { "content-type": "application/json", ...reply.headers } });
|
|
150
306
|
},
|
|
@@ -12,6 +12,8 @@ export type KeyStoreConfig = {
|
|
|
12
12
|
readonly type: "sqlite";
|
|
13
13
|
readonly path: string;
|
|
14
14
|
};
|
|
15
|
+
/** Where the pool keeps breakers, blacklists, headroom and budgets: in memory (a restart forgets them) or in a SQLite file. */
|
|
16
|
+
export type StateStoreConfig = KeyStoreConfig;
|
|
15
17
|
/** The `gateway` section of a config file, with every default applied and paths resolved from the file's folder. */
|
|
16
18
|
export interface GatewaySettings {
|
|
17
19
|
readonly port: number;
|
|
@@ -24,14 +26,17 @@ export interface GatewaySettings {
|
|
|
24
26
|
export interface GatewayFileConfig {
|
|
25
27
|
readonly pool: PoolConfig;
|
|
26
28
|
readonly gateway: GatewaySettings;
|
|
29
|
+
/** The pool's state store; `serveGateway` opens it and hands it to the router. */
|
|
30
|
+
readonly stateStore: StateStoreConfig;
|
|
27
31
|
}
|
|
28
32
|
/** Reads a referenced secret now: the variable's value or the file's first line, trimmed; undefined when absent or empty. */
|
|
29
33
|
export declare function readSecret(ref: SecretRef, env?: NodeJS.ProcessEnv): Promise<string | undefined>;
|
|
30
34
|
/**
|
|
31
35
|
* Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
|
|
32
36
|
* keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
|
|
33
|
-
* no restart)
|
|
37
|
+
* no restart); `keysFromEnv: PREFIX` finds the canonical `PREFIX_API_KEY`, `_2`, ... variables in `env` (src/env.ts).
|
|
38
|
+
* A field that would hold a secret's value, or a provider header that would carry one, is refused.
|
|
34
39
|
*/
|
|
35
|
-
export declare function parseGatewayConfig(raw: unknown, base?: string): GatewayFileConfig;
|
|
40
|
+
export declare function parseGatewayConfig(raw: unknown, base?: string, env?: NodeJS.ProcessEnv): GatewayFileConfig;
|
|
36
41
|
/** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
|
|
37
|
-
export declare function loadGatewayConfig(path: string): Promise<GatewayFileConfig>;
|
|
42
|
+
export declare function loadGatewayConfig(path: string, env?: NodeJS.ProcessEnv): Promise<GatewayFileConfig>;
|
|
@@ -2,8 +2,11 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
import { dirname, extname, resolve } from "node:path";
|
|
3
3
|
import { parse as parseYaml } from "yaml";
|
|
4
4
|
import { AiRouterError } from "../errors.js";
|
|
5
|
+
import { keysFromEnv } from "../env.js";
|
|
6
|
+
import { PROVIDER_PRESETS } from "../providers.js";
|
|
5
7
|
const CAPABILITIES = { list: { enum: ["chat", "streaming", "tools", "json", "embeddings", "vision"] } };
|
|
6
8
|
const BREAKER = { fields: { failureThreshold: "number", cooldownMs: "number", maxCooldownMs: "number" } };
|
|
9
|
+
const CANDIDATE = { fields: { model: "string", provider: "string", weight: "number" }, required: ["model"] };
|
|
7
10
|
const POLICY = {
|
|
8
11
|
fields: {
|
|
9
12
|
model: "string",
|
|
@@ -15,9 +18,12 @@ const POLICY = {
|
|
|
15
18
|
budget: { fields: { maxCost: "number" } },
|
|
16
19
|
providers: { list: "string" },
|
|
17
20
|
require: CAPABILITIES,
|
|
21
|
+
candidates: { list: { anyOf: [CANDIDATE, { list: CANDIDATE }] } },
|
|
22
|
+
pinned: "boolean",
|
|
18
23
|
},
|
|
19
24
|
};
|
|
20
25
|
const SECRET = { fields: { env: "string", file: "string" } };
|
|
26
|
+
const STORE = { fields: { type: { enum: ["memory", "sqlite"] }, path: "string" }, required: ["type"] };
|
|
21
27
|
const FILE = {
|
|
22
28
|
fields: {
|
|
23
29
|
gateway: {
|
|
@@ -28,14 +34,18 @@ const FILE = {
|
|
|
28
34
|
host: "string",
|
|
29
35
|
maxBodyBytes: "number",
|
|
30
36
|
adminToken: SECRET,
|
|
31
|
-
keyStore:
|
|
37
|
+
keyStore: STORE,
|
|
32
38
|
},
|
|
33
39
|
},
|
|
34
40
|
providers: {
|
|
35
41
|
list: {
|
|
36
42
|
fields: {
|
|
37
43
|
id: "string",
|
|
44
|
+
kind: { enum: Object.keys(PROVIDER_PRESETS) },
|
|
38
45
|
baseUrl: "string",
|
|
46
|
+
defaultMaxTokens: "number",
|
|
47
|
+
budget: { fields: { costPerDay: "number", costPerMonth: "number" } },
|
|
48
|
+
keysFromEnv: "string",
|
|
39
49
|
headers: { map: "string" },
|
|
40
50
|
weight: "number",
|
|
41
51
|
maxConcurrency: "number",
|
|
@@ -49,12 +59,14 @@ const FILE = {
|
|
|
49
59
|
cost: { fields: { input: "number", output: "number" }, required: ["input", "output"] },
|
|
50
60
|
capabilities: CAPABILITIES,
|
|
51
61
|
weight: "number",
|
|
62
|
+
identity: "string",
|
|
63
|
+
dimensions: "number",
|
|
52
64
|
},
|
|
53
65
|
required: ["id"],
|
|
54
66
|
},
|
|
55
67
|
},
|
|
56
68
|
},
|
|
57
|
-
required: ["id", "
|
|
69
|
+
required: ["id", "models"],
|
|
58
70
|
},
|
|
59
71
|
},
|
|
60
72
|
defaults: POLICY,
|
|
@@ -66,11 +78,13 @@ const FILE = {
|
|
|
66
78
|
backoff: { fields: { baseMs: "number", factor: "number", maxMs: "number" } },
|
|
67
79
|
authDisableMs: "number|null",
|
|
68
80
|
rateLimitCooldownMs: "number",
|
|
81
|
+
quotaBlacklistMs: "number",
|
|
69
82
|
},
|
|
70
83
|
},
|
|
71
84
|
selection: { fields: { latencyTolerance: "number", latencySlackMs: "number", ewmaAlpha: "number", lowHeadroom: "number" } },
|
|
72
85
|
telemetry: { fields: { includeContent: "boolean" } },
|
|
73
86
|
storeNamespace: "string",
|
|
87
|
+
stateStore: STORE,
|
|
74
88
|
},
|
|
75
89
|
required: ["providers"],
|
|
76
90
|
};
|
|
@@ -101,6 +115,19 @@ function check(value, schema, where) {
|
|
|
101
115
|
throw configError(`${where} must be true or false`);
|
|
102
116
|
return;
|
|
103
117
|
}
|
|
118
|
+
if ("anyOf" in schema) {
|
|
119
|
+
const errors = [];
|
|
120
|
+
for (const option of schema.anyOf) {
|
|
121
|
+
try {
|
|
122
|
+
check(value, option, where);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
throw configError(errors[0] ?? `${where} matches no allowed shape`);
|
|
130
|
+
}
|
|
104
131
|
if ("enum" in schema) {
|
|
105
132
|
if (typeof value !== "string" || !schema.enum.includes(value))
|
|
106
133
|
throw configError(`${where} must be one of ${schema.enum.join(", ")}`);
|
|
@@ -162,9 +189,10 @@ export async function readSecret(ref, env = process.env) {
|
|
|
162
189
|
/**
|
|
163
190
|
* Validates a parsed config file and turns it into a pool and gateway settings. Upstream keys stay references: `env`
|
|
164
191
|
* keys go to the router as they are, `file` keys become resolvers that read the file at each attempt (rotation needs
|
|
165
|
-
* no restart)
|
|
192
|
+
* no restart); `keysFromEnv: PREFIX` finds the canonical `PREFIX_API_KEY`, `_2`, ... variables in `env` (src/env.ts).
|
|
193
|
+
* A field that would hold a secret's value, or a provider header that would carry one, is refused.
|
|
166
194
|
*/
|
|
167
|
-
export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
195
|
+
export function parseGatewayConfig(raw, base = process.cwd(), env = process.env) {
|
|
168
196
|
check(raw, FILE, "config");
|
|
169
197
|
const file = raw;
|
|
170
198
|
const providers = file.providers.map((provider, index) => {
|
|
@@ -173,7 +201,12 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
|
173
201
|
if (SECRET_HEADER.test(name))
|
|
174
202
|
throw configError(`${where}.headers.${name} would carry a secret in the config file; give keys under keys[] by reference`);
|
|
175
203
|
}
|
|
176
|
-
const
|
|
204
|
+
const listed = Array.isArray(provider.keys) ? provider.keys.filter(isRecord) : [];
|
|
205
|
+
const prefix = typeof provider.keysFromEnv === "string" ? provider.keysFromEnv : undefined;
|
|
206
|
+
if (prefix === undefined && listed.length === 0)
|
|
207
|
+
throw configError(`${where} needs keys or keysFromEnv`);
|
|
208
|
+
const fromEnv = prefix === undefined ? [] : keysFromEnv(prefix, String(provider.id), env);
|
|
209
|
+
const declared = listed.map((key, at) => {
|
|
177
210
|
const ref = secretRef({ env: key.env, file: key.file }, `${where}.keys[${String(at)}]`, base);
|
|
178
211
|
const common = {
|
|
179
212
|
alias: key.alias,
|
|
@@ -182,7 +215,8 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
|
182
215
|
};
|
|
183
216
|
return "env" in ref ? { ...common, env: ref.env } : { ...common, resolve: () => readSecret(ref) };
|
|
184
217
|
});
|
|
185
|
-
|
|
218
|
+
const rest = Object.fromEntries(Object.entries(provider).filter(([field]) => field !== "keysFromEnv"));
|
|
219
|
+
return { ...rest, keys: [...declared, ...fromEnv] };
|
|
186
220
|
});
|
|
187
221
|
const gateway = file.gateway ?? {};
|
|
188
222
|
const store = (gateway.keyStore ?? { type: "memory" });
|
|
@@ -206,7 +240,10 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
|
206
240
|
throw configError("config.gateway.maxBodyBytes must be a positive integer");
|
|
207
241
|
const pool = { ...file, providers };
|
|
208
242
|
delete pool.gateway;
|
|
243
|
+
delete pool.stateStore;
|
|
244
|
+
const state = storeOf(file.stateStore, "config.stateStore", base);
|
|
209
245
|
return {
|
|
246
|
+
stateStore: state,
|
|
210
247
|
pool: pool,
|
|
211
248
|
gateway: {
|
|
212
249
|
port,
|
|
@@ -217,8 +254,16 @@ export function parseGatewayConfig(raw, base = process.cwd()) {
|
|
|
217
254
|
},
|
|
218
255
|
};
|
|
219
256
|
}
|
|
257
|
+
/** A store setting with its path resolved from the config file's folder; memory when absent. */
|
|
258
|
+
function storeOf(value, where, base) {
|
|
259
|
+
if (!isRecord(value) || value.type !== "sqlite")
|
|
260
|
+
return { type: "memory" };
|
|
261
|
+
if (typeof value.path !== "string")
|
|
262
|
+
throw configError(`${where}.path is required for a sqlite store`);
|
|
263
|
+
return { type: "sqlite", path: value.path === ":memory:" ? ":memory:" : resolve(base, value.path) };
|
|
264
|
+
}
|
|
220
265
|
/** Reads a YAML or JSON config file (by extension; anything but `.json` is read as YAML) and parses it. */
|
|
221
|
-
export async function loadGatewayConfig(path) {
|
|
266
|
+
export async function loadGatewayConfig(path, env = process.env) {
|
|
222
267
|
const full = resolve(path);
|
|
223
268
|
let text;
|
|
224
269
|
try {
|
|
@@ -234,5 +279,5 @@ export async function loadGatewayConfig(path) {
|
|
|
234
279
|
catch (error) {
|
|
235
280
|
throw configError(`the config file ${full} does not parse: ${error instanceof Error ? error.message : "unknown error"}`);
|
|
236
281
|
}
|
|
237
|
-
return parseGatewayConfig(raw, dirname(full));
|
|
282
|
+
return parseGatewayConfig(raw, dirname(full), env);
|
|
238
283
|
}
|
|
@@ -43,5 +43,7 @@ export interface AiRouterGateway {
|
|
|
43
43
|
/** Stops listening, ends open connections and releases the router subscription; the key store stays open. */
|
|
44
44
|
close(): Promise<void>;
|
|
45
45
|
}
|
|
46
|
+
/** The Retry-After of a pool whose keys are all blacklisted: until the earliest returns, else this many seconds. */
|
|
47
|
+
export declare const BLACKLISTED_RETRY_AFTER_S = 300;
|
|
46
48
|
/** Creates the gateway over a router; nothing listens until `listen`. */
|
|
47
49
|
export declare function createGateway(router: AiRouter, options: GatewayOptions): AiRouterGateway;
|
package/dist/gateway/gateway.js
CHANGED
|
@@ -39,6 +39,11 @@ function sendError(response, error, headers = {}) {
|
|
|
39
39
|
function secondsUntil(at, now) {
|
|
40
40
|
return Math.max(1, Math.ceil((at - now) / 1000));
|
|
41
41
|
}
|
|
42
|
+
/** The Retry-After of a pool whose keys are all blacklisted: until the earliest returns, else this many seconds. */
|
|
43
|
+
export const BLACKLISTED_RETRY_AFTER_S = 300;
|
|
44
|
+
function retryAfterOf(error, now) {
|
|
45
|
+
return error.retryAt === undefined ? BLACKLISTED_RETRY_AFTER_S : secondsUntil(error.retryAt, now);
|
|
46
|
+
}
|
|
42
47
|
/** Maps a router failure to the status a client can act on; the message is the router's, which never holds a key value. */
|
|
43
48
|
function fromRouterError(error, now) {
|
|
44
49
|
if (!(error instanceof AiRouterError))
|
|
@@ -55,12 +60,16 @@ function fromRouterError(error, now) {
|
|
|
55
60
|
return new GatewayError(503, "pool_unavailable", `no model of the pool can serve this request: ${error.message}`, retry);
|
|
56
61
|
case "rate_limit":
|
|
57
62
|
return new GatewayError(503, "pool_rate_limited", `every key of the pool is rate limited: ${error.message}`, retry);
|
|
63
|
+
case "quota":
|
|
64
|
+
// Every key that could serve is blacklisted for a spent quota or budget: come back when the first one returns.
|
|
65
|
+
return new GatewayError(503, "pool_quota_exhausted", `every upstream key for this request has exhausted its quota or credit: ${error.message}`, { retryAfter: retryAfterOf(error, now) });
|
|
58
66
|
case "deadline":
|
|
59
67
|
return new GatewayError(504, "pool_timeout", error.message);
|
|
60
68
|
case "cancelled":
|
|
61
69
|
return new GatewayError(499, "cancelled", error.message);
|
|
62
70
|
case "auth":
|
|
63
|
-
|
|
71
|
+
// Every key is refused or unset, until an operator seals a new secret: the client can only come back later.
|
|
72
|
+
return new GatewayError(503, "pool_auth_failed", "no upstream key of the pool is accepted: each is blacklisted until its secret changes", { retryAfter: retryAfterOf(error, now) });
|
|
64
73
|
case "transient":
|
|
65
74
|
case "timeout":
|
|
66
75
|
case "network":
|
|
@@ -186,19 +195,26 @@ export function createGateway(router, options) {
|
|
|
186
195
|
if (typeof name !== "string" || name === "")
|
|
187
196
|
throw new GatewayError(400, "missing_model", "model is required: a profile name or a model id");
|
|
188
197
|
const pinned = key.allowFallback ? {} : { allowFallback: false };
|
|
189
|
-
|
|
198
|
+
const needs = operation === "embeddings" ? "embeddings" : "chat";
|
|
199
|
+
const profile = catalog.profiles.find((entry) => entry.name === name);
|
|
200
|
+
if (profile !== undefined) {
|
|
190
201
|
if (!allows(key.profiles, name))
|
|
191
202
|
throw new GatewayError(403, "model_not_allowed", `this key may not call profile "${name}"`);
|
|
203
|
+
if (profile.serves.length > 0 && !profile.serves.includes(needs))
|
|
204
|
+
throw new GatewayError(400, "model_not_supported", `the profile "${name}" does not serve ${operation}`);
|
|
192
205
|
return { options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true };
|
|
193
206
|
}
|
|
194
207
|
const model = modelNamed(catalog, name);
|
|
195
208
|
if (model === undefined)
|
|
196
209
|
throw new GatewayError(404, "model_not_found", `the model "${name}" does not exist`);
|
|
210
|
+
// A model the key's allowlist does not name is refused, never remapped to a profile.
|
|
197
211
|
if (!modelAllowed(key, model))
|
|
198
212
|
throw new GatewayError(403, "model_not_allowed", `this key may not call model "${name}"`);
|
|
199
|
-
const needs = operation === "embeddings" ? "embeddings" : "chat";
|
|
200
213
|
if (!model.capabilities.includes(needs))
|
|
201
214
|
throw new GatewayError(400, "model_not_supported", `the model "${name}" does not serve ${operation}`);
|
|
215
|
+
// An embeddings call never falls back: another model is another vector space.
|
|
216
|
+
if (operation === "embeddings")
|
|
217
|
+
return { options: { policy: { model: name, allowFallback: false } }, dropModel: false };
|
|
202
218
|
// A direct call falls back only to the pool's default fallbacks this key may call itself.
|
|
203
219
|
const fallbacks = catalog.defaults.fallbacks.filter((fallback) => {
|
|
204
220
|
const entry = modelNamed(catalog, fallback);
|
package/dist/gateway/index.d.ts
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
export { AiRouterError } from "../errors.js";
|
|
2
2
|
export type { AttemptFailure } from "../errors.js";
|
|
3
3
|
export { createRouter } from "../router.js";
|
|
4
|
+
export { classifyFailure, readProviderError } from "../classify.js";
|
|
5
|
+
export type { Classified, ProviderErrorInfo } from "../classify.js";
|
|
6
|
+
export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
|
|
7
|
+
export type { EnvPoolSpec, EnvProviderSpec, EnvSource, VirtualKeyEnvSpec } from "../env.js";
|
|
8
|
+
export { PROVIDER_PRESETS } from "../providers.js";
|
|
9
|
+
export type { Protocol, ProviderPreset, QuotaWindow } from "../providers.js";
|
|
10
|
+
export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
|
|
11
|
+
export type { ClosableStateStore, RedisLike } from "../state-stores.js";
|
|
12
|
+
export { EMPTY_RECORD, MemoryStateStore, parseHealthRecord, toHealthRecord } from "../store.js";
|
|
4
13
|
export type { HealthRecord, StateStore } from "../store.js";
|
|
5
14
|
export type * from "../types.js";
|
|
6
15
|
export { loadGatewayConfig, parseGatewayConfig } from "./config-file.js";
|
|
7
|
-
export type { GatewayFileConfig, GatewaySettings, KeyStoreConfig, SecretRef } from "./config-file.js";
|
|
16
|
+
export type { GatewayFileConfig, GatewaySettings, KeyStoreConfig, SecretRef, StateStoreConfig } from "./config-file.js";
|
|
8
17
|
export { createGateway } from "./gateway.js";
|
|
9
18
|
export type { AiRouterGateway, GatewayOptions } from "./gateway.js";
|
|
10
19
|
export { KeyAdmin, KeyAdminError, MemoryKeyStore, MountedKeyError } from "./keys.js";
|
package/dist/gateway/index.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
export { AiRouterError } from "../errors.js";
|
|
2
2
|
export { createRouter } from "../router.js";
|
|
3
|
+
export { classifyFailure, readProviderError } from "../classify.js";
|
|
4
|
+
export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
|
|
5
|
+
export { PROVIDER_PRESETS } from "../providers.js";
|
|
6
|
+
export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
|
|
7
|
+
export { EMPTY_RECORD, MemoryStateStore, parseHealthRecord, toHealthRecord } from "../store.js";
|
|
3
8
|
export { loadGatewayConfig, parseGatewayConfig } from "./config-file.js";
|
|
4
9
|
export { createGateway } from "./gateway.js";
|
|
5
10
|
export { KeyAdmin, KeyAdminError, MemoryKeyStore, MountedKeyError } from "./keys.js";
|
package/dist/gateway/serve.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createRouter } from "../router.js";
|
|
2
|
+
import { openSqliteStateStore } from "../state-stores.js";
|
|
2
3
|
import { loadGatewayConfig, readSecret } from "./config-file.js";
|
|
3
4
|
import { createGateway } from "./gateway.js";
|
|
4
5
|
import { MemoryKeyStore } from "./keys.js";
|
|
@@ -9,8 +10,10 @@ export async function openKeyStore(config) {
|
|
|
9
10
|
}
|
|
10
11
|
/** Loads a config file, builds the router and the key store, and listens: the whole `ai-router serve`. */
|
|
11
12
|
export async function serveGateway(options) {
|
|
12
|
-
const file = await loadGatewayConfig(options.config);
|
|
13
|
-
|
|
13
|
+
const file = await loadGatewayConfig(options.config, options.env);
|
|
14
|
+
// A durable state store keeps blacklisted keys out across a restart; memory is the router's own default.
|
|
15
|
+
const state = file.stateStore.type === "sqlite" ? await openSqliteStateStore(file.stateStore.path) : undefined;
|
|
16
|
+
const router = createRouter({ ...file.pool, ...(state === undefined ? {} : { store: state }), ...(options.fetch === undefined ? {} : { fetch: options.fetch }) });
|
|
14
17
|
const keys = await openKeyStore(file.gateway.keyStore);
|
|
15
18
|
const admin = file.gateway.adminToken;
|
|
16
19
|
const gateway = createGateway(router, {
|
|
@@ -27,12 +30,14 @@ export async function serveGateway(options) {
|
|
|
27
30
|
async close() {
|
|
28
31
|
await gateway.close();
|
|
29
32
|
await keys.close?.();
|
|
33
|
+
state?.close();
|
|
30
34
|
},
|
|
31
35
|
};
|
|
32
36
|
}
|
|
33
37
|
catch (error) {
|
|
34
38
|
await gateway.close();
|
|
35
39
|
await keys.close?.();
|
|
40
|
+
state?.close();
|
|
36
41
|
throw error;
|
|
37
42
|
}
|
|
38
43
|
}
|
package/dist/headers.d.ts
CHANGED
|
@@ -11,5 +11,5 @@ export interface RateLimitInfo {
|
|
|
11
11
|
export declare function parseDuration(value: string): number | null;
|
|
12
12
|
/** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
|
|
13
13
|
export declare function retryAt(headers: Headers, now: number): number;
|
|
14
|
-
/** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
14
|
+
/** Reads the `x-ratelimit-*` (OpenAI, Groq, OpenRouter), `anthropic-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
15
15
|
export declare function readRateLimit(headers: Headers, now: number): RateLimitInfo;
|