@starci/ai-router 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +161 -21
- package/dist/ai-router.service.d.ts +5 -1
- package/dist/ai-router.service.js +8 -0
- package/dist/cache.d.ts +35 -0
- package/dist/cache.js +76 -0
- package/dist/config.js +31 -2
- package/dist/fake.d.ts +27 -0
- package/dist/fake.js +84 -15
- package/dist/gateway/config-file.js +9 -1
- package/dist/gateway/gateway.js +189 -17
- package/dist/langchain/index.d.ts +137 -0
- package/dist/langchain/index.js +182 -0
- package/dist/model-request.d.ts +7 -0
- package/dist/model-request.js +54 -0
- package/dist/pool.d.ts +5 -1
- package/dist/pool.js +17 -4
- package/dist/router.js +84 -21
- package/dist/testing.d.ts +1 -1
- package/dist/types.d.ts +130 -3
- package/dist/wire/anthropic.d.ts +18 -8
- package/dist/wire/anthropic.js +238 -58
- package/dist/wire/native.d.ts +29 -0
- package/dist/wire/native.js +56 -0
- package/dist/wire/thinking-memory.d.ts +32 -0
- package/dist/wire/thinking-memory.js +61 -0
- package/dist/wire/wire.d.ts +12 -7
- package/dist/wire/wire.js +10 -1
- package/docs/design.md +189 -19
- package/package.json +26 -6
package/dist/fake.js
CHANGED
|
@@ -75,6 +75,59 @@ export const PROVIDER_ERRORS = Object.freeze({
|
|
|
75
75
|
bad_request: { status: 400, body: { error: { message: "bad request", type: "invalid_request_error" } } },
|
|
76
76
|
},
|
|
77
77
|
});
|
|
78
|
+
function isMarked(value) {
|
|
79
|
+
return isPlainRecord(value) && isPlainRecord(value.cache_control);
|
|
80
|
+
}
|
|
81
|
+
/** The places a request body marks a cache breakpoint. */
|
|
82
|
+
function breakpointsOf(body) {
|
|
83
|
+
const out = isMarked(body) ? ["top"] : [];
|
|
84
|
+
const list = (value) => (Array.isArray(value) ? value : []);
|
|
85
|
+
list(body.system).forEach((block, index) => {
|
|
86
|
+
if (isMarked(block))
|
|
87
|
+
out.push(`system.${String(index)}`);
|
|
88
|
+
});
|
|
89
|
+
list(body.tools).forEach((tool, index) => {
|
|
90
|
+
if (isMarked(tool))
|
|
91
|
+
out.push(`tools.${String(index)}`);
|
|
92
|
+
});
|
|
93
|
+
list(body.messages).forEach((message, index) => {
|
|
94
|
+
if (isMarked(message))
|
|
95
|
+
out.push(`messages.${String(index)}`);
|
|
96
|
+
list(isPlainRecord(message) ? message.content : undefined).forEach((part, at) => {
|
|
97
|
+
if (isMarked(part))
|
|
98
|
+
out.push(`messages.${String(index)}.content.${String(at)}`);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
/** A scripted reply's usage as an OpenAI-compatible provider reports it. */
|
|
104
|
+
function openAiUsage(reply, fallback) {
|
|
105
|
+
const usage = reply.usage ?? fallback;
|
|
106
|
+
const cache = reply.cache;
|
|
107
|
+
const hour = cache?.write1h === undefined ? {} : { cache_write_1h_tokens: cache.write1h };
|
|
108
|
+
return {
|
|
109
|
+
...usage,
|
|
110
|
+
...(cache === undefined ? {} : { prompt_tokens_details: { cached_tokens: cache.read ?? 0, cache_write_tokens: cache.write ?? 0, ...hour } }),
|
|
111
|
+
...(reply.reasoningTokens === undefined ? {} : { completion_tokens_details: { reasoning_tokens: reply.reasoningTokens } }),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/** A scripted reply's usage as Anthropic reports it: the uncached input apart from the cache reads and writes. */
|
|
115
|
+
function anthropicUsage(reply, prompt, output) {
|
|
116
|
+
const cache = reply.cache;
|
|
117
|
+
const read = cache?.read ?? 0;
|
|
118
|
+
const write = cache?.write ?? 0;
|
|
119
|
+
const hour = cache?.write1h ?? 0;
|
|
120
|
+
return {
|
|
121
|
+
input: {
|
|
122
|
+
input_tokens: prompt - read - write,
|
|
123
|
+
...(cache === undefined ? {} : { cache_read_input_tokens: read, cache_creation_input_tokens: write, cache_creation: { ephemeral_5m_input_tokens: write - hour, ephemeral_1h_input_tokens: hour } }),
|
|
124
|
+
},
|
|
125
|
+
output: { output_tokens: output, ...(reply.reasoningTokens === undefined ? {} : { output_tokens_details: { thinking_tokens: reply.reasoningTokens } }) },
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function reasoningText(reply) {
|
|
129
|
+
return (reply.thinking ?? []).map((block) => block.text ?? "").filter((text) => text !== "").join("\n\n");
|
|
130
|
+
}
|
|
78
131
|
function wait(ms, signal) {
|
|
79
132
|
return new Promise((resolve, reject) => {
|
|
80
133
|
const aborted = () => (signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
|
|
@@ -106,11 +159,14 @@ function headerRecord(init) {
|
|
|
106
159
|
});
|
|
107
160
|
return out;
|
|
108
161
|
}
|
|
109
|
-
function openAiFrames(reply,
|
|
162
|
+
function openAiFrames(reply, call, pieces) {
|
|
110
163
|
const options = reply.stream ?? {};
|
|
111
|
-
const base = { id:
|
|
164
|
+
const base = { id: call.answerId, object: "chat.completion.chunk", created: 0, model: call.model };
|
|
112
165
|
const frame = (delta, finish = null) => ({ data: JSON.stringify({ ...base, choices: [{ index: 0, delta, finish_reason: finish }] }) });
|
|
113
166
|
const frames = [frame({ role: "assistant" })];
|
|
167
|
+
const reasoning = reasoningText(reply);
|
|
168
|
+
if (reasoning !== "")
|
|
169
|
+
frames.push(frame({ reasoning_content: reasoning }));
|
|
114
170
|
pieces.forEach((piece, index) => {
|
|
115
171
|
if (options.stallAfter === index)
|
|
116
172
|
frames.push("stall");
|
|
@@ -127,18 +183,26 @@ function openAiFrames(reply, model, pieces) {
|
|
|
127
183
|
});
|
|
128
184
|
frames.push(frame({}, reply.toolCalls === undefined ? "stop" : "tool_calls"));
|
|
129
185
|
if (options.includeUsage !== false) {
|
|
130
|
-
const usage = reply
|
|
186
|
+
const usage = openAiUsage(reply, { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length });
|
|
131
187
|
frames.push({ data: JSON.stringify({ ...base, choices: [], usage }) });
|
|
132
188
|
}
|
|
133
189
|
frames.push({ data: "[DONE]" });
|
|
134
190
|
return frames;
|
|
135
191
|
}
|
|
136
|
-
function anthropicFrames(reply,
|
|
192
|
+
function anthropicFrames(reply, call, pieces) {
|
|
193
|
+
const model = call.model;
|
|
137
194
|
const options = reply.stream ?? {};
|
|
138
195
|
const event = (name, data) => ({ event: name, data: JSON.stringify({ type: name, ...data }) });
|
|
139
|
-
const
|
|
140
|
-
const frames = [event("message_start", { message: { id:
|
|
196
|
+
const usage = anthropicUsage(reply, reply.usage?.prompt_tokens ?? 10, reply.usage?.completion_tokens ?? pieces.length);
|
|
197
|
+
const frames = [event("message_start", { message: { id: call.answerId, type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { ...usage.input, output_tokens: 1 } } }), event("ping", {})];
|
|
141
198
|
let block = 0;
|
|
199
|
+
for (const thinking of reply.thinking ?? []) {
|
|
200
|
+
frames.push(event("content_block_start", { index: block, content_block: { type: "thinking", thinking: "", signature: "" } }));
|
|
201
|
+
frames.push(event("content_block_delta", { index: block, delta: { type: "thinking_delta", thinking: thinking.text ?? "" } }));
|
|
202
|
+
frames.push(event("content_block_delta", { index: block, delta: { type: "signature_delta", signature: thinking.signature ?? "sig_fake" } }));
|
|
203
|
+
frames.push(event("content_block_stop", { index: block }));
|
|
204
|
+
block++;
|
|
205
|
+
}
|
|
142
206
|
if (pieces.length > 0) {
|
|
143
207
|
frames.push(event("content_block_start", { index: block, content_block: { type: "text", text: "" } }));
|
|
144
208
|
pieces.forEach((piece, index) => {
|
|
@@ -160,15 +224,14 @@ function anthropicFrames(reply, model, pieces) {
|
|
|
160
224
|
frames.push(event("content_block_stop", { index: block }));
|
|
161
225
|
block++;
|
|
162
226
|
}
|
|
163
|
-
|
|
164
|
-
frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: { output_tokens: output } }));
|
|
227
|
+
frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: usage.output }));
|
|
165
228
|
frames.push(event("message_stop", {}));
|
|
166
229
|
return frames;
|
|
167
230
|
}
|
|
168
231
|
function streamBody(reply, call, signal) {
|
|
169
232
|
const options = reply.stream ?? {};
|
|
170
233
|
const pieces = options.chunks ?? (reply.toolCalls !== undefined && reply.text === undefined ? [] : (reply.text ?? "ok").split(/(?<= )/));
|
|
171
|
-
const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call
|
|
234
|
+
const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call, pieces) : openAiFrames(reply, call, pieces);
|
|
172
235
|
const failure = call.protocol === "anthropic"
|
|
173
236
|
? `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })}\n\n`
|
|
174
237
|
: `data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`;
|
|
@@ -233,30 +296,33 @@ function jsonBody(reply, call) {
|
|
|
233
296
|
};
|
|
234
297
|
}
|
|
235
298
|
const text = reply.text ?? (reply.toolCalls === undefined ? "ok" : "");
|
|
236
|
-
const usage = reply
|
|
299
|
+
const usage = openAiUsage(reply, { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 });
|
|
237
300
|
if (call.protocol === "anthropic") {
|
|
301
|
+
const counted = anthropicUsage(reply, usage.prompt_tokens, usage.completion_tokens ?? 0);
|
|
238
302
|
const content = [
|
|
303
|
+
...(reply.thinking ?? []).map((thinking) => ({ type: "thinking", thinking: thinking.text ?? "", signature: thinking.signature ?? "sig_fake" })),
|
|
239
304
|
...(text === "" ? [] : [{ type: "text", text }]),
|
|
240
305
|
...(reply.toolCalls ?? []).map(toolUseBlock),
|
|
241
306
|
];
|
|
242
307
|
return {
|
|
243
|
-
id:
|
|
308
|
+
id: call.answerId,
|
|
244
309
|
type: "message",
|
|
245
310
|
role: "assistant",
|
|
246
311
|
model: call.model,
|
|
247
312
|
content,
|
|
248
313
|
stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use",
|
|
249
314
|
stop_sequence: null,
|
|
250
|
-
usage: {
|
|
315
|
+
usage: { ...counted.input, ...counted.output },
|
|
251
316
|
};
|
|
252
317
|
}
|
|
318
|
+
const reasoning = reasoningText(reply);
|
|
253
319
|
const calls = (reply.toolCalls ?? []).map((tool) => ({ id: tool.id, type: "function", function: { name: tool.name, arguments: tool.arguments } }));
|
|
254
320
|
return {
|
|
255
|
-
id:
|
|
321
|
+
id: call.answerId,
|
|
256
322
|
object: "chat.completion",
|
|
257
323
|
created: 0,
|
|
258
324
|
model: call.model,
|
|
259
|
-
choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
|
|
325
|
+
choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(reasoning === "" ? {} : { reasoning_content: reasoning }), ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
|
|
260
326
|
usage,
|
|
261
327
|
};
|
|
262
328
|
}
|
|
@@ -279,15 +345,18 @@ export function createFakeProvider() {
|
|
|
279
345
|
const headers = headerRecord(init);
|
|
280
346
|
const parsed = typeof init.body === "string" ? JSON.parse(init.body) : {};
|
|
281
347
|
const body = isPlainRecord(parsed) ? parsed : {};
|
|
348
|
+
const protocol = url.pathname.endsWith("/messages") ? "anthropic" : "openai";
|
|
282
349
|
const call = {
|
|
283
350
|
url: input,
|
|
284
351
|
path: url.pathname,
|
|
285
352
|
key: headers["x-api-key"] ?? (headers.authorization ?? "").replace(/^Bearer /, ""),
|
|
286
353
|
model: typeof body.model === "string" ? body.model : "",
|
|
287
354
|
stream: body.stream === true,
|
|
288
|
-
protocol
|
|
355
|
+
protocol,
|
|
289
356
|
headers,
|
|
290
357
|
body,
|
|
358
|
+
cacheBreakpoints: breakpointsOf(body),
|
|
359
|
+
answerId: `${protocol === "anthropic" ? "msg_fake_" : "fake-"}${String(calls.length + 1)}`,
|
|
291
360
|
};
|
|
292
361
|
calls.push(call);
|
|
293
362
|
const script = scripts.find((candidate) => matches(candidate.match, call));
|
|
@@ -45,6 +45,7 @@ const FILE = {
|
|
|
45
45
|
kind: { enum: Object.keys(PROVIDER_PRESETS) },
|
|
46
46
|
baseUrl: "string",
|
|
47
47
|
defaultMaxTokens: "number",
|
|
48
|
+
thinking: { enum: ["drop", "replay"] },
|
|
48
49
|
budget: { fields: { costPerDay: "number", costPerMonth: "number" } },
|
|
49
50
|
keysFromEnv: "string",
|
|
50
51
|
headers: { map: "string" },
|
|
@@ -57,7 +58,11 @@ const FILE = {
|
|
|
57
58
|
id: "string",
|
|
58
59
|
aliases: { list: "string" },
|
|
59
60
|
contextWindow: "number",
|
|
60
|
-
cost: { fields: { input: "number", output: "number" }, required: ["input", "output"] },
|
|
61
|
+
cost: { fields: { input: "number", output: "number", cacheRead: "number", cacheWrite: "number", cacheWrite1h: "number" }, required: ["input", "output"] },
|
|
62
|
+
params: { map: "any" },
|
|
63
|
+
omit: { list: "string" },
|
|
64
|
+
forcedToolUse: "boolean",
|
|
65
|
+
cacheControl: "boolean",
|
|
61
66
|
capabilities: CAPABILITIES,
|
|
62
67
|
weight: "number",
|
|
63
68
|
identity: "string",
|
|
@@ -96,6 +101,9 @@ function configError(message) {
|
|
|
96
101
|
return new AiRouterError({ errorClass: "config", message });
|
|
97
102
|
}
|
|
98
103
|
function check(value, schema, where) {
|
|
104
|
+
// A model's request defaults are the provider's own fields: the file names them, the provider judges them.
|
|
105
|
+
if (schema === "any")
|
|
106
|
+
return;
|
|
99
107
|
if (schema === "string") {
|
|
100
108
|
if (typeof value !== "string")
|
|
101
109
|
throw configError(`${where} must be a string`);
|
package/dist/gateway/gateway.js
CHANGED
|
@@ -133,6 +133,21 @@ function write(response, text) {
|
|
|
133
133
|
response.on("close", done);
|
|
134
134
|
});
|
|
135
135
|
}
|
|
136
|
+
/** A request id a client may name its call by: short, and nothing a header or a log line could be broken with. */
|
|
137
|
+
const REQUEST_ID = /^[A-Za-z0-9._:-]{1,128}$/;
|
|
138
|
+
/** The header by which a client keeps its call on the providers of one protocol. */
|
|
139
|
+
const KIND_HEADER = "x-ai-router-kind";
|
|
140
|
+
/** A usage with the lane that answered inside it: the one place every client library hands on untouched. */
|
|
141
|
+
function withRouting(usage, served) {
|
|
142
|
+
return isPlainRecord(usage) && served !== undefined ? { ...usage, served } : usage;
|
|
143
|
+
}
|
|
144
|
+
/** A refusal in Anthropic's error shape, for a client that speaks the Messages API. */
|
|
145
|
+
function nativeErrorBody(error) {
|
|
146
|
+
return { type: "error", error: { type: error.type, message: error.message, code: error.code } };
|
|
147
|
+
}
|
|
148
|
+
function servedHeaders(served) {
|
|
149
|
+
return { "x-ai-router-provider": served.provider, "x-ai-router-model": served.model, "x-ai-router-attempts": String(served.attempts), ...(served.cost === undefined ? {} : { "x-ai-router-cost": String(served.cost) }) };
|
|
150
|
+
}
|
|
136
151
|
/** Creates the gateway over a router; nothing listens until `listen`. */
|
|
137
152
|
export function createGateway(router, options) {
|
|
138
153
|
const clock = options.clock ?? Date.now;
|
|
@@ -153,7 +168,17 @@ export function createGateway(router, options) {
|
|
|
153
168
|
slot.usage = event.usage;
|
|
154
169
|
if (event.cost !== undefined)
|
|
155
170
|
slot.cost = event.cost;
|
|
171
|
+
slot.served = { provider: event.provider, model: event.model, attempts: event.attempt, ...(event.cost === undefined ? {} : { cost: event.cost }) };
|
|
156
172
|
});
|
|
173
|
+
const inflight = new Set();
|
|
174
|
+
/**
|
|
175
|
+
* The id of one call: the client's own `x-request-id` when it sends a well-formed one no call in flight uses, so an
|
|
176
|
+
* app in the router's process can match its call to the router's events; else a new one.
|
|
177
|
+
*/
|
|
178
|
+
function requestIdOf(request) {
|
|
179
|
+
const named = request.headers["x-request-id"];
|
|
180
|
+
return typeof named === "string" && REQUEST_ID.test(named) && !inflight.has(named) ? named : randomUUID();
|
|
181
|
+
}
|
|
157
182
|
function report(error) {
|
|
158
183
|
try {
|
|
159
184
|
options.onError?.(error);
|
|
@@ -177,9 +202,11 @@ export function createGateway(router, options) {
|
|
|
177
202
|
}
|
|
178
203
|
}
|
|
179
204
|
async function authenticate(request) {
|
|
180
|
-
|
|
205
|
+
// An Anthropic client sends its key as x-api-key; it is the same virtual key.
|
|
206
|
+
const named = request.headers["x-api-key"];
|
|
207
|
+
const secret = bearer(request) ?? (typeof named === "string" && named !== "" ? named : undefined);
|
|
181
208
|
if (secret === undefined)
|
|
182
|
-
throw new GatewayError(401, "missing_api_key", "send the key as Authorization: Bearer sk-starci-...");
|
|
209
|
+
throw new GatewayError(401, "missing_api_key", "send the key as Authorization: Bearer sk-starci-... (or as x-api-key)");
|
|
183
210
|
const key = await admin.authenticate(secret);
|
|
184
211
|
if (key === undefined)
|
|
185
212
|
throw new GatewayError(401, "invalid_api_key", "the API key is not valid");
|
|
@@ -189,7 +216,21 @@ export function createGateway(router, options) {
|
|
|
189
216
|
throw new GatewayError(401, "invalid_api_key", "the API key is disabled");
|
|
190
217
|
return key;
|
|
191
218
|
}
|
|
192
|
-
|
|
219
|
+
/** The providers that speak one protocol; a call narrowed to a kind is routed among them only. */
|
|
220
|
+
function providersOf(kind) {
|
|
221
|
+
return catalog.providers.filter((provider) => provider.protocol === kind).map((provider) => provider.id);
|
|
222
|
+
}
|
|
223
|
+
/** Narrows a target to the providers of one protocol, refusing a name none of them serves. */
|
|
224
|
+
function narrowed(routed, name, served, kind) {
|
|
225
|
+
if (kind === undefined)
|
|
226
|
+
return routed;
|
|
227
|
+
if (!served.includes(kind)) {
|
|
228
|
+
const route = served.includes("anthropic") ? "/v1/messages" : "/v1/chat/completions";
|
|
229
|
+
throw new GatewayError(400, "model_not_supported", `"${name}" is not served through the ${kind} protocol: call it at ${route}`);
|
|
230
|
+
}
|
|
231
|
+
return { ...routed, options: { ...routed.options, policy: { ...routed.options.policy, providers: providersOf(kind) } } };
|
|
232
|
+
}
|
|
233
|
+
function target(key, name, operation, kind) {
|
|
193
234
|
if (typeof name !== "string" || name === "")
|
|
194
235
|
throw new GatewayError(400, "missing_model", "model is required: a profile name or a model id");
|
|
195
236
|
const pinned = key.allowFallback ? {} : { allowFallback: false };
|
|
@@ -200,7 +241,7 @@ export function createGateway(router, options) {
|
|
|
200
241
|
throw new GatewayError(403, "model_not_allowed", `this key may not call profile "${name}"`);
|
|
201
242
|
if (profile.serves.length > 0 && !profile.serves.includes(needs))
|
|
202
243
|
throw new GatewayError(400, "model_not_supported", `the profile "${name}" does not serve ${operation}`);
|
|
203
|
-
return { options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true };
|
|
244
|
+
return narrowed({ options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true }, name, profile.protocols, kind);
|
|
204
245
|
}
|
|
205
246
|
const model = modelNamed(catalog, name);
|
|
206
247
|
if (model === undefined)
|
|
@@ -218,7 +259,7 @@ export function createGateway(router, options) {
|
|
|
218
259
|
const entry = modelNamed(catalog, fallback);
|
|
219
260
|
return entry !== undefined && entry.id !== model.id && modelAllowed(key, entry);
|
|
220
261
|
});
|
|
221
|
-
return { options: { policy: { model: name, fallbacks, ...pinned } }, dropModel: false };
|
|
262
|
+
return narrowed({ options: { policy: { model: name, fallbacks, ...pinned } }, dropModel: false }, name, model.protocols, kind);
|
|
222
263
|
}
|
|
223
264
|
async function admit(key, now) {
|
|
224
265
|
const quota = key.quota;
|
|
@@ -249,7 +290,7 @@ export function createGateway(router, options) {
|
|
|
249
290
|
}
|
|
250
291
|
}
|
|
251
292
|
/** Reads the body and applies the key's bindings and quotas; a refusal is recorded against the key. */
|
|
252
|
-
async function prepare(key, request, operation) {
|
|
293
|
+
async function prepare(key, request, operation, native = false) {
|
|
253
294
|
const started = clock();
|
|
254
295
|
try {
|
|
255
296
|
const body = await readBody(request, maxBody);
|
|
@@ -257,11 +298,16 @@ export function createGateway(router, options) {
|
|
|
257
298
|
throw new GatewayError(400, "invalid_request", "the request body must be a JSON object");
|
|
258
299
|
if (operation === "chat" && !Array.isArray(body.messages))
|
|
259
300
|
throw new GatewayError(400, "invalid_request", "messages is required");
|
|
301
|
+
const asked = request.headers[KIND_HEADER];
|
|
302
|
+
if (asked !== undefined && asked !== "openai" && asked !== "anthropic")
|
|
303
|
+
throw new GatewayError(400, "invalid_request", `${KIND_HEADER} must be openai or anthropic`);
|
|
304
|
+
// A Messages request is served by Anthropic providers only: nothing translates it for another protocol.
|
|
305
|
+
const kind = native ? "anthropic" : operation === "chat" ? asked : undefined;
|
|
260
306
|
if (operation === "embeddings" && typeof body.input !== "string" && !Array.isArray(body.input))
|
|
261
307
|
throw new GatewayError(400, "invalid_request", "input is required");
|
|
262
|
-
const resolved = target(key, body.model, operation);
|
|
308
|
+
const resolved = target(key, body.model, operation, kind);
|
|
263
309
|
await admit(key, started);
|
|
264
|
-
return { key, body, target: resolved, started };
|
|
310
|
+
return { key, body, target: resolved, started, requestId: requestIdOf(request) };
|
|
265
311
|
}
|
|
266
312
|
catch (error) {
|
|
267
313
|
if (error instanceof GatewayError)
|
|
@@ -284,14 +330,15 @@ export function createGateway(router, options) {
|
|
|
284
330
|
});
|
|
285
331
|
return controller;
|
|
286
332
|
}
|
|
287
|
-
async function unary(response, operation, { key, body, target: routed, started }) {
|
|
288
|
-
const requestId = randomUUID();
|
|
333
|
+
async function unary(response, operation, { key, body, target: routed, started, requestId }) {
|
|
289
334
|
const headers = { "x-request-id": requestId };
|
|
290
335
|
const options = { ...routed.options, requestId, signal: controllerFor(response).signal };
|
|
336
|
+
inflight.add(requestId);
|
|
291
337
|
try {
|
|
292
338
|
const result = operation === "chat" ? await router.chat(callBody(body, routed), options) : await router.embeddings(callBody(body, routed), options);
|
|
293
339
|
const { routing, ...answer } = result;
|
|
294
|
-
|
|
340
|
+
const served = { provider: routing.provider, model: routing.model, attempts: routing.attempts, ...(routing.cost === undefined ? {} : { cost: routing.cost }) };
|
|
341
|
+
send(response, 200, { ...answer, ...(answer.usage === undefined ? {} : { usage: withRouting(answer.usage, served) }) }, { ...headers, ...servedHeaders(served) });
|
|
295
342
|
await record(key, started, "success", result.usage, routing.cost);
|
|
296
343
|
}
|
|
297
344
|
catch (error) {
|
|
@@ -299,9 +346,11 @@ export function createGateway(router, options) {
|
|
|
299
346
|
await record(key, started, `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`, undefined, undefined);
|
|
300
347
|
sendError(response, refusal, headers);
|
|
301
348
|
}
|
|
349
|
+
finally {
|
|
350
|
+
inflight.delete(requestId);
|
|
351
|
+
}
|
|
302
352
|
}
|
|
303
|
-
async function stream(response, { key, body, target: routed, started }) {
|
|
304
|
-
const requestId = randomUUID();
|
|
353
|
+
async function stream(response, { key, body, target: routed, started, requestId }) {
|
|
305
354
|
const headers = { "x-request-id": requestId };
|
|
306
355
|
const controller = controllerFor(response);
|
|
307
356
|
// Usage is always asked of the provider, for the key's accounting; the client gets the usage chunk only if it asked.
|
|
@@ -310,6 +359,9 @@ export function createGateway(router, options) {
|
|
|
310
359
|
const call = { ...callBody(body, routed), stream_options: { ...asked, include_usage: true } };
|
|
311
360
|
const slot = {};
|
|
312
361
|
captured.set(requestId, slot);
|
|
362
|
+
inflight.add(requestId);
|
|
363
|
+
// The usage chunk closes the stream: it is held until the attempt has settled, so it can say which lane answered.
|
|
364
|
+
let last;
|
|
313
365
|
let outcome = "success";
|
|
314
366
|
let iterator;
|
|
315
367
|
try {
|
|
@@ -320,9 +372,11 @@ export function createGateway(router, options) {
|
|
|
320
372
|
try {
|
|
321
373
|
while (next.done !== true) {
|
|
322
374
|
const chunk = next.value;
|
|
323
|
-
if (
|
|
375
|
+
if (chunk.choices.length === 0 && chunk.usage !== undefined && chunk.usage !== null)
|
|
376
|
+
last = chunk;
|
|
377
|
+
else if (wantsUsage || chunk.usage === undefined || chunk.usage === null)
|
|
324
378
|
await write(response, `data: ${JSON.stringify(chunk)}\n\n`);
|
|
325
|
-
else
|
|
379
|
+
else {
|
|
326
380
|
const rest = { ...chunk };
|
|
327
381
|
delete rest.usage;
|
|
328
382
|
await write(response, `data: ${JSON.stringify(rest)}\n\n`);
|
|
@@ -333,6 +387,8 @@ export function createGateway(router, options) {
|
|
|
333
387
|
}
|
|
334
388
|
next = await iterator.next();
|
|
335
389
|
}
|
|
390
|
+
if (outcome === "success" && wantsUsage && last !== undefined)
|
|
391
|
+
await write(response, `data: ${JSON.stringify({ ...last, usage: withRouting(last.usage, slot.served) })}\n\n`);
|
|
336
392
|
if (outcome === "success")
|
|
337
393
|
response.end("data: [DONE]\n\n");
|
|
338
394
|
}
|
|
@@ -350,25 +406,124 @@ export function createGateway(router, options) {
|
|
|
350
406
|
finally {
|
|
351
407
|
await iterator?.return?.();
|
|
352
408
|
captured.delete(requestId);
|
|
409
|
+
inflight.delete(requestId);
|
|
353
410
|
await record(key, started, outcome, slot.usage, slot.cost);
|
|
354
411
|
}
|
|
355
412
|
}
|
|
413
|
+
/**
|
|
414
|
+
* What the calling key may name, each with the protocols it is served through: `kind` is the first (a profile's first
|
|
415
|
+
* candidate's), `kinds` all of them in the order they are tried; an embeddings name says its vector length.
|
|
416
|
+
*/
|
|
356
417
|
function listed(key) {
|
|
357
418
|
const out = new Map();
|
|
419
|
+
const described = (protocols, serves, dimensions) => ({
|
|
420
|
+
...(protocols[0] === undefined ? {} : { kind: protocols[0] }),
|
|
421
|
+
kinds: protocols,
|
|
422
|
+
serves,
|
|
423
|
+
...(dimensions === undefined ? {} : { dimensions }),
|
|
424
|
+
});
|
|
358
425
|
for (const profile of catalog.profiles) {
|
|
359
426
|
if (allows(key.profiles, profile.name))
|
|
360
|
-
out.set(profile.name, { id: profile.name, object: "model", created: 0, owned_by: "profile" });
|
|
427
|
+
out.set(profile.name, { id: profile.name, object: "model", created: 0, owned_by: "profile", ...described(profile.protocols, profile.serves, profile.dimensions) });
|
|
361
428
|
}
|
|
362
429
|
for (const model of catalog.models) {
|
|
363
430
|
if (!modelAllowed(key, model))
|
|
364
431
|
continue;
|
|
432
|
+
const serves = ["chat", "embeddings"].filter((operation) => model.capabilities.includes(operation));
|
|
365
433
|
for (const name of [model.id, ...model.aliases]) {
|
|
366
434
|
if (!out.has(name))
|
|
367
|
-
out.set(name, { id: name, object: "model", created: 0, owned_by: model.providers.join(",") });
|
|
435
|
+
out.set(name, { id: name, object: "model", created: 0, owned_by: model.providers.join(","), ...described(model.protocols, serves, model.dimensions) });
|
|
368
436
|
}
|
|
369
437
|
}
|
|
370
438
|
return [...out.values()];
|
|
371
439
|
}
|
|
440
|
+
function sendNativeError(response, error, headers = {}) {
|
|
441
|
+
send(response, error.status, nativeErrorBody(error), error.retryAfter === undefined ? headers : { ...headers, "retry-after": String(error.retryAfter) });
|
|
442
|
+
}
|
|
443
|
+
/** The headers of an Anthropic client the provider must see: its beta features. */
|
|
444
|
+
function nativeHeaders(request) {
|
|
445
|
+
const beta = request.headers["anthropic-beta"];
|
|
446
|
+
return typeof beta === "string" && beta !== "" ? { "anthropic-beta": beta } : {};
|
|
447
|
+
}
|
|
448
|
+
/** A Messages request answered by an Anthropic provider, its body untouched but for the lane inside `usage`. */
|
|
449
|
+
async function messagesUnary(request, response, { key, body, target: routed, started, requestId }) {
|
|
450
|
+
const headers = { "x-request-id": requestId };
|
|
451
|
+
inflight.add(requestId);
|
|
452
|
+
try {
|
|
453
|
+
const result = await router.messages(callBody(body, routed), { ...routed.options, requestId, signal: controllerFor(response).signal, headers: nativeHeaders(request) });
|
|
454
|
+
const routing = result.routing;
|
|
455
|
+
const served = { provider: routing.provider, model: routing.model, attempts: routing.attempts, ...(routing.cost === undefined ? {} : { cost: routing.cost }) };
|
|
456
|
+
send(response, 200, { ...result.message, usage: withRouting(result.message.usage, served) }, { ...headers, ...servedHeaders(served) });
|
|
457
|
+
await record(key, started, "success", result.usage, routing.cost);
|
|
458
|
+
}
|
|
459
|
+
catch (error) {
|
|
460
|
+
await record(key, started, `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`, undefined, undefined);
|
|
461
|
+
sendNativeError(response, fromRouterError(error, clock()), headers);
|
|
462
|
+
}
|
|
463
|
+
finally {
|
|
464
|
+
inflight.delete(requestId);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
/** A Messages stream: the provider's own events, the first one naming the lane inside the message's `usage`. */
|
|
468
|
+
async function messagesStream(request, response, { key, body, target: routed, started, requestId }) {
|
|
469
|
+
const headers = { "x-request-id": requestId };
|
|
470
|
+
const controller = controllerFor(response);
|
|
471
|
+
const slot = {};
|
|
472
|
+
captured.set(requestId, slot);
|
|
473
|
+
inflight.add(requestId);
|
|
474
|
+
let outcome = "success";
|
|
475
|
+
let iterator;
|
|
476
|
+
const frame = (event, data) => `event: ${event}\ndata: ${data}\n\n`;
|
|
477
|
+
try {
|
|
478
|
+
iterator = router.messagesStream(callBody(body, routed), { ...routed.options, requestId, signal: controller.signal, headers: nativeHeaders(request) })[Symbol.asyncIterator]();
|
|
479
|
+
// Nothing is sent before the first event, so a pool failure before the first content is an ordinary error response.
|
|
480
|
+
let next = await iterator.next();
|
|
481
|
+
const first = next.done === true ? undefined : next.value;
|
|
482
|
+
response.writeHead(200, {
|
|
483
|
+
"content-type": "text/event-stream; charset=utf-8",
|
|
484
|
+
"cache-control": "no-cache",
|
|
485
|
+
connection: "keep-alive",
|
|
486
|
+
"x-accel-buffering": "no",
|
|
487
|
+
...headers,
|
|
488
|
+
...(first === undefined ? {} : { "x-ai-router-provider": first.provider, "x-ai-router-model": first.model }),
|
|
489
|
+
});
|
|
490
|
+
try {
|
|
491
|
+
while (next.done !== true) {
|
|
492
|
+
const event = next.value;
|
|
493
|
+
let data = event.data;
|
|
494
|
+
if (event.event === "message_start") {
|
|
495
|
+
const parsed = JSON.parse(data);
|
|
496
|
+
if (isPlainRecord(parsed) && isPlainRecord(parsed.message))
|
|
497
|
+
data = JSON.stringify({ ...parsed, message: { ...parsed.message, usage: withRouting(parsed.message.usage ?? {}, { provider: event.provider, model: event.model, attempts: 1 }) } });
|
|
498
|
+
}
|
|
499
|
+
await write(response, frame(event.event, data));
|
|
500
|
+
if (response.destroyed) {
|
|
501
|
+
outcome = "cancelled";
|
|
502
|
+
break;
|
|
503
|
+
}
|
|
504
|
+
next = await iterator.next();
|
|
505
|
+
}
|
|
506
|
+
if (outcome === "success")
|
|
507
|
+
response.end();
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
// After the first event the status is sent; the failure goes down the stream as Anthropic's error event.
|
|
511
|
+
outcome = `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`;
|
|
512
|
+
if (!response.destroyed)
|
|
513
|
+
response.end(frame("error", JSON.stringify(nativeErrorBody(fromRouterError(error, clock())))));
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
catch (error) {
|
|
517
|
+
outcome = `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`;
|
|
518
|
+
sendNativeError(response, fromRouterError(error, clock()), headers);
|
|
519
|
+
}
|
|
520
|
+
finally {
|
|
521
|
+
await iterator?.return?.();
|
|
522
|
+
captured.delete(requestId);
|
|
523
|
+
inflight.delete(requestId);
|
|
524
|
+
await record(key, started, outcome, slot.usage, slot.cost);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
372
527
|
async function models(request, response, id) {
|
|
373
528
|
const key = await authenticate(request);
|
|
374
529
|
const data = listed(key);
|
|
@@ -493,6 +648,23 @@ export function createGateway(router, options) {
|
|
|
493
648
|
await unary(response, operation, prepared);
|
|
494
649
|
return;
|
|
495
650
|
}
|
|
651
|
+
if (path === "/v1/messages") {
|
|
652
|
+
// A client of the Messages API reads refusals in Anthropic's error shape.
|
|
653
|
+
try {
|
|
654
|
+
only("POST");
|
|
655
|
+
const prepared = await prepare(await authenticate(request), request, "chat", true);
|
|
656
|
+
if (prepared.body.stream === true)
|
|
657
|
+
await messagesStream(request, response, prepared);
|
|
658
|
+
else
|
|
659
|
+
await messagesUnary(request, response, prepared);
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
if (!(error instanceof GatewayError) || response.headersSent)
|
|
663
|
+
throw error;
|
|
664
|
+
sendNativeError(response, error);
|
|
665
|
+
}
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
496
668
|
throw new GatewayError(404, "not_found", `no route ${method} ${path}`);
|
|
497
669
|
}
|
|
498
670
|
async function handle(request, response) {
|