@starci/ai-router 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/fake.d.ts CHANGED
@@ -22,6 +22,20 @@ export interface FakeToolCall {
22
22
  readonly arguments: string;
23
23
  readonly argumentChunks?: readonly string[];
24
24
  }
25
+ /** One thinking block a scripted reply opens with; its text is empty unless given, as when the provider hides it. */
26
+ export interface FakeThinking {
27
+ readonly text?: string;
28
+ /** The opaque signature an Anthropic thinking block carries. Default `sig_fake`. */
29
+ readonly signature?: string;
30
+ }
31
+ /** How much of a scripted reply's prompt the provider's cache served or stored; each is a part of `usage.prompt_tokens`. */
32
+ export interface FakeCacheUsage {
33
+ readonly read?: number;
34
+ /** Tokens written, every lifetime together. */
35
+ readonly write?: number;
36
+ /** The part of `write` stored for an hour. */
37
+ readonly write1h?: number;
38
+ }
25
39
  /** One scripted answer. Everything is optional: the default is a 200 chat reply `ok`. */
26
40
  export interface FakeReply {
27
41
  readonly status?: number;
@@ -34,7 +48,13 @@ export interface FakeReply {
34
48
  readonly text?: string;
35
49
  /** Tool calls the reply makes (OpenAI `tool_calls`, Anthropic `tool_use` blocks). */
36
50
  readonly toolCalls?: readonly FakeToolCall[];
51
+ /** Thinking before the text: Anthropic `thinking` blocks with a signature, OpenAI `reasoning_content` for their text. */
52
+ readonly thinking?: readonly FakeThinking[];
37
53
  readonly usage?: Usage;
54
+ /** Cache reads and writes, reported in the protocol's own usage shape. */
55
+ readonly cache?: FakeCacheUsage;
56
+ /** Thinking tokens, reported in the protocol's own usage shape. */
57
+ readonly reasoningTokens?: number;
38
58
  /** The vector of every embedding. Default `[0.1, 0.2, 0.3]`. */
39
59
  readonly embedding?: readonly number[];
40
60
  /** Replaces the whole JSON body. */
@@ -62,6 +82,13 @@ export interface FakeCall {
62
82
  readonly protocol: "openai" | "anthropic";
63
83
  readonly headers: Readonly<Record<string, string>>;
64
84
  readonly body: Readonly<Record<string, unknown>>;
85
+ /**
86
+ * Where the request marks cache breakpoints, in request order: `top` (automatic caching), `system.<i>`, `tools.<i>`,
87
+ * `messages.<i>` (on a message) and `messages.<i>.content.<j>` (on a part or block). The same in either protocol.
88
+ */
89
+ readonly cacheBreakpoints: readonly string[];
90
+ /** The id its answer carries, one per request (`msg_fake_<n>`, `fake-<n>`): a client that keys messages by id sees each once. */
91
+ readonly answerId: string;
65
92
  }
66
93
  /** A scriptable provider that answers in process, OpenAI-compatible and Anthropic Messages alike, for consumers' tests. */
67
94
  export interface FakeProvider {
package/dist/fake.js CHANGED
@@ -75,6 +75,59 @@ export const PROVIDER_ERRORS = Object.freeze({
75
75
  bad_request: { status: 400, body: { error: { message: "bad request", type: "invalid_request_error" } } },
76
76
  },
77
77
  });
78
+ function isMarked(value) {
79
+ return isPlainRecord(value) && isPlainRecord(value.cache_control);
80
+ }
81
+ /** The places a request body marks a cache breakpoint. */
82
+ function breakpointsOf(body) {
83
+ const out = isMarked(body) ? ["top"] : [];
84
+ const list = (value) => (Array.isArray(value) ? value : []);
85
+ list(body.system).forEach((block, index) => {
86
+ if (isMarked(block))
87
+ out.push(`system.${String(index)}`);
88
+ });
89
+ list(body.tools).forEach((tool, index) => {
90
+ if (isMarked(tool))
91
+ out.push(`tools.${String(index)}`);
92
+ });
93
+ list(body.messages).forEach((message, index) => {
94
+ if (isMarked(message))
95
+ out.push(`messages.${String(index)}`);
96
+ list(isPlainRecord(message) ? message.content : undefined).forEach((part, at) => {
97
+ if (isMarked(part))
98
+ out.push(`messages.${String(index)}.content.${String(at)}`);
99
+ });
100
+ });
101
+ return out;
102
+ }
103
+ /** A scripted reply's usage as an OpenAI-compatible provider reports it. */
104
+ function openAiUsage(reply, fallback) {
105
+ const usage = reply.usage ?? fallback;
106
+ const cache = reply.cache;
107
+ const hour = cache?.write1h === undefined ? {} : { cache_write_1h_tokens: cache.write1h };
108
+ return {
109
+ ...usage,
110
+ ...(cache === undefined ? {} : { prompt_tokens_details: { cached_tokens: cache.read ?? 0, cache_write_tokens: cache.write ?? 0, ...hour } }),
111
+ ...(reply.reasoningTokens === undefined ? {} : { completion_tokens_details: { reasoning_tokens: reply.reasoningTokens } }),
112
+ };
113
+ }
114
+ /** A scripted reply's usage as Anthropic reports it: the uncached input apart from the cache reads and writes. */
115
+ function anthropicUsage(reply, prompt, output) {
116
+ const cache = reply.cache;
117
+ const read = cache?.read ?? 0;
118
+ const write = cache?.write ?? 0;
119
+ const hour = cache?.write1h ?? 0;
120
+ return {
121
+ input: {
122
+ input_tokens: prompt - read - write,
123
+ ...(cache === undefined ? {} : { cache_read_input_tokens: read, cache_creation_input_tokens: write, cache_creation: { ephemeral_5m_input_tokens: write - hour, ephemeral_1h_input_tokens: hour } }),
124
+ },
125
+ output: { output_tokens: output, ...(reply.reasoningTokens === undefined ? {} : { output_tokens_details: { thinking_tokens: reply.reasoningTokens } }) },
126
+ };
127
+ }
128
+ function reasoningText(reply) {
129
+ return (reply.thinking ?? []).map((block) => block.text ?? "").filter((text) => text !== "").join("\n\n");
130
+ }
78
131
  function wait(ms, signal) {
79
132
  return new Promise((resolve, reject) => {
80
133
  const aborted = () => (signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
@@ -106,11 +159,14 @@ function headerRecord(init) {
106
159
  });
107
160
  return out;
108
161
  }
109
- function openAiFrames(reply, model, pieces) {
162
+ function openAiFrames(reply, call, pieces) {
110
163
  const options = reply.stream ?? {};
111
- const base = { id: "fake-stream", object: "chat.completion.chunk", created: 0, model };
164
+ const base = { id: call.answerId, object: "chat.completion.chunk", created: 0, model: call.model };
112
165
  const frame = (delta, finish = null) => ({ data: JSON.stringify({ ...base, choices: [{ index: 0, delta, finish_reason: finish }] }) });
113
166
  const frames = [frame({ role: "assistant" })];
167
+ const reasoning = reasoningText(reply);
168
+ if (reasoning !== "")
169
+ frames.push(frame({ reasoning_content: reasoning }));
114
170
  pieces.forEach((piece, index) => {
115
171
  if (options.stallAfter === index)
116
172
  frames.push("stall");
@@ -127,18 +183,26 @@ function openAiFrames(reply, model, pieces) {
127
183
  });
128
184
  frames.push(frame({}, reply.toolCalls === undefined ? "stop" : "tool_calls"));
129
185
  if (options.includeUsage !== false) {
130
- const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length };
186
+ const usage = openAiUsage(reply, { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length });
131
187
  frames.push({ data: JSON.stringify({ ...base, choices: [], usage }) });
132
188
  }
133
189
  frames.push({ data: "[DONE]" });
134
190
  return frames;
135
191
  }
136
- function anthropicFrames(reply, model, pieces) {
192
+ function anthropicFrames(reply, call, pieces) {
193
+ const model = call.model;
137
194
  const options = reply.stream ?? {};
138
195
  const event = (name, data) => ({ event: name, data: JSON.stringify({ type: name, ...data }) });
139
- const input = reply.usage?.prompt_tokens ?? 10;
140
- const frames = [event("message_start", { message: { id: "msg_fake", type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { input_tokens: input, output_tokens: 1 } } }), event("ping", {})];
196
+ const usage = anthropicUsage(reply, reply.usage?.prompt_tokens ?? 10, reply.usage?.completion_tokens ?? pieces.length);
197
+ const frames = [event("message_start", { message: { id: call.answerId, type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { ...usage.input, output_tokens: 1 } } }), event("ping", {})];
141
198
  let block = 0;
199
+ for (const thinking of reply.thinking ?? []) {
200
+ frames.push(event("content_block_start", { index: block, content_block: { type: "thinking", thinking: "", signature: "" } }));
201
+ frames.push(event("content_block_delta", { index: block, delta: { type: "thinking_delta", thinking: thinking.text ?? "" } }));
202
+ frames.push(event("content_block_delta", { index: block, delta: { type: "signature_delta", signature: thinking.signature ?? "sig_fake" } }));
203
+ frames.push(event("content_block_stop", { index: block }));
204
+ block++;
205
+ }
142
206
  if (pieces.length > 0) {
143
207
  frames.push(event("content_block_start", { index: block, content_block: { type: "text", text: "" } }));
144
208
  pieces.forEach((piece, index) => {
@@ -160,15 +224,14 @@ function anthropicFrames(reply, model, pieces) {
160
224
  frames.push(event("content_block_stop", { index: block }));
161
225
  block++;
162
226
  }
163
- const output = reply.usage?.completion_tokens ?? pieces.length;
164
- frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: { output_tokens: output } }));
227
+ frames.push(event("message_delta", { delta: { stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use", stop_sequence: null }, usage: usage.output }));
165
228
  frames.push(event("message_stop", {}));
166
229
  return frames;
167
230
  }
168
231
  function streamBody(reply, call, signal) {
169
232
  const options = reply.stream ?? {};
170
233
  const pieces = options.chunks ?? (reply.toolCalls !== undefined && reply.text === undefined ? [] : (reply.text ?? "ok").split(/(?<= )/));
171
- const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call.model, pieces) : openAiFrames(reply, call.model, pieces);
234
+ const frames = call.protocol === "anthropic" ? anthropicFrames(reply, call, pieces) : openAiFrames(reply, call, pieces);
172
235
  const failure = call.protocol === "anthropic"
173
236
  ? `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })}\n\n`
174
237
  : `data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`;
@@ -233,30 +296,33 @@ function jsonBody(reply, call) {
233
296
  };
234
297
  }
235
298
  const text = reply.text ?? (reply.toolCalls === undefined ? "ok" : "");
236
- const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 };
299
+ const usage = openAiUsage(reply, { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 });
237
300
  if (call.protocol === "anthropic") {
301
+ const counted = anthropicUsage(reply, usage.prompt_tokens, usage.completion_tokens ?? 0);
238
302
  const content = [
303
+ ...(reply.thinking ?? []).map((thinking) => ({ type: "thinking", thinking: thinking.text ?? "", signature: thinking.signature ?? "sig_fake" })),
239
304
  ...(text === "" ? [] : [{ type: "text", text }]),
240
305
  ...(reply.toolCalls ?? []).map(toolUseBlock),
241
306
  ];
242
307
  return {
243
- id: "msg_fake",
308
+ id: call.answerId,
244
309
  type: "message",
245
310
  role: "assistant",
246
311
  model: call.model,
247
312
  content,
248
313
  stop_reason: reply.toolCalls === undefined ? "end_turn" : "tool_use",
249
314
  stop_sequence: null,
250
- usage: { input_tokens: usage.prompt_tokens, output_tokens: usage.completion_tokens ?? 0 },
315
+ usage: { ...counted.input, ...counted.output },
251
316
  };
252
317
  }
318
+ const reasoning = reasoningText(reply);
253
319
  const calls = (reply.toolCalls ?? []).map((tool) => ({ id: tool.id, type: "function", function: { name: tool.name, arguments: tool.arguments } }));
254
320
  return {
255
- id: `fake-${String(Date.now())}`,
321
+ id: call.answerId,
256
322
  object: "chat.completion",
257
323
  created: 0,
258
324
  model: call.model,
259
- choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
325
+ choices: [{ index: 0, message: { role: "assistant", content: text === "" && calls.length > 0 ? null : text, ...(reasoning === "" ? {} : { reasoning_content: reasoning }), ...(calls.length > 0 ? { tool_calls: calls } : {}) }, finish_reason: calls.length > 0 ? "tool_calls" : "stop" }],
260
326
  usage,
261
327
  };
262
328
  }
@@ -279,15 +345,18 @@ export function createFakeProvider() {
279
345
  const headers = headerRecord(init);
280
346
  const parsed = typeof init.body === "string" ? JSON.parse(init.body) : {};
281
347
  const body = isPlainRecord(parsed) ? parsed : {};
348
+ const protocol = url.pathname.endsWith("/messages") ? "anthropic" : "openai";
282
349
  const call = {
283
350
  url: input,
284
351
  path: url.pathname,
285
352
  key: headers["x-api-key"] ?? (headers.authorization ?? "").replace(/^Bearer /, ""),
286
353
  model: typeof body.model === "string" ? body.model : "",
287
354
  stream: body.stream === true,
288
- protocol: url.pathname.endsWith("/messages") ? "anthropic" : "openai",
355
+ protocol,
289
356
  headers,
290
357
  body,
358
+ cacheBreakpoints: breakpointsOf(body),
359
+ answerId: `${protocol === "anthropic" ? "msg_fake_" : "fake-"}${String(calls.length + 1)}`,
291
360
  };
292
361
  calls.push(call);
293
362
  const script = scripts.find((candidate) => matches(candidate.match, call));
@@ -45,6 +45,7 @@ const FILE = {
45
45
  kind: { enum: Object.keys(PROVIDER_PRESETS) },
46
46
  baseUrl: "string",
47
47
  defaultMaxTokens: "number",
48
+ thinking: { enum: ["drop", "replay"] },
48
49
  budget: { fields: { costPerDay: "number", costPerMonth: "number" } },
49
50
  keysFromEnv: "string",
50
51
  headers: { map: "string" },
@@ -57,7 +58,11 @@ const FILE = {
57
58
  id: "string",
58
59
  aliases: { list: "string" },
59
60
  contextWindow: "number",
60
- cost: { fields: { input: "number", output: "number" }, required: ["input", "output"] },
61
+ cost: { fields: { input: "number", output: "number", cacheRead: "number", cacheWrite: "number", cacheWrite1h: "number" }, required: ["input", "output"] },
62
+ params: { map: "any" },
63
+ omit: { list: "string" },
64
+ forcedToolUse: "boolean",
65
+ cacheControl: "boolean",
61
66
  capabilities: CAPABILITIES,
62
67
  weight: "number",
63
68
  identity: "string",
@@ -96,6 +101,9 @@ function configError(message) {
96
101
  return new AiRouterError({ errorClass: "config", message });
97
102
  }
98
103
  function check(value, schema, where) {
104
+ // A model's request defaults are the provider's own fields: the file names them, the provider judges them.
105
+ if (schema === "any")
106
+ return;
99
107
  if (schema === "string") {
100
108
  if (typeof value !== "string")
101
109
  throw configError(`${where} must be a string`);
@@ -133,6 +133,21 @@ function write(response, text) {
133
133
  response.on("close", done);
134
134
  });
135
135
  }
136
+ /** A request id a client may name its call by: short, and nothing a header or a log line could be broken with. */
137
+ const REQUEST_ID = /^[A-Za-z0-9._:-]{1,128}$/;
138
+ /** The header by which a client keeps its call on the providers of one protocol. */
139
+ const KIND_HEADER = "x-ai-router-kind";
140
+ /** A usage with the lane that answered inside it: the one place every client library hands on untouched. */
141
+ function withRouting(usage, served) {
142
+ return isPlainRecord(usage) && served !== undefined ? { ...usage, served } : usage;
143
+ }
144
+ /** A refusal in Anthropic's error shape, for a client that speaks the Messages API. */
145
+ function nativeErrorBody(error) {
146
+ return { type: "error", error: { type: error.type, message: error.message, code: error.code } };
147
+ }
148
+ function servedHeaders(served) {
149
+ return { "x-ai-router-provider": served.provider, "x-ai-router-model": served.model, "x-ai-router-attempts": String(served.attempts), ...(served.cost === undefined ? {} : { "x-ai-router-cost": String(served.cost) }) };
150
+ }
136
151
  /** Creates the gateway over a router; nothing listens until `listen`. */
137
152
  export function createGateway(router, options) {
138
153
  const clock = options.clock ?? Date.now;
@@ -153,7 +168,17 @@ export function createGateway(router, options) {
153
168
  slot.usage = event.usage;
154
169
  if (event.cost !== undefined)
155
170
  slot.cost = event.cost;
171
+ slot.served = { provider: event.provider, model: event.model, attempts: event.attempt, ...(event.cost === undefined ? {} : { cost: event.cost }) };
156
172
  });
173
+ const inflight = new Set();
174
+ /**
175
+ * The id of one call: the client's own `x-request-id` when it sends a well-formed one no call in flight uses, so an
176
+ * app in the router's process can match its call to the router's events; else a new one.
177
+ */
178
+ function requestIdOf(request) {
179
+ const named = request.headers["x-request-id"];
180
+ return typeof named === "string" && REQUEST_ID.test(named) && !inflight.has(named) ? named : randomUUID();
181
+ }
157
182
  function report(error) {
158
183
  try {
159
184
  options.onError?.(error);
@@ -177,9 +202,11 @@ export function createGateway(router, options) {
177
202
  }
178
203
  }
179
204
  async function authenticate(request) {
180
- const secret = bearer(request);
205
+ // An Anthropic client sends its key as x-api-key; it is the same virtual key.
206
+ const named = request.headers["x-api-key"];
207
+ const secret = bearer(request) ?? (typeof named === "string" && named !== "" ? named : undefined);
181
208
  if (secret === undefined)
182
- throw new GatewayError(401, "missing_api_key", "send the key as Authorization: Bearer sk-starci-...");
209
+ throw new GatewayError(401, "missing_api_key", "send the key as Authorization: Bearer sk-starci-... (or as x-api-key)");
183
210
  const key = await admin.authenticate(secret);
184
211
  if (key === undefined)
185
212
  throw new GatewayError(401, "invalid_api_key", "the API key is not valid");
@@ -189,7 +216,21 @@ export function createGateway(router, options) {
189
216
  throw new GatewayError(401, "invalid_api_key", "the API key is disabled");
190
217
  return key;
191
218
  }
192
- function target(key, name, operation) {
219
+ /** The providers that speak one protocol; a call narrowed to a kind is routed among them only. */
220
+ function providersOf(kind) {
221
+ return catalog.providers.filter((provider) => provider.protocol === kind).map((provider) => provider.id);
222
+ }
223
+ /** Narrows a target to the providers of one protocol, refusing a name none of them serves. */
224
+ function narrowed(routed, name, served, kind) {
225
+ if (kind === undefined)
226
+ return routed;
227
+ if (!served.includes(kind)) {
228
+ const route = served.includes("anthropic") ? "/v1/messages" : "/v1/chat/completions";
229
+ throw new GatewayError(400, "model_not_supported", `"${name}" is not served through the ${kind} protocol: call it at ${route}`);
230
+ }
231
+ return { ...routed, options: { ...routed.options, policy: { ...routed.options.policy, providers: providersOf(kind) } } };
232
+ }
233
+ function target(key, name, operation, kind) {
193
234
  if (typeof name !== "string" || name === "")
194
235
  throw new GatewayError(400, "missing_model", "model is required: a profile name or a model id");
195
236
  const pinned = key.allowFallback ? {} : { allowFallback: false };
@@ -200,7 +241,7 @@ export function createGateway(router, options) {
200
241
  throw new GatewayError(403, "model_not_allowed", `this key may not call profile "${name}"`);
201
242
  if (profile.serves.length > 0 && !profile.serves.includes(needs))
202
243
  throw new GatewayError(400, "model_not_supported", `the profile "${name}" does not serve ${operation}`);
203
- return { options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true };
244
+ return narrowed({ options: { profile: name, ...(key.allowFallback ? {} : { policy: pinned }) }, dropModel: true }, name, profile.protocols, kind);
204
245
  }
205
246
  const model = modelNamed(catalog, name);
206
247
  if (model === undefined)
@@ -218,7 +259,7 @@ export function createGateway(router, options) {
218
259
  const entry = modelNamed(catalog, fallback);
219
260
  return entry !== undefined && entry.id !== model.id && modelAllowed(key, entry);
220
261
  });
221
- return { options: { policy: { model: name, fallbacks, ...pinned } }, dropModel: false };
262
+ return narrowed({ options: { policy: { model: name, fallbacks, ...pinned } }, dropModel: false }, name, model.protocols, kind);
222
263
  }
223
264
  async function admit(key, now) {
224
265
  const quota = key.quota;
@@ -249,7 +290,7 @@ export function createGateway(router, options) {
249
290
  }
250
291
  }
251
292
  /** Reads the body and applies the key's bindings and quotas; a refusal is recorded against the key. */
252
- async function prepare(key, request, operation) {
293
+ async function prepare(key, request, operation, native = false) {
253
294
  const started = clock();
254
295
  try {
255
296
  const body = await readBody(request, maxBody);
@@ -257,11 +298,16 @@ export function createGateway(router, options) {
257
298
  throw new GatewayError(400, "invalid_request", "the request body must be a JSON object");
258
299
  if (operation === "chat" && !Array.isArray(body.messages))
259
300
  throw new GatewayError(400, "invalid_request", "messages is required");
301
+ const asked = request.headers[KIND_HEADER];
302
+ if (asked !== undefined && asked !== "openai" && asked !== "anthropic")
303
+ throw new GatewayError(400, "invalid_request", `${KIND_HEADER} must be openai or anthropic`);
304
+ // A Messages request is served by Anthropic providers only: nothing translates it for another protocol.
305
+ const kind = native ? "anthropic" : operation === "chat" ? asked : undefined;
260
306
  if (operation === "embeddings" && typeof body.input !== "string" && !Array.isArray(body.input))
261
307
  throw new GatewayError(400, "invalid_request", "input is required");
262
- const resolved = target(key, body.model, operation);
308
+ const resolved = target(key, body.model, operation, kind);
263
309
  await admit(key, started);
264
- return { key, body, target: resolved, started };
310
+ return { key, body, target: resolved, started, requestId: requestIdOf(request) };
265
311
  }
266
312
  catch (error) {
267
313
  if (error instanceof GatewayError)
@@ -284,14 +330,15 @@ export function createGateway(router, options) {
284
330
  });
285
331
  return controller;
286
332
  }
287
- async function unary(response, operation, { key, body, target: routed, started }) {
288
- const requestId = randomUUID();
333
+ async function unary(response, operation, { key, body, target: routed, started, requestId }) {
289
334
  const headers = { "x-request-id": requestId };
290
335
  const options = { ...routed.options, requestId, signal: controllerFor(response).signal };
336
+ inflight.add(requestId);
291
337
  try {
292
338
  const result = operation === "chat" ? await router.chat(callBody(body, routed), options) : await router.embeddings(callBody(body, routed), options);
293
339
  const { routing, ...answer } = result;
294
- send(response, 200, answer, { ...headers, "x-ai-router-model": routing.model, "x-ai-router-attempts": String(routing.attempts) });
340
+ const served = { provider: routing.provider, model: routing.model, attempts: routing.attempts, ...(routing.cost === undefined ? {} : { cost: routing.cost }) };
341
+ send(response, 200, { ...answer, ...(answer.usage === undefined ? {} : { usage: withRouting(answer.usage, served) }) }, { ...headers, ...servedHeaders(served) });
295
342
  await record(key, started, "success", result.usage, routing.cost);
296
343
  }
297
344
  catch (error) {
@@ -299,9 +346,11 @@ export function createGateway(router, options) {
299
346
  await record(key, started, `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`, undefined, undefined);
300
347
  sendError(response, refusal, headers);
301
348
  }
349
+ finally {
350
+ inflight.delete(requestId);
351
+ }
302
352
  }
303
- async function stream(response, { key, body, target: routed, started }) {
304
- const requestId = randomUUID();
353
+ async function stream(response, { key, body, target: routed, started, requestId }) {
305
354
  const headers = { "x-request-id": requestId };
306
355
  const controller = controllerFor(response);
307
356
  // Usage is always asked of the provider, for the key's accounting; the client gets the usage chunk only if it asked.
@@ -310,6 +359,9 @@ export function createGateway(router, options) {
310
359
  const call = { ...callBody(body, routed), stream_options: { ...asked, include_usage: true } };
311
360
  const slot = {};
312
361
  captured.set(requestId, slot);
362
+ inflight.add(requestId);
363
+ // The usage chunk closes the stream: it is held until the attempt has settled, so it can say which lane answered.
364
+ let last;
313
365
  let outcome = "success";
314
366
  let iterator;
315
367
  try {
@@ -320,9 +372,11 @@ export function createGateway(router, options) {
320
372
  try {
321
373
  while (next.done !== true) {
322
374
  const chunk = next.value;
323
- if (wantsUsage || chunk.usage === undefined || chunk.usage === null)
375
+ if (chunk.choices.length === 0 && chunk.usage !== undefined && chunk.usage !== null)
376
+ last = chunk;
377
+ else if (wantsUsage || chunk.usage === undefined || chunk.usage === null)
324
378
  await write(response, `data: ${JSON.stringify(chunk)}\n\n`);
325
- else if (chunk.choices.length > 0) {
379
+ else {
326
380
  const rest = { ...chunk };
327
381
  delete rest.usage;
328
382
  await write(response, `data: ${JSON.stringify(rest)}\n\n`);
@@ -333,6 +387,8 @@ export function createGateway(router, options) {
333
387
  }
334
388
  next = await iterator.next();
335
389
  }
390
+ if (outcome === "success" && wantsUsage && last !== undefined)
391
+ await write(response, `data: ${JSON.stringify({ ...last, usage: withRouting(last.usage, slot.served) })}\n\n`);
336
392
  if (outcome === "success")
337
393
  response.end("data: [DONE]\n\n");
338
394
  }
@@ -350,25 +406,124 @@ export function createGateway(router, options) {
350
406
  finally {
351
407
  await iterator?.return?.();
352
408
  captured.delete(requestId);
409
+ inflight.delete(requestId);
353
410
  await record(key, started, outcome, slot.usage, slot.cost);
354
411
  }
355
412
  }
413
+ /**
414
+ * What the calling key may name, each with the protocols it is served through: `kind` is the first (a profile's first
415
+ * candidate's), `kinds` all of them in the order they are tried; an embeddings name says its vector length.
416
+ */
356
417
  function listed(key) {
357
418
  const out = new Map();
419
+ const described = (protocols, serves, dimensions) => ({
420
+ ...(protocols[0] === undefined ? {} : { kind: protocols[0] }),
421
+ kinds: protocols,
422
+ serves,
423
+ ...(dimensions === undefined ? {} : { dimensions }),
424
+ });
358
425
  for (const profile of catalog.profiles) {
359
426
  if (allows(key.profiles, profile.name))
360
- out.set(profile.name, { id: profile.name, object: "model", created: 0, owned_by: "profile" });
427
+ out.set(profile.name, { id: profile.name, object: "model", created: 0, owned_by: "profile", ...described(profile.protocols, profile.serves, profile.dimensions) });
361
428
  }
362
429
  for (const model of catalog.models) {
363
430
  if (!modelAllowed(key, model))
364
431
  continue;
432
+ const serves = ["chat", "embeddings"].filter((operation) => model.capabilities.includes(operation));
365
433
  for (const name of [model.id, ...model.aliases]) {
366
434
  if (!out.has(name))
367
- out.set(name, { id: name, object: "model", created: 0, owned_by: model.providers.join(",") });
435
+ out.set(name, { id: name, object: "model", created: 0, owned_by: model.providers.join(","), ...described(model.protocols, serves, model.dimensions) });
368
436
  }
369
437
  }
370
438
  return [...out.values()];
371
439
  }
440
+ function sendNativeError(response, error, headers = {}) {
441
+ send(response, error.status, nativeErrorBody(error), error.retryAfter === undefined ? headers : { ...headers, "retry-after": String(error.retryAfter) });
442
+ }
443
+ /** The headers of an Anthropic client the provider must see: its beta features. */
444
+ function nativeHeaders(request) {
445
+ const beta = request.headers["anthropic-beta"];
446
+ return typeof beta === "string" && beta !== "" ? { "anthropic-beta": beta } : {};
447
+ }
448
+ /** A Messages request answered by an Anthropic provider, its body untouched but for the lane inside `usage`. */
449
+ async function messagesUnary(request, response, { key, body, target: routed, started, requestId }) {
450
+ const headers = { "x-request-id": requestId };
451
+ inflight.add(requestId);
452
+ try {
453
+ const result = await router.messages(callBody(body, routed), { ...routed.options, requestId, signal: controllerFor(response).signal, headers: nativeHeaders(request) });
454
+ const routing = result.routing;
455
+ const served = { provider: routing.provider, model: routing.model, attempts: routing.attempts, ...(routing.cost === undefined ? {} : { cost: routing.cost }) };
456
+ send(response, 200, { ...result.message, usage: withRouting(result.message.usage, served) }, { ...headers, ...servedHeaders(served) });
457
+ await record(key, started, "success", result.usage, routing.cost);
458
+ }
459
+ catch (error) {
460
+ await record(key, started, `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`, undefined, undefined);
461
+ sendNativeError(response, fromRouterError(error, clock()), headers);
462
+ }
463
+ finally {
464
+ inflight.delete(requestId);
465
+ }
466
+ }
467
+ /** A Messages stream: the provider's own events, the first one naming the lane inside the message's `usage`. */
468
+ async function messagesStream(request, response, { key, body, target: routed, started, requestId }) {
469
+ const headers = { "x-request-id": requestId };
470
+ const controller = controllerFor(response);
471
+ const slot = {};
472
+ captured.set(requestId, slot);
473
+ inflight.add(requestId);
474
+ let outcome = "success";
475
+ let iterator;
476
+ const frame = (event, data) => `event: ${event}\ndata: ${data}\n\n`;
477
+ try {
478
+ iterator = router.messagesStream(callBody(body, routed), { ...routed.options, requestId, signal: controller.signal, headers: nativeHeaders(request) })[Symbol.asyncIterator]();
479
+ // Nothing is sent before the first event, so a pool failure before the first content is an ordinary error response.
480
+ let next = await iterator.next();
481
+ const first = next.done === true ? undefined : next.value;
482
+ response.writeHead(200, {
483
+ "content-type": "text/event-stream; charset=utf-8",
484
+ "cache-control": "no-cache",
485
+ connection: "keep-alive",
486
+ "x-accel-buffering": "no",
487
+ ...headers,
488
+ ...(first === undefined ? {} : { "x-ai-router-provider": first.provider, "x-ai-router-model": first.model }),
489
+ });
490
+ try {
491
+ while (next.done !== true) {
492
+ const event = next.value;
493
+ let data = event.data;
494
+ if (event.event === "message_start") {
495
+ const parsed = JSON.parse(data);
496
+ if (isPlainRecord(parsed) && isPlainRecord(parsed.message))
497
+ data = JSON.stringify({ ...parsed, message: { ...parsed.message, usage: withRouting(parsed.message.usage ?? {}, { provider: event.provider, model: event.model, attempts: 1 }) } });
498
+ }
499
+ await write(response, frame(event.event, data));
500
+ if (response.destroyed) {
501
+ outcome = "cancelled";
502
+ break;
503
+ }
504
+ next = await iterator.next();
505
+ }
506
+ if (outcome === "success")
507
+ response.end();
508
+ }
509
+ catch (error) {
510
+ // After the first event the status is sent; the failure goes down the stream as Anthropic's error event.
511
+ outcome = `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`;
512
+ if (!response.destroyed)
513
+ response.end(frame("error", JSON.stringify(nativeErrorBody(fromRouterError(error, clock())))));
514
+ }
515
+ }
516
+ catch (error) {
517
+ outcome = `error:${error instanceof AiRouterError ? error.errorClass : "unknown"}`;
518
+ sendNativeError(response, fromRouterError(error, clock()), headers);
519
+ }
520
+ finally {
521
+ await iterator?.return?.();
522
+ captured.delete(requestId);
523
+ inflight.delete(requestId);
524
+ await record(key, started, outcome, slot.usage, slot.cost);
525
+ }
526
+ }
372
527
  async function models(request, response, id) {
373
528
  const key = await authenticate(request);
374
529
  const data = listed(key);
@@ -493,6 +648,23 @@ export function createGateway(router, options) {
493
648
  await unary(response, operation, prepared);
494
649
  return;
495
650
  }
651
+ if (path === "/v1/messages") {
652
+ // A client of the Messages API reads refusals in Anthropic's error shape.
653
+ try {
654
+ only("POST");
655
+ const prepared = await prepare(await authenticate(request), request, "chat", true);
656
+ if (prepared.body.stream === true)
657
+ await messagesStream(request, response, prepared);
658
+ else
659
+ await messagesUnary(request, response, prepared);
660
+ }
661
+ catch (error) {
662
+ if (!(error instanceof GatewayError) || response.headersSent)
663
+ throw error;
664
+ sendNativeError(response, error);
665
+ }
666
+ return;
667
+ }
496
668
  throw new GatewayError(404, "not_found", `no route ${method} ${path}`);
497
669
  }
498
670
  async function handle(request, response) {
@@ -5,6 +5,8 @@ export { classifyFailure, readProviderError } from "../classify.js";
5
5
  export type { Classified, ProviderErrorInfo } from "../classify.js";
6
6
  export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
7
7
  export type { EnvPoolSpec, EnvProviderSpec, EnvSource, VirtualKeyEnvSpec } from "../env.js";
8
+ export { availableModels, clearOpenRouterCatalogCache, defaultProviderSpecs, EFFORT_TIERS, fetchOpenRouterModels, KNOWN_MODELS, knownModelConfigs, withAutoProfiles } from "../known-models.js";
9
+ export type { AvailableModel, DefaultProvidersOptions, EffortTier, KnownModel, OpenRouterCatalogOptions } from "../known-models.js";
8
10
  export { PROVIDER_PRESETS } from "../providers.js";
9
11
  export type { Protocol, ProviderPreset, QuotaWindow } from "../providers.js";
10
12
  export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
@@ -2,6 +2,7 @@ export { AI_ROUTER_ERROR_CODES, AiRouterError } from "../errors.js";
2
2
  export { createRouter } from "../router.js";
3
3
  export { classifyFailure, readProviderError } from "../classify.js";
4
4
  export { keysFromEnv, poolFromEnv, providersFromEnv, virtualKeyFromEnv } from "../env.js";
5
+ export { availableModels, clearOpenRouterCatalogCache, defaultProviderSpecs, EFFORT_TIERS, fetchOpenRouterModels, KNOWN_MODELS, knownModelConfigs, withAutoProfiles } from "../known-models.js";
5
6
  export { PROVIDER_PRESETS } from "../providers.js";
6
7
  export { openSqliteStateStore, redisStateStore, STATE_STORE_SCHEMA } from "../state-stores.js";
7
8
  export { EMPTY_RECORD, MemoryStateStore, parseHealthRecord, toHealthRecord } from "../store.js";