@starci/ai-router 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,61 @@
1
+ import { isPlainRecord } from "@starci/common";
2
+ import { createHash } from "node:crypto";
3
+ /** The most turns one provider keeps; the oldest is forgotten first. A forgotten turn is sent without its thinking. */
4
+ export const THINKING_MEMORY_LIMIT = 2000;
5
+ /** Assistant turns by the ids of the tool calls they made, least recently used first. */
6
+ export class ThinkingMemory {
7
+ #turns = new Map();
8
+ #limit;
9
+ constructor(limit = THINKING_MEMORY_LIMIT) {
10
+ this.#limit = limit;
11
+ }
12
+ /** The turn that made these tool calls, when it is still remembered. */
13
+ get(key) {
14
+ const turn = this.#turns.get(key);
15
+ if (turn === undefined)
16
+ return undefined;
17
+ this.#turns.delete(key);
18
+ this.#turns.set(key, turn);
19
+ return turn;
20
+ }
21
+ /** Remembers a turn, forgetting the least recently used one past the limit. */
22
+ set(key, turn) {
23
+ this.#turns.delete(key);
24
+ this.#turns.set(key, turn);
25
+ const oldest = this.#turns.size > this.#limit ? this.#turns.keys().next().value : undefined;
26
+ if (oldest !== undefined)
27
+ this.#turns.delete(oldest);
28
+ }
29
+ /** How many turns are remembered. */
30
+ get size() {
31
+ return this.#turns.size;
32
+ }
33
+ }
34
+ const memories = new WeakMap();
35
+ /** The memory of a provider that replays thinking; null for one that drops it (the default). */
36
+ export function memoryOf(provider) {
37
+ if (provider.thinking !== "replay")
38
+ return null;
39
+ let memory = memories.get(provider);
40
+ if (memory === undefined) {
41
+ memory = new ThinkingMemory();
42
+ memories.set(provider, memory);
43
+ }
44
+ return memory;
45
+ }
46
+ /** JSON with sorted keys and without cache breakpoints: neither key order nor a moved breakpoint is an edit of the prefix. */
47
+ function canonical(value) {
48
+ if (Array.isArray(value))
49
+ return value.map(canonical);
50
+ if (!isPlainRecord(value))
51
+ return value;
52
+ const out = {};
53
+ for (const key of Object.keys(value).sort())
54
+ if (key !== "cache_control")
55
+ out[key] = canonical(value[key]);
56
+ return out;
57
+ }
58
+ /** The digest of a prefix extended by one more piece (the system and tools first, then each turn). */
59
+ export function digest(previous, piece) {
60
+ return createHash("sha256").update(previous).update("\n").update(JSON.stringify(canonical(piece))).digest("hex");
61
+ }
@@ -1,13 +1,16 @@
1
1
  import type { SseEvent } from "../sse.js";
2
- import type { ProviderConfig } from "../types.js";
2
+ import type { ModelConfig, ProviderConfig } from "../types.js";
3
3
  /** A request in the provider's own wire format: the path under its base URL and the JSON body. */
4
4
  export interface WireRequest {
5
5
  readonly path: string;
6
6
  readonly body: Record<string, unknown>;
7
+ /** What the adapter needs again to read this request's answer; the router hands it back untouched. */
8
+ readonly context?: unknown;
7
9
  }
8
10
  /**
9
11
  * One item a provider stream yields, already in OpenAI's shape: a chunk (a `chat.completion.chunk` record), an error the
10
- * provider streamed (its raw error body, classified by the router), or an event that is not JSON.
12
+ * provider streamed (its raw error body, classified by the router), an event that is not JSON, or a sign of life that
13
+ * carries nothing for the caller (a ping, a hidden thinking delta): it restarts the attempt's idle timeout.
11
14
  */
12
15
  export type WireEvent = {
13
16
  readonly kind: "chunk";
@@ -17,6 +20,8 @@ export type WireEvent = {
17
20
  readonly body: unknown;
18
21
  } | {
19
22
  readonly kind: "malformed";
23
+ } | {
24
+ readonly kind: "alive";
20
25
  };
21
26
  /**
22
27
  * A protocol adapter: it turns the router's OpenAI chat-completions and embeddings requests into a provider's own and
@@ -28,13 +33,13 @@ export interface Wire {
28
33
  /** The auth and protocol headers of one attempt. */
29
34
  headers(secret: string, provider: ProviderConfig): Record<string, string>;
30
35
  /** A chat request; throws an `AttemptError` (`invalid_request`) when the request cannot be expressed. */
31
- chat(body: Record<string, unknown>, model: string, stream: boolean, provider: ProviderConfig): WireRequest;
36
+ chat(body: Record<string, unknown>, model: ModelConfig, stream: boolean, provider: ProviderConfig): WireRequest;
32
37
  /** An embeddings request. */
33
38
  embeddings(body: Record<string, unknown>, model: string): WireRequest;
34
- /** A unary chat answer as an OpenAI `chat.completion` record. */
35
- chatResponse(json: Record<string, unknown>): Record<string, unknown>;
36
- /** A provider stream as OpenAI chunks; the router stops at the end of the iterable. */
37
- chunks(events: AsyncIterable<SseEvent>): AsyncGenerator<WireEvent, void, undefined>;
39
+ /** A unary chat answer as an OpenAI `chat.completion` record; `context` is the request's. */
40
+ chatResponse(json: Record<string, unknown>, context?: unknown): Record<string, unknown>;
41
+ /** A provider stream as OpenAI chunks; the router stops at the end of the iterable. `context` is the request's. */
42
+ chunks(events: AsyncIterable<SseEvent>, context?: unknown): AsyncGenerator<WireEvent, void, undefined>;
38
43
  }
39
44
  /** The OpenAI chat-completions and embeddings API with a bearer key, which every OpenAI-compatible provider speaks. */
40
45
  export declare const openAiWire: Wire;
package/dist/wire/wire.js CHANGED
@@ -1,9 +1,18 @@
1
1
  import { isPlainRecord } from "@starci/common";
2
+ import { takesCacheControl, withoutCacheControl, withPartCacheControl } from "../cache.js";
3
+ /**
4
+ * The cache breakpoints of a request as an OpenAI-compatible provider takes them: none where the provider caches a
5
+ * prefix on its own (DeepSeek, OpenAI, Gemini, Groq and most others would meet an unknown field), on content parts and
6
+ * at the top level where it documents them (OpenRouter for `anthropic/*`, or a model that sets `cacheControl`).
7
+ */
8
+ function cacheShape(body, model, provider) {
9
+ return takesCacheControl(provider, model) ? withPartCacheControl(body) : withoutCacheControl(body);
10
+ }
2
11
  /** The OpenAI chat-completions and embeddings API with a bearer key, which every OpenAI-compatible provider speaks. */
3
12
  export const openAiWire = {
4
13
  secretHeaders: ["authorization"],
5
14
  headers: (secret) => ({ authorization: `Bearer ${secret}` }),
6
- chat: (body, model, stream) => ({ path: "/chat/completions", body: { ...body, model, ...(stream ? { stream: true } : {}) } }),
15
+ chat: (body, model, stream, provider) => ({ path: "/chat/completions", body: { ...cacheShape(body, model, provider), model: model.id, ...(stream ? { stream: true } : {}) } }),
7
16
  embeddings: (body, model) => ({ path: "/embeddings", body: { ...body, model } }),
8
17
  chatResponse: (json) => json,
9
18
  async *chunks(events) {
package/docs/design.md CHANGED
@@ -1,8 +1,9 @@
1
1
  # @starci/ai-router design: many providers below, one virtual key above
2
2
 
3
- This is the design of `@starci/ai-router` 0.5: how a pool of upstream providers, each with an array of keys, is
4
- load-balanced behind one OpenAI-compatible gateway, how a key whose quota is spent or whose secret is refused is
5
- blacklisted, and how an app declares all of it from env. The README is the reference of every option; this document
3
+ This is the design of `@starci/ai-router`: how a pool of upstream providers, each with an array of keys, is
4
+ load-balanced behind one gateway that speaks the OpenAI and the Anthropic protocols, how a key whose quota is spent or
5
+ whose secret is refused is blacklisted, how prompt caching and thinking cross the protocols, and how an app declares
6
+ all of it from env. The README is the reference of every option; this document
6
7
  says why the pieces are shaped as they are, and what was read to shape them.
7
8
 
8
9
  ## The shape
@@ -19,9 +20,10 @@ says why the pieces are shaped as they are, and what was read to shape them.
19
20
  └── groq, ollama, vllm, ... OpenAI
20
21
  ```
21
22
 
22
- - **Above:** every client of an app speaks OpenAI chat-completions and embeddings to the gateway with one virtual key
23
- per app and environment (`STARCI_AI_KEY`, `sk-starci-<label>-...`), mounted from a sealed secret
24
- (`virtualKeyFromEnv`). It names a profile as its `model`. No app uses the admin token or the admin API; both stay in
23
+ - **Above:** every client of an app speaks OpenAI chat-completions and embeddings, or Anthropic Messages
24
+ (`/v1/messages`), to the gateway with one virtual key per app and environment (`STARCI_AI_KEY`,
25
+ `sk-starci-<label>-...`), mounted from a sealed secret (`virtualKeyFromEnv`). It names a profile as its `model`; a
26
+ LangChain app asks `@starci/ai-router/langchain` for the client of that name. No app uses the admin token or the admin API; both stay in
25
27
  the package for an operator who runs the standalone gateway.
26
28
  - **Below:** each provider is declared with an array of keys (`<PREFIX>_API_KEY`, `_2`, ...). A lane is one
27
29
  (provider, key, model). The router picks a lane per attempt by health, weight and headroom, and translates the
@@ -34,7 +36,7 @@ package's. The table is what `PROVIDER_PRESETS` (`src/providers.ts`) encodes.
34
36
 
35
37
  | kind | default base URL | protocol | auth header | embeddings | env prefix | docs read |
36
38
  |---|---|---|---|---|---|---|
37
- | `anthropic` | `https://api.anthropic.com/v1` (`/messages`) | Anthropic Messages, through the adapter | `x-api-key`, `anthropic-version: 2023-06-01` | none | `ANTHROPIC` | platform.claude.com/docs/en/api/errors, /api/rate-limits, /api/messages, /build-with-claude/streaming, /build-with-claude/handling-stop-reasons, /build-with-claude/structured-outputs, /models/overview |
39
+ | `anthropic` | `https://api.anthropic.com/v1` (`/messages`) | Anthropic Messages, through the adapter | `x-api-key`, `anthropic-version: 2023-06-01` | none | `ANTHROPIC` | platform.claude.com/docs/en/api/errors, /api/rate-limits, /api/messages, /build-with-claude/streaming, /build-with-claude/handling-stop-reasons, /build-with-claude/structured-outputs, /models/overview; on 2026-11-10 /build-with-claude/prompt-caching, /thinking, /thinking-tool-workflows, /preserved-thinking |
38
40
  | `openrouter` | `https://openrouter.ai/api/v1` | OpenAI | `Authorization: Bearer` (optional `HTTP-Referer`, `X-OpenRouter-Title` as provider `headers`) | yes (`/embeddings`) | `OPENROUTER` | openrouter.ai/docs/api/reference/overview, /errors-and-debugging, /limits, /embeddings |
39
41
  | `qwen` | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` (Singapore; `DASHSCOPE_BASE_URL` for `dashscope.aliyuncs.com` or a workspace host `{WorkspaceId}.<region>.maas.aliyuncs.com`) | OpenAI (DashScope compatible mode) | `Authorization: Bearer` | yes (`text-embedding-v4`, default 1024 dims) | `DASHSCOPE` | alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope, /error-code, /embedding, /models |
40
42
  | `deepseek` | `https://api.deepseek.com` | OpenAI | `Authorization: Bearer` | none | `DEEPSEEK` | api-docs.deepseek.com, /quick_start/error_codes, /quick_start/rate_limit |
@@ -184,11 +186,12 @@ auth headers, request body, unary answer and stream events.
184
186
 
185
187
  | OpenAI request | Anthropic request |
186
188
  |---|---|
187
- | `system` and `developer` messages | top-level `system`, joined by a blank line |
189
+ | `system` and `developer` messages | top-level `system`: one string joined by a blank line, or blocks when a part marks a cache breakpoint |
190
+ | `cache_control` on a part, a message, a tool or the top level | the same breakpoint on the block, the tool or the request ("Prompt caching") |
188
191
  | `user` content string or parts; `image_url` (data URL or URL) | `text` blocks; `image` with a `base64` or `url` source; empty text left out |
189
192
  | `assistant` content and `tool_calls` | `text` blocks then `tool_use` blocks (`arguments` parsed; not JSON is refused) |
190
193
  | `tool` messages | `tool_result` blocks in a user turn; consecutive turns of one role merge |
191
- | `max_completion_tokens` / `max_tokens` | `max_tokens` (required: else the provider's `defaultMaxTokens`, else 4096) |
194
+ | `max_completion_tokens` / `max_tokens` | `max_tokens` (required: else the model's `params`, else the provider's `defaultMaxTokens`, else 16000) |
192
195
  | `stop` | `stop_sequences` |
193
196
  | `tools` (`function`) | `{ name, description, input_schema, strict }`; other tool types pass as they are |
194
197
  | `tool_choice` `auto` / `none` / `required` / `{ function }`, `parallel_tool_calls: false` | `auto` / `none` / `any` / `{ type: "tool", name }`, `disable_parallel_tool_use` |
@@ -199,21 +202,188 @@ auth headers, request body, unary answer and stream events.
199
202
  | `n` above 1 | refused, `invalid_request` |
200
203
  | `seed`, penalties, `logprobs`, `stream_options` | dropped |
201
204
 
202
- Answers: text blocks join into `content`, `tool_use` blocks become `tool_calls`, thinking blocks are dropped;
203
- `stop_reason` maps `end_turn`/`stop_sequence`/`pause_turn` to `stop`, `max_tokens`/`model_context_window_exceeded` to
204
- `length`, `tool_use` to `tool_calls`, `refusal` to `content_filter`; usage counts cache reads and writes into
205
- `prompt_tokens` and reports cache reads as `prompt_tokens_details.cached_tokens`.
205
+ Answers: text blocks join into `content`, `tool_use` blocks become `tool_calls`, the text of thinking blocks (empty
206
+ unless the request asked to see it) becomes `reasoning_content`, and the blocks themselves follow "Thinking with
207
+ tools"; `stop_reason` maps `end_turn`/`stop_sequence`/`pause_turn` to `stop`, `max_tokens`/
208
+ `model_context_window_exceeded` to `length`, `tool_use` to `tool_calls`, `refusal` to `content_filter` with the reason
209
+ as `message.refusal`; usage is the shape of "Prompt caching".
206
210
 
207
211
  Streams: `message_start` becomes the role chunk; `text_delta` content chunks; a `tool_use` block start a `tool_calls`
208
212
  delta with id and name, and its `input_json_delta` pieces argument deltas at the tool call's own index; `message_delta`
209
- the finish reason (its usage is cumulative); `message_stop` a final usage chunk. `ping`, `content_block_stop`, thinking
210
- deltas and event types Anthropic adds later are skipped, as its docs ask. An `error` event is classified like an error
213
+ the finish reason (its usage is cumulative, and the last one holds the thinking tokens); `message_stop` a final usage
214
+ chunk. `ping`, `content_block_stop`, hidden thinking deltas and event types Anthropic adds later carry nothing for the
215
+ caller, as its docs ask, but each restarts the attempt's idle timeout: a model that thinks for a minute before its
216
+ first visible token is at work, not stalled (the call's `deadlineMs` still bounds it). An `error` event is classified like an error
211
217
  status; a stream that ends before `message_stop` is a failed attempt. Failover is before the first token only, as for
212
218
  every provider.
213
219
 
214
- Current models need care the adapter does not take for you: forced tool use (`tool_choice` `any`/`tool`) is refused on
215
- Opus 5.5 and Sonnet 5.5, a `temperature` below 1 on 4.7 and later, and assistant prefill on 4.6 and later. These answer 400
216
- and are not retried.
220
+ Current models refuse requests older ones took: a forced tool choice (`tool_choice` `any`/`tool`) on Opus 5.5 and
221
+ Sonnet 5.5, non-default sampling parameters on 4.7 and later, `thinking: { type: "disabled" }` on Opus 5.5 and
222
+ Sonnet 5.5, assistant prefill on 4.6 and later. They answer 400 and are not retried. The first three are a model's
223
+ config ("A model's own request shape"), never the app's; prefill has no replacement but structured output.
224
+
225
+ The adapter sends `anthropic-version: 2023-06-01`, the version it is written against; a provider header overrides it.
226
+ `2026-10-21` refuses `temperature`, `top_p` and `top_k` at any value.
227
+
228
+ ## Prompt caching
229
+
230
+ A provider caches a prompt prefix either on its own (DeepSeek, OpenAI, Gemini, Groq, most OpenAI-compatible endpoints)
231
+ or at breakpoints the request marks (Anthropic, and `anthropic/*` through OpenRouter). An app marks breakpoints once, in
232
+ the OpenAI shape, and each lane receives them in the form its provider takes, or not at all:
233
+
234
+ ```ts
235
+ await router.chat({
236
+ model: "auto-heavy",
237
+ messages: [
238
+ { role: "system", content: [
239
+ { type: "text", text: craft, cache_control: { type: "ephemeral", ttl: "1h" } }, // L1; the tools render before it
240
+ { type: "text", text: shopRules, cache_control: { type: "ephemeral" } }, // L2
241
+ { type: "text", text: stepGoal, cache_control: { type: "ephemeral" } }, // L3
242
+ ] },
243
+ ...conversation, // L4, append-only
244
+ ],
245
+ tools,
246
+ cache_control: { type: "ephemeral" }, // automatic: one more breakpoint on the last cacheable block
247
+ })
248
+ ```
249
+
250
+ | where the request marks it | Anthropic provider | `anthropic/*` on OpenRouter | every other lane |
251
+ |---|---|---|---|
252
+ | a content part (`text`, `image_url`) of a system, user or assistant message | `cache_control` on that block; a marked system prompt is sent as blocks, one per part | kept on the part | removed |
253
+ | a message (`cache_control` beside `content`) | on the message's last block; on the `tool_result` for a tool message | moved to the last part (a string content becomes one text part) | removed |
254
+ | a tool (`cache_control` beside `function`) | on the tool definition | removed: tools render before the system prompt, whose breakpoint covers them | removed |
255
+ | the top level | top-level `cache_control` (automatic caching) | kept | removed |
256
+
257
+ - **Limit.** Anthropic takes at most four breakpoints, the top-level one included; a request that marks more is refused
258
+ by the adapter as `invalid_request`, before it is sent. The TTL (`"5m"`, the default, or `"1h"`) passes as written; a
259
+ one-hour entry must sit before any five-minute one, which the API enforces.
260
+ - **Who takes markers.** `ModelConfig.cacheControl` decides for a model; without it, an `anthropic` provider and
261
+ `anthropic/*` models on `openrouter` take them and nothing else does. A direct Qwen model or a Gemini model on
262
+ OpenRouter, which take explicit breakpoints too, set `cacheControl: true`.
263
+ - **What is cached.** The render order is tools, system, messages, and a breakpoint caches everything before it. A
264
+ system prompt with no breakpoint is one string; a marked one becomes blocks, so adding the first
265
+ breakpoint changes the prefix once. A prompt below the model's minimum (512 tokens on Sonnet 5.5) is not cached and
266
+ no error says so: read the usage.
267
+
268
+ Usage is one shape for every lane. `prompt_tokens` is the whole prompt; its cache parts are in
269
+ `prompt_tokens_details`: `cached_tokens` (read), `cache_write_tokens` (written, every lifetime together) and
270
+ `cache_write_1h_tokens` (the one-hour part, when the provider splits it); thinking is
271
+ `completion_tokens_details.reasoning_tokens`, a part of `completion_tokens`. Anthropic lists the uncached input apart
272
+ from the cache reads and writes; the adapter adds them up. A model's `cost` prices each part: `cacheRead`, `cacheWrite`
273
+ and `cacheWrite1h` per million tokens, each defaulting to `input` (`cacheWrite1h` to `cacheWrite`), so a model that
274
+ declares only `input` and `output` is priced at those two alone.
275
+
276
+ ## Thinking with tools
277
+
278
+ Current Claude models think by default, and on Sonnet 5.5 and Opus 5.5 thinking cannot be turned off (`thinking:
279
+ { type: "disabled" }` is a 400; Sonnet 5.5's lowest setting is `{ type: "between_tools" }`, which no other model
280
+ accepts). An answer that calls a tool carries `thinking` blocks with a signature, and the API binds each block to the
281
+ model, the account and the conversation before it (system, tools and earlier messages).
282
+
283
+ The OpenAI shape has no field for those blocks, so the adapter chooses per provider, `ProviderConfig.thinking`:
284
+
285
+ - **`drop`, the default.** The blocks are never sent back: the next request of the tool loop carries the assistant turn
286
+ as text and `tool_use` only. Anthropic's preserved-thinking page lists removing every thinking block as valid ("the
287
+ model loses that reasoning"), nothing fails, and the prompt cache still matches, since the cached prefix is the
288
+ request and not the answer. It is the default because it can never be refused: nothing replayed, nothing to bind.
289
+ - **`replay`, the opt-in.** The adapter keeps each answer that thought and then called tools (its content blocks as
290
+ received, up to 2000 turns per provider in this process, least recently used first) under the ids of its tool calls.
291
+ When a later request carries an assistant message with exactly those tool calls, the turn goes back as it was
292
+ received, thinking and signatures included, whatever the client above forwarded. It is replayed only while the
293
+ request up to that turn (system, tools, earlier turns; key order and breakpoints aside) has the digest it had when the
294
+ turn was produced; after an edit the turn goes back rebuilt, without thinking, and so does every later one, which is
295
+ the documented way to continue. A turn the memory forgot, or a second process, degrades to `drop`.
296
+
297
+ Thinking text reaches the caller as `reasoning_content` (the message, or stream deltas) when the request asks to see it
298
+ (`thinking: { type: "adaptive", display: "summarized" }`); by default Claude returns it empty. Thinking tokens are
299
+ billed as output and count toward `max_tokens`, which is why the adapter's default is 16000.
300
+
301
+ An app that speaks the Anthropic protocol to the gateway (below) needs neither: its client carries the blocks itself.
302
+
303
+ ## A model's own request shape
304
+
305
+ What one model needs and another refuses is the model's config, applied to each attempt just before the request is
306
+ translated for that lane, so a request that fails over reaches the next model in that model's shape:
307
+
308
+ | `ModelConfig` | effect |
309
+ |---|---|
310
+ | `params` | request fields sent when the request does not set them (`{ max_tokens: 32000, output_config: { effort: "medium" } }`); shallow |
311
+ | `omit` | request fields never sent (`["temperature", "top_p", "top_k"]` for Sonnet 5.5, which refuses non-default sampling) |
312
+ | `forcedToolUse: false` | `tool_choice` `required` or a named function becomes `auto` plus a system instruction to make the call, for models that refuse a forced choice (Opus 5.5, Sonnet 5.5, Fable 5.1 and later); `auto` guarantees no call, so the caller checks one was made |
313
+ | `cacheControl` | whether its provider takes cache breakpoints (above) |
314
+
315
+ ## The native Anthropic route
316
+
317
+ `POST /v1/messages` on the gateway speaks the Anthropic Messages protocol, under the same virtual key (read from
318
+ `x-api-key`, or a bearer token). The request goes to an Anthropic provider untranslated: only `model` becomes the
319
+ lane's model id, and the model's own shape applies (`params`, `omit`, and `forcedToolUse: false`, which turns
320
+ `tool_choice` `any` or `tool` into `auto` with the instruction at the end of `system`). Cache breakpoints, thinking blocks, tool results and
321
+ whatever the API adds later pass through, and so does the client's `anthropic-beta` header. The answer and the stream
322
+ events come back as the provider sent them. The router reads only what it accounts for: usage, the first content
323
+ (failover is over once a `content_block_delta` arrived), an `error` event, a stream cut before `message_stop`.
324
+
325
+ - **Routing.** The call is narrowed to the providers of the Anthropic protocol; keys rotate and fail over among them as
326
+ for any call. `router.messages()` and `router.messagesStream()` are the same calls in process.
327
+ - **A name no Anthropic provider serves** is refused 400 `model_not_supported`, naming the route that serves it.
328
+ Nothing translates a Messages request for an OpenAI-protocol provider: that adapter would be a second protocol
329
+ surface to keep correct for a path with no caller, since an app asks for the client of the name's own protocol.
330
+ - **Errors** on this route are in Anthropic's shape (`{ type: "error", error: { type, message, code } }`), and a failure
331
+ after the first event is an `error` event.
332
+
333
+ A profile whose candidates span two protocols is served on each route by its own protocol's candidates. On
334
+ `/v1/chat/completions` such a profile is served by every candidate, Anthropic ones through the adapter, unless the
335
+ request carries `x-ai-router-kind: openai` (or `anthropic`), which narrows the call to that protocol's providers.
336
+
337
+ `GET /v1/models` says how each name is served: `kind` (the protocol of its first candidate), `kinds` (every protocol,
338
+ in the order the pool tries them), `serves` (`chat`, `embeddings`) and, for an embeddings name, `dimensions`.
339
+
340
+ ## Who served, and what it cost
341
+
342
+ Every answer names its lane where each client library hands it on untouched: inside the provider's `usage`, as
343
+ `usage.served = { provider, model, attempts, cost? }` (the pool's provider id and model id, never a key). It is there
344
+ on both routes, unary and streamed: in a chat completion's `usage` and its final usage chunk, in an Anthropic message's
345
+ `usage` and in the `usage` of its `message_start` event. The response headers say the same (`x-ai-router-provider`,
346
+ `x-ai-router-model`, `x-ai-router-attempts`, `x-ai-router-cost`). In process, `ChatCompletion.routing` and the `attempt`
347
+ event carry it with the key's alias; a client may name its call with an `x-request-id` header (letters, digits and
348
+ `._:-`, up to 128, unique per call since providers see it as the idempotency key) to match the gateway's call to
349
+ those events.
350
+
351
+ ## The LangChain entry
352
+
353
+ `@starci/ai-router/langchain` gives an app LangChain clients over the gateway, by name:
354
+
355
+ ```ts
356
+ const ai = createRoutedModels({ gatewayUrl, virtualKey })
357
+ const picked = await ai.chat(node.model) // a pool model id or a profile
358
+ const agent = createAgent({ model: picked.model, tools })
359
+ const usage = usageOf(reply) // tokens by part, model, provider: the same for every kind
360
+ ```
361
+
362
+ - **`kind` is the protocol**, never the model family: `{ kind: "openai", model: ChatOpenAI }` for every OpenAI-compatible
363
+ provider, `{ kind: "anthropic", model: ChatAnthropic }` for an Anthropic one. It comes from the gateway's catalog
364
+ (`GET /v1/models`, read once and again when a name is unknown). The common path uses `picked.model` and never reads
365
+ `kind`; a provider's own feature does.
366
+ - **A profile of two protocols** is the client of its first candidate's protocol, with the other protocols' clients in
367
+ `picked.fallbacks`, in the order the pool tries them. Each client sends `x-ai-router-kind`, so it stays on its own
368
+ protocol's providers: the router fails over between keys and providers inside a protocol, and moving to the next
369
+ protocol is the app's retry. The fallbacks are not wrapped into one runnable because LangChain's
370
+ `RunnableWithFallbacks` is no chat model (it has no `bindTools`), so an agent could not take it, and because the
371
+ clients never retry (`maxRetries: 0`): the app owns its tries and decides which one moves on.
372
+ - **New kinds** extend one interface, `RoutedChatKinds` (module augmentation), and register a factory with
373
+ `registerChatKind`; the core and the built-in kinds are untouched.
374
+ - **Optional peers.** `@langchain/openai`, `@langchain/anthropic` and `@langchain/core` are optional peer dependencies,
375
+ imported the first time a kind is built; the router's core imports none of them.
376
+ - **Usage.** `usageOf(reply)` reads LangChain's `usage_metadata` (which both clients fill the same way: the whole
377
+ input, with `cache_read` and `cache_creation` as parts) and the `usage.served` the gateway wrote, which both keep in
378
+ `response_metadata.usage`: `inputTokens` (full price), `cacheReadTokens`, `cacheWriteTokens`, `outputTokens`,
379
+ `reasoningTokens`, `model`, `provider`, `cost`.
380
+ - **Embeddings.** `ai.embeddings(name)` is `{ kind: "openai", embeddings: OpenAIEmbeddings, dimensions }` for a pinned
381
+ profile or an embeddings model; it never changes model identity, and Anthropic has no embeddings API.
382
+
383
+ Verified against `@langchain/openai` 1.8.2, `@langchain/anthropic` 1.6.2 and `@langchain/core` 1.3.6: the constructor
384
+ fields the entry sets (`configuration.baseURL`, `anthropicApiUrl`, `clientOptions`, `maxRetries`, `streamUsage`), and
385
+ where each puts usage. `ChatAnthropic` sends `thinking` only when the app sets it, and its own `max_tokens` default for
386
+ a name it does not know is 4096, so the entry sends 16000 unless told otherwise.
217
387
 
218
388
  ## Config from env
219
389
 
@@ -237,7 +407,7 @@ has; a profile left with nothing is refused at startup naming the variables that
237
407
  ```ts
238
408
  const pool = poolFromEnv({
239
409
  providers: [
240
- { kind: "anthropic", models: [{ id: "claude-sonnet-5-5", cost: { input: 3, output: 15 } }] },
410
+ { kind: "anthropic", models: [{ id: "claude-sonnet-5-5", cost: { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5, cacheWrite1h: 4 } }] },
241
411
  { kind: "openrouter", models: [{ id: "anthropic/claude-sonnet-5-5" }, { id: "openai/text-embedding-3-small", identity: "text-embedding-3-small", dimensions: 1536, capabilities: ["embeddings"] }] },
242
412
  { kind: "deepseek", models: [{ id: "deepseek-v4-pro" }] },
243
413
  { kind: "qwen", models: [{ id: "qwen3.8-max" }] },
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@starci/ai-router",
3
- "version": "0.6.1",
4
- "description": "A router for OpenAI-compatible calls across many providers (Anthropic through a protocol adapter, OpenRouter, Qwen, DeepSeek, OpenAI, Gemini, Groq and any OpenAI-compatible endpoint), each with an array of keys, behind one virtual key: an OpenAI-compatible gateway with virtual keys, quotas and usage, a NestJS module (AiRouterModule, AiRouterService, AiRouterGatewayModule), and createRouter for plain Node, with health-aware selection, a persisted key blacklist for spent quotas and refused secrets, circuit breakers, rate-limit awareness, failover across providers and models by profile, pinned embeddings, config from env and redacted telemetry.",
3
+ "version": "0.7.0",
4
+ "description": "A router for chat and embeddings calls across many providers (Anthropic, OpenRouter, Qwen, DeepSeek, OpenAI, Gemini, Groq and any OpenAI-compatible endpoint), each with an array of keys, behind one virtual key: a gateway that speaks the OpenAI and the Anthropic Messages protocols with virtual keys, quotas and usage, LangChain clients over it (@starci/ai-router/langchain), a NestJS module (AiRouterModule, AiRouterService, AiRouterGatewayModule), and createRouter for plain Node, with health-aware selection, a persisted key blacklist for spent quotas and refused secrets, circuit breakers, rate-limit awareness, failover across providers and models by profile, prompt-cache breakpoints and cache-priced usage, pinned embeddings, config from env and redacted telemetry.",
5
5
  "keywords": [
6
6
  "ai",
7
7
  "llm",
@@ -26,7 +26,8 @@
26
26
  "deepseek",
27
27
  "gemini",
28
28
  "quota",
29
- "blacklist"
29
+ "blacklist",
30
+ "prompt-caching"
30
31
  ],
31
32
  "license": "MIT",
32
33
  "author": "starci183",
@@ -60,6 +61,10 @@
60
61
  "types": "./dist/testing.d.ts",
61
62
  "default": "./dist/testing.js"
62
63
  },
64
+ "./langchain": {
65
+ "types": "./dist/langchain/index.d.ts",
66
+ "default": "./dist/langchain/index.js"
67
+ },
63
68
  "./package.json": "./package.json"
64
69
  },
65
70
  "files": [
@@ -74,6 +79,9 @@
74
79
  "node": ">=20"
75
80
  },
76
81
  "peerDependencies": {
82
+ "@langchain/anthropic": "^1.6.2",
83
+ "@langchain/core": "^1.3.5",
84
+ "@langchain/openai": "^1.8.2",
77
85
  "@nestjs/common": "^11.0.0 || ^12.0.0",
78
86
  "@nestjs/core": "^11.0.0 || ^12.0.0",
79
87
  "reflect-metadata": "^0.1.12 || ^0.2.0"
@@ -91,10 +99,13 @@
91
99
  },
92
100
  "devDependencies": {
93
101
  "@eslint/js": "10.0.1",
94
- "@starci/common": "file:../common",
102
+ "@langchain/anthropic": "1.6.2",
103
+ "@langchain/core": "1.3.6",
104
+ "@langchain/openai": "1.8.2",
95
105
  "@nestjs/common": "12.1.3",
96
106
  "@nestjs/core": "12.1.3",
97
107
  "@nestjs/testing": "12.1.3",
108
+ "@starci/common": "file:../common",
98
109
  "@types/node": "24.19.2",
99
110
  "eslint": "10.13.0",
100
111
  "jiti": "2.7.0",
@@ -108,6 +119,15 @@
108
119
  "yaml": "^2.9.1"
109
120
  },
110
121
  "peerDependenciesMeta": {
122
+ "@langchain/anthropic": {
123
+ "optional": "true"
124
+ },
125
+ "@langchain/core": {
126
+ "optional": "true"
127
+ },
128
+ "@langchain/openai": {
129
+ "optional": "true"
130
+ },
111
131
  "@nestjs/common": {
112
132
  "optional": "true"
113
133
  },
@@ -118,6 +138,6 @@
118
138
  "optional": "true"
119
139
  }
120
140
  },
121
- "gitHead": "5ba3e3393ad44562f0245d829949e70a2313af81",
122
- "starciSource": "236cf93297937b0438f9af5156b9a4794d2d1c54c05b91331e0fd16db80cbae6"
141
+ "gitHead": "11cc31d88671720f7e383e2bb3bd7a5d00bec945",
142
+ "starciSource": "cfdf256ccc3c4cd8e9046ff4c70e703a3c3d9e704cbabf022672fa421f46fbcc"
123
143
  }