@starci/ai-router 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,137 @@
1
+ import { DomainError } from "@starci/common";
2
+ import type { ChatAnthropic } from "@langchain/anthropic";
3
+ import type { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai";
4
+ /**
5
+ * LangChain clients over the gateway (docs/design.md, "The LangChain entry"). An app holds one virtual key and names a
6
+ * pool model or a profile; it gets back the LangChain chat model that speaks the protocol the gateway serves that name
7
+ * through. The app never builds a provider client and never branches on a provider.
8
+ *
9
+ * The LangChain packages are optional peers, imported the first time a kind is asked for: an app that never asks for an
10
+ * `anthropic` model never needs `@langchain/anthropic`, and the router's core imports none of them.
11
+ */
12
+ /**
13
+ * The chat model of each kind. A kind is a wire protocol, never a model family: every OpenAI-compatible provider
14
+ * (DeepSeek, OpenRouter, Qwen, Groq, ...) is `openai`. A new kind is added by declaring its model here (module
15
+ * augmentation) and registering its factory with `registerChatKind`; nothing else changes.
16
+ */
17
+ export interface RoutedChatKinds {
18
+ openai: ChatOpenAI;
19
+ anthropic: ChatAnthropic;
20
+ }
21
+ /** A wire protocol the gateway serves a name through. */
22
+ export type RoutedKind = keyof RoutedChatKinds;
23
+ /** One chat model the gateway routes, with the protocol it speaks. The common path uses `model` and never reads `kind`. */
24
+ export type RoutedChat<K extends RoutedKind = RoutedKind> = {
25
+ [Kind in K]: {
26
+ readonly kind: Kind;
27
+ /** The pool model or profile this client names on every call. */
28
+ readonly name: string;
29
+ readonly model: RoutedChatKinds[Kind];
30
+ /**
31
+ * The same name through its other protocols, in the order the pool tries them: a profile whose candidates span two
32
+ * protocols is one client here and the next ones there. The router fails over inside a protocol on its own; moving
33
+ * to the next protocol is the caller's retry, which it already owns.
34
+ */
35
+ readonly fallbacks: readonly RoutedChat[];
36
+ };
37
+ }[K];
38
+ /** The embeddings client of a pinned profile or an embeddings model, with the length of its vectors. */
39
+ export interface RoutedEmbeddings {
40
+ readonly kind: "openai";
41
+ readonly name: string;
42
+ readonly embeddings: OpenAIEmbeddings;
43
+ /** The vector length the pool declares for it; undefined when the pool declares none. */
44
+ readonly dimensions: number | undefined;
45
+ }
46
+ /** A `fetch`-compatible function. */
47
+ export type RoutedFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
48
+ /** How the routed clients reach the gateway. */
49
+ export interface RoutedModelsOptions {
50
+ /** The gateway's origin, without `/v1` (`http://127.0.0.1:4100`). */
51
+ readonly gatewayUrl: string;
52
+ /** The app's one virtual key (`STARCI_AI_KEY`). */
53
+ readonly virtualKey: string;
54
+ /** The `fetch` every client and the catalog read use. Default: the global one. */
55
+ readonly fetch?: RoutedFetch;
56
+ /** The limit of one call in milliseconds, for every client unless a call to `chat` sets its own. */
57
+ readonly timeoutMs?: number;
58
+ /** Extra headers on every request to the gateway. */
59
+ readonly headers?: Readonly<Record<string, string>>;
60
+ }
61
+ /** What one `chat` call may set on the client it gets. */
62
+ export interface RoutedChatOptions {
63
+ /** The limit of one call in milliseconds. */
64
+ readonly timeoutMs?: number;
65
+ /** The most tokens an answer may hold. An `anthropic` client must send one: default 16000. */
66
+ readonly maxTokens?: number;
67
+ /** Extra constructor fields of the kind's own client, for a provider's own features; they win over the defaults. */
68
+ readonly fields?: Readonly<Partial<Record<RoutedKind, Readonly<Record<string, unknown>>>>>;
69
+ }
70
+ /** What a kind's factory is given to build its client. */
71
+ export interface RoutedChatContext {
72
+ readonly name: string;
73
+ /** The gateway's origin, without a trailing slash. */
74
+ readonly gatewayUrl: string;
75
+ readonly virtualKey: string;
76
+ readonly fetch: RoutedFetch | undefined;
77
+ readonly timeoutMs: number | undefined;
78
+ readonly maxTokens: number | undefined;
79
+ /** The headers every request carries: the caller's, and the one that keeps the call on this kind's providers. */
80
+ readonly headers: Readonly<Record<string, string>>;
81
+ /** The caller's own constructor fields for this kind. */
82
+ readonly fields: Readonly<Record<string, unknown>>;
83
+ }
84
+ /** Builds the chat model of one kind; it imports its LangChain package itself, so the package loads only when asked for. */
85
+ export type RoutedChatFactory<K extends RoutedKind = RoutedKind> = (context: RoutedChatContext) => Promise<RoutedChatKinds[K]>;
86
+ /** Why the routed models refused. */
87
+ export type RoutedModelsErrorCode = "ROUTED_CATALOG_UNAVAILABLE" | "ROUTED_NAME_UNKNOWN" | "ROUTED_KIND_UNSUPPORTED" | "ROUTED_NOT_CHAT" | "ROUTED_NOT_EMBEDDINGS" | "ROUTED_PACKAGE_MISSING";
88
+ /** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
89
+ export declare class RoutedModelsError extends DomainError<RoutedModelsErrorCode> {
90
+ constructor(code: RoutedModelsErrorCode, message: string, cause?: unknown);
91
+ }
92
+ /** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
93
+ export declare const ROUTED_ANTHROPIC_MAX_TOKENS = 16000;
94
+ /** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
95
+ export declare function registerChatKind<K extends RoutedKind>(kind: K, factory: RoutedChatFactory<K>): void;
96
+ /** The routed clients of one app. */
97
+ export interface RoutedModels {
98
+ /**
99
+ * The chat model of a pool model or profile, speaking the protocol of its first candidate. `createAgent({ model:
100
+ * (await ai.chat(name)).model })` works for every kind; `kind` is read only for a provider's own features.
101
+ */
102
+ chat(name: string, options?: RoutedChatOptions): Promise<RoutedChat>;
103
+ /** The embeddings client of a pinned profile or an embeddings model. It never falls back to another model identity. */
104
+ embeddings(name: string): Promise<RoutedEmbeddings>;
105
+ /** Reads the gateway's catalog again: after the pool's config changed. */
106
+ refresh(): Promise<void>;
107
+ }
108
+ /** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
109
+ export declare function createRoutedModels(options: RoutedModelsOptions): RoutedModels;
110
+ /** What one answer cost in tokens and who served it, the same for every kind: what credit billing reads. */
111
+ export interface RoutedUsage {
112
+ /** Prompt tokens billed at the full input price: the prompt without its cache reads and writes. */
113
+ readonly inputTokens: number;
114
+ /** Prompt tokens read from the provider's prompt cache. */
115
+ readonly cacheReadTokens: number;
116
+ /** Prompt tokens written to the provider's prompt cache. */
117
+ readonly cacheWriteTokens: number;
118
+ /** Completion tokens, thinking included. */
119
+ readonly outputTokens: number;
120
+ /** The part of `outputTokens` the model spent thinking; 0 when the provider does not count it. */
121
+ readonly reasoningTokens: number;
122
+ /** The pool model that answered, as the pool's config names it; undefined when the answer carried no usage. */
123
+ readonly model: string | undefined;
124
+ /** The pool provider that answered. */
125
+ readonly provider: string | undefined;
126
+ /** The cost the router computed from the model's prices; undefined when the model declares none or the answer was streamed natively. */
127
+ readonly cost: number | undefined;
128
+ }
129
+ /**
130
+ * The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
131
+ * in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
132
+ * gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
133
+ */
134
+ export declare function usageOf(reply: {
135
+ readonly usage_metadata?: unknown;
136
+ readonly response_metadata?: unknown;
137
+ }): RoutedUsage;
@@ -0,0 +1,182 @@
1
+ import { DomainError, isPlainRecord } from "@starci/common";
2
+ /** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
3
+ export class RoutedModelsError extends DomainError {
4
+ constructor(code, message, cause) {
5
+ super({ code, ...(cause === undefined ? {} : { cause }) });
6
+ this.message = message;
7
+ this.name = "RoutedModelsError";
8
+ }
9
+ }
10
+ /** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
11
+ export const ROUTED_ANTHROPIC_MAX_TOKENS = 16_000;
12
+ /** The header that keeps a call on the providers of one protocol. */
13
+ const KIND_HEADER = "x-ai-router-kind";
14
+ async function load(name, importer) {
15
+ try {
16
+ return await importer();
17
+ }
18
+ catch (error) {
19
+ throw new RoutedModelsError("ROUTED_PACKAGE_MISSING", `install ${name} to use this kind: it is an optional peer of @starci/ai-router/langchain`, error);
20
+ }
21
+ }
22
+ const factories = new Map();
23
+ /** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
24
+ export function registerChatKind(kind, factory) {
25
+ factories.set(kind, factory);
26
+ }
27
+ registerChatKind("openai", async (context) => {
28
+ const { ChatOpenAI: Model } = await load("@langchain/openai", () => import("@langchain/openai"));
29
+ return new Model({
30
+ model: context.name,
31
+ apiKey: context.virtualKey,
32
+ // The app owns its retries: the router already fails over between keys and providers of the kind.
33
+ maxRetries: 0,
34
+ streamUsage: true,
35
+ ...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }),
36
+ ...(context.maxTokens === undefined ? {} : { maxTokens: context.maxTokens }),
37
+ ...context.fields,
38
+ configuration: { baseURL: `${context.gatewayUrl}/v1`, defaultHeaders: { ...context.headers }, ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
39
+ });
40
+ });
41
+ registerChatKind("anthropic", async (context) => {
42
+ const { ChatAnthropic: Model } = await load("@langchain/anthropic", () => import("@langchain/anthropic"));
43
+ return new Model({
44
+ model: context.name,
45
+ apiKey: context.virtualKey,
46
+ // The Anthropic client appends /v1/messages to its base URL itself.
47
+ anthropicApiUrl: context.gatewayUrl,
48
+ maxRetries: 0,
49
+ streamUsage: true,
50
+ maxTokens: context.maxTokens ?? ROUTED_ANTHROPIC_MAX_TOKENS,
51
+ ...context.fields,
52
+ clientOptions: { defaultHeaders: { ...context.headers }, ...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }), ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
53
+ });
54
+ });
55
+ function strings(value) {
56
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
57
+ }
58
+ function listedOf(entry) {
59
+ if (!isPlainRecord(entry) || typeof entry.id !== "string")
60
+ return undefined;
61
+ return { id: entry.id, kinds: strings(entry.kinds), serves: strings(entry.serves), dimensions: typeof entry.dimensions === "number" ? entry.dimensions : undefined };
62
+ }
63
+ /** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
64
+ export function createRoutedModels(options) {
65
+ const gatewayUrl = options.gatewayUrl.replace(/\/+$/, "").replace(/\/v1$/, "");
66
+ const fetcher = options.fetch ?? ((input, init) => fetch(input, init));
67
+ let catalog;
68
+ async function read() {
69
+ let body;
70
+ try {
71
+ const response = await fetcher(`${gatewayUrl}/v1/models`, { headers: { authorization: `Bearer ${options.virtualKey}`, ...options.headers } });
72
+ if (!response.ok)
73
+ throw new Error(`HTTP ${String(response.status)}`);
74
+ body = await response.json();
75
+ }
76
+ catch (error) {
77
+ throw new RoutedModelsError("ROUTED_CATALOG_UNAVAILABLE", `the gateway at ${gatewayUrl} did not list its models: ${error instanceof Error ? error.message : "the request failed"}`, error);
78
+ }
79
+ const data = isPlainRecord(body) && Array.isArray(body.data) ? body.data : [];
80
+ return new Map(data.map(listedOf).filter((entry) => entry !== undefined).map((entry) => [entry.id, entry]));
81
+ }
82
+ function listed() {
83
+ catalog ??= read().catch((error) => {
84
+ // A failed read is not remembered: the next call asks again.
85
+ catalog = undefined;
86
+ throw error;
87
+ });
88
+ return catalog;
89
+ }
90
+ async function find(name) {
91
+ let entry = (await listed()).get(name);
92
+ if (entry === undefined) {
93
+ // The pool may have gained the name since the catalog was read.
94
+ catalog = undefined;
95
+ entry = (await listed()).get(name);
96
+ }
97
+ if (entry === undefined)
98
+ throw new RoutedModelsError("ROUTED_NAME_UNKNOWN", `the gateway serves no model or profile "${name}" to this key`);
99
+ return entry;
100
+ }
101
+ async function build(kind, name, call) {
102
+ const factory = factories.get(kind);
103
+ if (factory === undefined)
104
+ throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `"${name}" is served through the ${kind} protocol, which has no registered chat kind`);
105
+ const fields = call.fields?.[kind] ?? {};
106
+ const model = await factory({
107
+ name,
108
+ gatewayUrl,
109
+ virtualKey: options.virtualKey,
110
+ fetch: options.fetch,
111
+ timeoutMs: call.timeoutMs ?? options.timeoutMs,
112
+ maxTokens: call.maxTokens,
113
+ headers: { ...options.headers, [KIND_HEADER]: kind },
114
+ fields,
115
+ });
116
+ // The registry holds each kind's factory under its own key, so the model is the kind's.
117
+ return { kind, name, model, fallbacks: [] };
118
+ }
119
+ return {
120
+ async chat(name, call = {}) {
121
+ const entry = await find(name);
122
+ if (!entry.serves.includes("chat"))
123
+ throw new RoutedModelsError("ROUTED_NOT_CHAT", `"${name}" does not serve chat`);
124
+ const [first, ...rest] = entry.kinds;
125
+ if (first === undefined)
126
+ throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `the gateway names no protocol for "${name}"`);
127
+ const primary = await build(first, name, call);
128
+ const fallbacks = await Promise.all(rest.filter((kind) => factories.has(kind)).map((kind) => build(kind, name, call)));
129
+ return { ...primary, fallbacks };
130
+ },
131
+ async embeddings(name) {
132
+ const entry = await find(name);
133
+ if (!entry.serves.includes("embeddings"))
134
+ throw new RoutedModelsError("ROUTED_NOT_EMBEDDINGS", `"${name}" does not serve embeddings`);
135
+ const { OpenAIEmbeddings: Embeddings } = await load("@langchain/openai", () => import("@langchain/openai"));
136
+ const embeddings = new Embeddings({
137
+ model: name,
138
+ apiKey: options.virtualKey,
139
+ maxRetries: 0,
140
+ ...(options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }),
141
+ configuration: { baseURL: `${gatewayUrl}/v1`, defaultHeaders: { ...options.headers }, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) },
142
+ });
143
+ return { kind: "openai", name, embeddings, dimensions: entry.dimensions };
144
+ },
145
+ async refresh() {
146
+ catalog = undefined;
147
+ await listed();
148
+ },
149
+ };
150
+ }
151
+ function count(value) {
152
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
153
+ }
154
+ function record(value) {
155
+ return isPlainRecord(value) ? value : {};
156
+ }
157
+ /**
158
+ * The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
159
+ * in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
160
+ * gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
161
+ */
162
+ export function usageOf(reply) {
163
+ const usage = record(reply.usage_metadata);
164
+ const metadata = record(reply.response_metadata);
165
+ const raw = record(metadata.usage);
166
+ const routing = record(raw.served);
167
+ const inputDetails = record(usage.input_token_details);
168
+ const cacheReadTokens = count(inputDetails.cache_read);
169
+ const cacheWriteTokens = count(inputDetails.cache_creation);
170
+ // OpenAI's raw usage names thinking tokens `reasoning_tokens`, Anthropic's `thinking_tokens`.
171
+ const reasoningTokens = count(record(usage.output_token_details).reasoning) || count(record(raw.completion_tokens_details).reasoning_tokens) || count(record(raw.output_tokens_details).thinking_tokens);
172
+ return {
173
+ inputTokens: Math.max(0, count(usage.input_tokens) - cacheReadTokens - cacheWriteTokens),
174
+ cacheReadTokens,
175
+ cacheWriteTokens,
176
+ outputTokens: count(usage.output_tokens),
177
+ reasoningTokens,
178
+ model: typeof routing.model === "string" ? routing.model : undefined,
179
+ provider: typeof routing.provider === "string" ? routing.provider : undefined,
180
+ cost: typeof routing.cost === "number" ? routing.cost : undefined,
181
+ };
182
+ }
@@ -0,0 +1,7 @@
1
+ import type { ModelConfig } from "./types.js";
2
+ /**
3
+ * The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
4
+ * unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
5
+ * fields, and its forced choice is unforced in that shape.
6
+ */
7
+ export declare function shapeRequest(body: Record<string, unknown>, model: ModelConfig, native?: boolean): Record<string, unknown>;
@@ -0,0 +1,54 @@
1
+ import { isPlainRecord } from "@starci/common";
2
+ /**
3
+ * What one model needs of a chat request and another would refuse, applied from the model's own config just before the
4
+ * request is sent to its provider: default fields (`params`), fields never sent (`omit`) and a forced tool choice the
5
+ * model refuses (`forcedToolUse: false`). The app above sends one request shape whatever lane serves it.
6
+ */
7
+ const ANY_TOOL = "You must call one of the provided tools in this response.";
8
+ function namedTool(name) {
9
+ return `You must call the tool "${name}" in this response.`;
10
+ }
11
+ function isSystem(message) {
12
+ return isPlainRecord(message) && (message.role === "system" || message.role === "developer");
13
+ }
14
+ /** An OpenAI request's forced tool choice as `auto` plus a system instruction, placed after the leading system messages. */
15
+ function unforced(body) {
16
+ const choice = body.tool_choice;
17
+ const text = choice === "required" ? ANY_TOOL : isPlainRecord(choice) && isPlainRecord(choice.function) && typeof choice.function.name === "string" ? namedTool(choice.function.name) : undefined;
18
+ if (text === undefined)
19
+ return body;
20
+ const messages = Array.isArray(body.messages) ? body.messages : [];
21
+ const lead = messages.findIndex((message) => !isSystem(message));
22
+ const at = lead === -1 ? messages.length : lead;
23
+ return { ...body, tool_choice: "auto", messages: [...messages.slice(0, at), { role: "system", content: text }, ...messages.slice(at)] };
24
+ }
25
+ /** An Anthropic request's forced tool choice (`any`, `tool`) as `auto` plus an instruction at the end of its system prompt. */
26
+ function unforcedNative(body) {
27
+ const choice = body.tool_choice;
28
+ if (!isPlainRecord(choice))
29
+ return body;
30
+ const text = choice.type === "any" ? ANY_TOOL : choice.type === "tool" && typeof choice.name === "string" ? namedTool(choice.name) : undefined;
31
+ if (text === undefined)
32
+ return body;
33
+ const system = body.system;
34
+ const block = { type: "text", text };
35
+ return {
36
+ ...body,
37
+ tool_choice: { type: "auto", ...(choice.disable_parallel_tool_use === true ? { disable_parallel_tool_use: true } : {}) },
38
+ system: Array.isArray(system) ? [...system, block] : typeof system === "string" && system !== "" ? `${system}\n\n${text}` : text,
39
+ };
40
+ }
41
+ /**
42
+ * The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
43
+ * unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
44
+ * fields, and its forced choice is unforced in that shape.
45
+ */
46
+ export function shapeRequest(body, model, native = false) {
47
+ if (model.params === undefined && model.omit === undefined && model.forcedToolUse !== false)
48
+ return body;
49
+ const omitted = new Set(model.omit ?? []);
50
+ const out = Object.fromEntries(Object.entries({ ...model.params, ...body }).filter(([field]) => !omitted.has(field)));
51
+ if (model.forcedToolUse !== false)
52
+ return out;
53
+ return native ? unforcedNative(out) : unforced(out);
54
+ }
package/dist/pool.d.ts CHANGED
@@ -56,7 +56,11 @@ export interface Blocked {
56
56
  /** Set when every lane is out because its key is blacklisted or its provider's budget is spent. */
57
57
  readonly blacklisted: Blacklisted | null;
58
58
  }
59
- /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
59
+ /**
60
+ * The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
61
+ * is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
62
+ * one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
63
+ */
60
64
  export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
61
65
  /**
62
66
  * The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
package/dist/pool.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { tokenParts } from "./cache.js";
1
2
  import { windowEnd } from "./classify.js";
2
3
  import { DEFAULT_CAPABILITIES, identityOf, modelMatches } from "./config.js";
3
4
  import { presetOf } from "./providers.js";
@@ -13,12 +14,20 @@ function breakerState(record, now) {
13
14
  return "closed";
14
15
  return record.openUntil > now ? "open" : "half-open";
15
16
  }
16
- /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
17
+ /**
18
+ * The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
19
+ * is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
20
+ * one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
21
+ */
17
22
  export function costOf(model, usage) {
18
23
  if (model.cost === undefined || usage === undefined)
19
24
  return undefined;
20
- const completion = usage.completion_tokens ?? 0;
21
- return (usage.prompt_tokens * model.cost.input + completion * model.cost.output) / 1_000_000;
25
+ const { input, output } = model.cost;
26
+ const cacheWrite = model.cost.cacheWrite ?? input;
27
+ const parts = tokenParts(usage);
28
+ const plain = usage.prompt_tokens - parts.cached - parts.cacheWrite;
29
+ const prompt = plain * input + parts.cached * (model.cost.cacheRead ?? input) + (parts.cacheWrite - parts.cacheWrite1h) * cacheWrite + parts.cacheWrite1h * (model.cost.cacheWrite1h ?? cacheWrite);
30
+ return (prompt + (usage.completion_tokens ?? 0) * output) / 1_000_000;
22
31
  }
23
32
  /** The UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`) of a time. */
24
33
  function periods(now) {
@@ -343,7 +352,7 @@ export class PoolState {
343
352
  #count(lane) {
344
353
  let counters = this.#counters.get(lane.id);
345
354
  if (counters === undefined) {
346
- counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cost: 0 };
355
+ counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, cost: 0 };
347
356
  this.#counters.set(lane.id, counters);
348
357
  }
349
358
  return counters;
@@ -427,6 +436,10 @@ export class PoolState {
427
436
  const cost = costOf(lane.model, usage);
428
437
  counters.promptTokens += usage?.prompt_tokens ?? 0;
429
438
  counters.completionTokens += usage?.completion_tokens ?? 0;
439
+ const parts = tokenParts(usage);
440
+ counters.cachedTokens += parts.cached;
441
+ counters.cacheWriteTokens += parts.cacheWrite;
442
+ counters.reasoningTokens += parts.reasoning;
430
443
  counters.cost += cost ?? 0;
431
444
  await this.#spend(lane, cost, now);
432
445
  return cost;
package/dist/router.js CHANGED
@@ -10,8 +10,10 @@ import { baseUrlOf, presetOf } from "./providers.js";
10
10
  import { resolveSecret } from "./secrets.js";
11
11
  import { readChunk, readCompletion, readEmbeddings, readUsage, vectorLength } from "./shapes.js";
12
12
  import { parseSse } from "./sse.js";
13
+ import { shapeRequest } from "./model-request.js";
13
14
  import { MemoryStateStore } from "./store.js";
14
15
  import { wireOf } from "./wire/index.js";
16
+ import { nativeItems, nativeRequest, nativeUsage } from "./wire/native.js";
15
17
  const BUSY_POLL_MS = 250;
16
18
  const ERROR_BODY_LIMIT = 4096;
17
19
  /** One call's lifetime: its deadline, its cancellation, the pairs it tried and the failures it met. */
@@ -153,6 +155,21 @@ function chatNeed(request, streaming, policy) {
153
155
  dimensions: null,
154
156
  };
155
157
  }
158
+ /** What a request in the Anthropic Messages shape needs of a lane. */
159
+ function messagesNeed(request, streaming, policy) {
160
+ const capabilities = new Set(["chat", ...policy.require]);
161
+ if (streaming)
162
+ capabilities.add("streaming");
163
+ if (request.tools !== undefined && request.tools.length > 0)
164
+ capabilities.add("tools");
165
+ return {
166
+ capabilities: [...capabilities],
167
+ promptTokens: estimateTokens(JSON.stringify([request.system ?? "", request.messages])),
168
+ outputTokens: request.max_tokens ?? 0,
169
+ oneIdentity: policy.pinned,
170
+ dimensions: null,
171
+ };
172
+ }
156
173
  function hasToken(chunk) {
157
174
  return chunk.choices.some((choice) => Object.entries(choice.delta).some(([field, value]) => field !== "role" && value !== null && value !== undefined && value !== "" && !(Array.isArray(value) && value.length === 0)));
158
175
  }
@@ -206,6 +223,12 @@ export function createRouter(config) {
206
223
  const fetchImpl = config.fetch ?? ((input, init) => fetch(input, init));
207
224
  const random = config.random ?? Math.random;
208
225
  const catalog = buildCatalog(config);
226
+ const anthropicProviders = config.providers.filter((provider) => presetOf(provider).protocol === "anthropic").map((provider) => provider.id);
227
+ /** The call's options with its providers narrowed to those that speak the Anthropic protocol. */
228
+ function nativeOptions(options) {
229
+ const asked = options.policy?.providers;
230
+ return { ...options, policy: { ...options.policy, providers: asked === undefined ? anthropicProviders : anthropicProviders.filter((id) => asked.includes(id)) } };
231
+ }
209
232
  // The policy is resolved at once, so a bad profile or model throws from the call itself; the clock starts at start().
210
233
  function prepare(operation, model, options, need) {
211
234
  const policy = resolvePolicy(config, options, model);
@@ -300,13 +323,20 @@ export function createRouter(config) {
300
323
  "idempotency-key": call.requestId,
301
324
  };
302
325
  }
303
- async function open(call, pick, scope, operation, body, options) {
326
+ /** Sends one attempt; `context` is what the lane's adapter needs again to read the answer. */
327
+ async function open(call, pick, scope, operation, body, options, native = false) {
304
328
  const secret = await resolveSecret(pick.lane.key);
305
329
  if (secret === undefined)
306
330
  throw new AttemptError("auth", `key ${pick.lane.key.alias} resolved to no value`, { missingKey: true });
307
331
  scope.secret = secret;
308
332
  const wire = wireOf(pick.lane.provider);
309
- const request = operation === "embeddings" ? wire.embeddings(body, pick.lane.model.id) : wire.chat(body, pick.lane.model.id, operation === "stream", pick.lane.provider);
333
+ // A chat request is shaped for the model that serves this attempt: the next lane may need another shape.
334
+ const streaming = operation === "stream";
335
+ const request = operation === "embeddings"
336
+ ? wire.embeddings(body, pick.lane.model.id)
337
+ : native
338
+ ? nativeRequest(shapeRequest(body, pick.lane.model, true), pick.lane.model, streaming)
339
+ : wire.chat(shapeRequest(body, pick.lane.model), pick.lane.model, streaming, pick.lane.provider);
310
340
  let response;
311
341
  try {
312
342
  response = await fetchImpl(`${baseUrlOf(pick.lane.provider)}${request.path}`, {
@@ -321,7 +351,7 @@ export function createRouter(config) {
321
351
  }
322
352
  if (!response.ok)
323
353
  throw await errorOf(pick.lane, response);
324
- return response;
354
+ return { response, context: request.context };
325
355
  }
326
356
  function begin(call, pick) {
327
357
  call.attempts++;
@@ -403,7 +433,7 @@ export function createRouter(config) {
403
433
  return error;
404
434
  return call.error("network", error instanceof Error ? error.message : "the call failed");
405
435
  }
406
- async function unary(call, operation, body, options, parse) {
436
+ async function unary(call, operation, body, options, parse, native = false) {
407
437
  try {
408
438
  for (;;) {
409
439
  const pick = await next(call);
@@ -411,7 +441,7 @@ export function createRouter(config) {
411
441
  const scope = new AttemptScope(call, call.policy.timeoutMs);
412
442
  const started = Date.now();
413
443
  try {
414
- const response = await open(call, pick, scope, operation, body, options);
444
+ const { response, context } = await open(call, pick, scope, operation, body, options, native);
415
445
  let json;
416
446
  try {
417
447
  json = await response.json();
@@ -424,7 +454,7 @@ export function createRouter(config) {
424
454
  // An error inside a 200 body (OpenRouter does this) is classified like an error status.
425
455
  if (!isPlainRecord(json) || (json.error !== undefined && json.error !== null))
426
456
  throw failureOf(pick.lane, undefined, json, null, "the provider answered an error");
427
- const parsed = parse(json, pick.lane);
457
+ const parsed = parse(json, pick.lane, context);
428
458
  const latencyMs = Date.now() - started;
429
459
  const routing = await succeed(call, pick, readRateLimit(response.headers, Date.now()), latencyMs, parsed.usage, { request: body, response: json });
430
460
  settle(call, routing, null);
@@ -452,7 +482,19 @@ export function createRouter(config) {
452
482
  throw routerError;
453
483
  }
454
484
  }
455
- async function* streamCall(call, body, options) {
485
+ /** A provider stream as OpenAI chunks: the lane's adapter translates, and a chunk with content is the first token. */
486
+ async function* chatItems(lane, events, context) {
487
+ for await (const event of wireOf(lane.provider).chunks(events, context)) {
488
+ if (event.kind !== "chunk") {
489
+ yield event;
490
+ continue;
491
+ }
492
+ const chunk = readChunk(event.chunk);
493
+ if (chunk !== undefined)
494
+ yield { kind: "item", item: chunk, token: hasToken(chunk), usage: readUsage(chunk.usage) };
495
+ }
496
+ }
497
+ async function* streamCall(call, body, options, items, native = false) {
456
498
  let settled = false;
457
499
  try {
458
500
  for (;;) {
@@ -466,29 +508,30 @@ export function createRouter(config) {
466
508
  // "pending" in the finally block means the consumer stopped reading: no catch ran and no success did.
467
509
  let outcome = "pending";
468
510
  try {
469
- const response = await open(call, pick, scope, "stream", body, options);
511
+ const { response, context } = await open(call, pick, scope, "stream", body, options, native);
470
512
  info = readRateLimit(response.headers, Date.now());
471
513
  if (response.body === null)
472
514
  throw new AttemptError("transient", "the provider answered no stream body");
473
515
  const held = [];
474
- for await (const event of wireOf(pick.lane.provider).chunks(parseSse(response.body))) {
516
+ for await (const event of items(pick.lane, parseSse(response.body), context)) {
517
+ // The provider is at work (a ping, hidden thinking) with nothing for the caller yet: the idle timeout starts over.
518
+ if (event.kind === "alive") {
519
+ scope.arm(call.policy.timeoutMs);
520
+ continue;
521
+ }
475
522
  if (event.kind === "malformed")
476
523
  throw new AttemptError("stream", "the provider streamed an event that is not JSON");
477
524
  if (event.kind === "error")
478
525
  throw failureOf(pick.lane, undefined, event.body, null, "the provider streamed an error");
479
- const chunk = readChunk(event.chunk);
480
- if (chunk === undefined)
481
- continue;
482
- const chunkUsage = readUsage(chunk.usage);
483
- if (chunkUsage !== undefined)
484
- usage = chunkUsage;
526
+ if (event.usage !== undefined)
527
+ usage = event.usage;
485
528
  scope.arm(call.policy.timeoutMs);
486
529
  if (firstToken) {
487
- yield chunk;
530
+ yield event.item;
488
531
  continue;
489
532
  }
490
- held.push(chunk);
491
- if (hasToken(chunk)) {
533
+ held.push(event.item);
534
+ if (event.token) {
492
535
  firstToken = true;
493
536
  for (const early of held.splice(0))
494
537
  yield early;
@@ -543,8 +586,8 @@ export function createRouter(config) {
543
586
  const body = { ...request };
544
587
  delete body.stream;
545
588
  const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy));
546
- const { value, routing } = await unary(start(), "chat", body, options, (json, lane) => {
547
- const completion = readCompletion(wireOf(lane.provider).chatResponse(json));
589
+ const { value, routing } = await unary(start(), "chat", body, options, (json, lane, context) => {
590
+ const completion = readCompletion(wireOf(lane.provider).chatResponse(json, context));
548
591
  if (completion === undefined)
549
592
  throw new AttemptError("transient", "the provider answered a completion without choices");
550
593
  return { value: completion, usage: completion.usage };
@@ -557,7 +600,27 @@ export function createRouter(config) {
557
600
  const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy));
558
601
  return {
559
602
  requestId,
560
- [Symbol.asyncIterator]: () => streamCall(start(), body, options),
603
+ [Symbol.asyncIterator]: () => streamCall(start(), body, options, chatItems),
604
+ };
605
+ },
606
+ async messages(request, options = {}) {
607
+ const native = nativeOptions(options);
608
+ const { start } = prepare("chat", request.model, native, (policy) => messagesNeed(request, false, policy));
609
+ const { value, routing } = await unary(start(), "chat", { ...request }, native, (json) => {
610
+ if (json.type !== "message" || !Array.isArray(json.content))
611
+ throw new AttemptError("transient", "the provider answered a body that is not a message");
612
+ return { value: json, usage: nativeUsage(json.usage) };
613
+ }, true);
614
+ const usage = nativeUsage(value.usage);
615
+ return { message: value, ...(usage === undefined ? {} : { usage }), routing };
616
+ },
617
+ messagesStream(request, options = {}) {
618
+ const native = nativeOptions(options);
619
+ const { requestId, start } = prepare("stream", request.model, native, (policy) => messagesNeed(request, true, policy));
620
+ const items = (lane, events) => nativeItems(lane.provider, lane.model, events);
621
+ return {
622
+ requestId,
623
+ [Symbol.asyncIterator]: () => streamCall(start(), { ...request }, native, items, true),
561
624
  };
562
625
  },
563
626
  async embeddings(request, options = {}) {