@starci/ai-router 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,81 @@
1
+ import type { EnvProviderSpec } from "./env.js";
2
+ import type { FetchLike, ModelConfig, ModelCost, PoolConfig } from "./types.js";
3
+ /**
4
+ * The models the router knows by id, with the facts their vendors state. A fact a vendor's page does not state is left
5
+ * out, never guessed. Sources, read 2026-11-16:
6
+ *
7
+ * - Anthropic, https://platform.claude.com/docs/en/models/overview: ids, 1M-token context window, text and image input,
8
+ * tool use, price per million tokens (Fable 5.2 8 and 40, Opus 5.5 4 and 20, Sonnet 5.5 2 and 10) and the cache-read share
9
+ * of the input price (2.5% on Fable 5.2, 5% on Opus 5.5 and Sonnet 5.5). Haiku 5.5 is priced "From $0.10 / $0.50", so it
10
+ * carries no cost, and the page states no cache write price, so none is declared.
11
+ * - OpenAI, https://developers.openai.com/api/docs/models: ids, 1.05M-token context window, image input and function
12
+ * calling for the latest models. Their price is not declared here: the router prices what a deployment sets.
13
+ *
14
+ * `tier` is this package's own ranking by the vendors' descriptions and prices ("most demanding work" is `max`,
15
+ * "high-volume tasks" is `low`); it orders the `auto-*` profiles and states nothing a vendor claims.
16
+ */
17
+ export type EffortTier = "low" | "medium" | "high" | "max";
18
+ export declare const EFFORT_TIERS: readonly EffortTier[];
19
+ export interface KnownModel {
20
+ readonly id: string;
21
+ readonly kind: "anthropic" | "openai";
22
+ readonly label: string;
23
+ readonly tier: EffortTier;
24
+ readonly vision: boolean;
25
+ readonly tools: boolean;
26
+ readonly contextWindow: number;
27
+ readonly cost?: ModelCost;
28
+ }
29
+ export declare const KNOWN_MODELS: readonly KnownModel[];
30
+ /** The known models of one provider kind as the `ModelConfig`s of a provider. */
31
+ export declare function knownModelConfigs(kind: "anthropic" | "openai"): ModelConfig[];
32
+ export interface OpenRouterCatalogOptions {
33
+ /** Default `https://openrouter.ai/api/v1`. */
34
+ readonly baseUrl?: string;
35
+ /** Sent as a bearer token when given; the endpoint's auth is optional (https://openrouter.ai/docs/api-reference/list-available-models). */
36
+ readonly apiKey?: string;
37
+ readonly fetch?: FetchLike;
38
+ /** How long a fetched catalog is reused. Default one hour; 0 never caches. */
39
+ readonly ttlMs?: number;
40
+ readonly now?: () => number;
41
+ }
42
+ /**
43
+ * OpenRouter's catalog as `ModelConfig`s: the id sent to OpenRouter is `<vendor>/<model>` and the name a caller uses is
44
+ * `openrouter/<vendor>/<model>` (an alias). The endpoint answers text-output models by default. Vision comes from
45
+ * `architecture.input_modalities` holding `image`, tools from `supported_parameters` holding `tools`, the window from
46
+ * `context_length`; an entry that omits a field leaves that fact out. Prices are not read: the page does not state their
47
+ * unit. The answer is cached per base URL for `ttlMs`.
48
+ */
49
+ export declare function fetchOpenRouterModels(options?: OpenRouterCatalogOptions): Promise<ModelConfig[]>;
50
+ /** Forgets every cached OpenRouter catalog (tests, or a deployment that wants a refresh now). */
51
+ export declare function clearOpenRouterCatalogCache(): void;
52
+ export interface DefaultProvidersOptions {
53
+ /** Models for the `openrouter` provider, from `fetchOpenRouterModels`; without them it is left out (nothing to route). */
54
+ readonly openRouterModels?: readonly ModelConfig[];
55
+ }
56
+ /**
57
+ * Provider specs for `poolFromEnv`: Anthropic and OpenAI with the known models, and OpenRouter with the models given.
58
+ * Each is present only when its key is in env (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY`): without a
59
+ * key the provider and its models are absent from the catalog, and nothing fails at boot.
60
+ */
61
+ export declare function defaultProviderSpecs(options?: DefaultProvidersOptions): EnvProviderSpec[];
62
+ /** One model a caller may choose, from the providers whose keys are configured. */
63
+ export interface AvailableModel {
64
+ /** The name to ask for: the model id, or `openrouter/<vendor>/<model>` on OpenRouter. */
65
+ readonly id: string;
66
+ /** The id of the provider serving it. */
67
+ readonly provider: string;
68
+ readonly label: string;
69
+ readonly vision: boolean;
70
+ readonly tools: boolean;
71
+ readonly contextTokens?: number;
72
+ }
73
+ /** The chat models of a pool, one per provider serving them; a pool built from env holds only providers with keys. */
74
+ export declare function availableModels(config: PoolConfig): AvailableModel[];
75
+ /**
76
+ * The pool with `auto-low`, `auto-medium`, `auto-high`, `auto-max` and `auto` (the medium one) profiles added: each tries
77
+ * the known models the pool serves, starting at its own effort tier, then the tiers below it from the nearest down, then
78
+ * those above from the nearest up. A profile with no known model to try is not added, and a profile the app declares is
79
+ * kept as it is. The pool holds only providers with keys, so `auto` never leaves them.
80
+ */
81
+ export declare function withAutoProfiles<P extends string>(config: PoolConfig<P>): PoolConfig;
@@ -0,0 +1,142 @@
1
+ import { matchesOf } from "./config.js";
2
+ import { AiRouterError } from "./errors.js";
3
+ export const EFFORT_TIERS = ["low", "medium", "high", "max"];
4
+ const CLAUDE_CONTEXT = 1_000_000;
5
+ const OPENAI_CONTEXT = 1_050_000;
6
+ export const KNOWN_MODELS = Object.freeze([
7
+ { id: "claude-fable-5-2", kind: "anthropic", label: "Claude Fable 5.2", tier: "max", vision: true, tools: true, contextWindow: CLAUDE_CONTEXT, cost: { input: 8, output: 40, cacheRead: 0.2 } },
8
+ { id: "claude-opus-5-5", kind: "anthropic", label: "Claude Opus 5.5", tier: "high", vision: true, tools: true, contextWindow: CLAUDE_CONTEXT, cost: { input: 4, output: 20, cacheRead: 0.2 } },
9
+ { id: "claude-sonnet-5-5", kind: "anthropic", label: "Claude Sonnet 5.5", tier: "medium", vision: true, tools: true, contextWindow: CLAUDE_CONTEXT, cost: { input: 2, output: 10, cacheRead: 0.1 } },
10
+ { id: "claude-haiku-5-5", kind: "anthropic", label: "Claude Haiku 5.5", tier: "low", vision: true, tools: true, contextWindow: CLAUDE_CONTEXT },
11
+ { id: "gpt-6-astra", kind: "openai", label: "GPT-6 Astra", tier: "max", vision: true, tools: true, contextWindow: OPENAI_CONTEXT },
12
+ { id: "gpt-6.1-sol", kind: "openai", label: "GPT-6.1 Sol", tier: "high", vision: true, tools: true, contextWindow: OPENAI_CONTEXT },
13
+ { id: "gpt-6.1-terra", kind: "openai", label: "GPT-6.1 Terra", tier: "medium", vision: true, tools: true, contextWindow: OPENAI_CONTEXT },
14
+ { id: "gpt-6-luna", kind: "openai", label: "GPT-6 Luna", tier: "low", vision: true, tools: true, contextWindow: OPENAI_CONTEXT },
15
+ ]);
16
+ function capabilitiesOf(model) {
17
+ return ["chat", "streaming", "json", ...(model.tools ? ["tools"] : []), ...(model.vision ? ["vision"] : [])];
18
+ }
19
+ /** The known models of one provider kind as the `ModelConfig`s of a provider. */
20
+ export function knownModelConfigs(kind) {
21
+ return KNOWN_MODELS.filter((model) => model.kind === kind).map((model) => ({
22
+ id: model.id,
23
+ contextWindow: model.contextWindow,
24
+ capabilities: capabilitiesOf(model),
25
+ ...(model.cost === undefined ? {} : { cost: model.cost }),
26
+ // Claude Opus 5.5 and Sonnet 5.5 refuse a forced tool choice (docs/design.md, "A model's own request shape").
27
+ ...(model.id === "claude-opus-5-5" || model.id === "claude-sonnet-5-5" ? { forcedToolUse: false } : {}),
28
+ }));
29
+ }
30
+ function strings(value) {
31
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
32
+ }
33
+ const cache = new Map();
34
+ /**
35
+ * OpenRouter's catalog as `ModelConfig`s: the id sent to OpenRouter is `<vendor>/<model>` and the name a caller uses is
36
+ * `openrouter/<vendor>/<model>` (an alias). The endpoint answers text-output models by default. Vision comes from
37
+ * `architecture.input_modalities` holding `image`, tools from `supported_parameters` holding `tools`, the window from
38
+ * `context_length`; an entry that omits a field leaves that fact out. Prices are not read: the page does not state their
39
+ * unit. The answer is cached per base URL for `ttlMs`.
40
+ */
41
+ export async function fetchOpenRouterModels(options = {}) {
42
+ const baseUrl = (options.baseUrl ?? "https://openrouter.ai/api/v1").replace(/\/+$/, "");
43
+ const now = options.now ?? Date.now;
44
+ const ttl = options.ttlMs ?? 3_600_000;
45
+ const hit = cache.get(baseUrl);
46
+ if (hit !== undefined && ttl > 0 && now() - hit.at < ttl)
47
+ return hit.models;
48
+ const doFetch = options.fetch ?? ((input, init) => fetch(input, init));
49
+ const response = await doFetch(`${baseUrl}/models`, { method: "GET", headers: options.apiKey === undefined ? {} : { authorization: `Bearer ${options.apiKey}` } });
50
+ if (!response.ok)
51
+ throw new AiRouterError({ errorClass: "config", message: `OpenRouter catalog answered ${String(response.status)}` });
52
+ const body = (await response.json());
53
+ const models = [];
54
+ for (const entry of Array.isArray(body.data) ? body.data : []) {
55
+ if (typeof entry.id !== "string" || entry.id === "")
56
+ continue;
57
+ const outputs = strings(entry.architecture?.output_modalities);
58
+ if (outputs.length > 0 && !outputs.includes("text"))
59
+ continue;
60
+ models.push({
61
+ id: entry.id,
62
+ aliases: [`openrouter/${entry.id}`],
63
+ capabilities: capabilitiesOf({ tools: strings(entry.supported_parameters).includes("tools"), vision: strings(entry.architecture?.input_modalities).includes("image") }),
64
+ ...(typeof entry.context_length === "number" && entry.context_length > 0 ? { contextWindow: entry.context_length } : {}),
65
+ });
66
+ }
67
+ cache.set(baseUrl, { at: now(), models });
68
+ return models;
69
+ }
70
+ /** Forgets every cached OpenRouter catalog (tests, or a deployment that wants a refresh now). */
71
+ export function clearOpenRouterCatalogCache() {
72
+ cache.clear();
73
+ }
74
+ /**
75
+ * Provider specs for `poolFromEnv`: Anthropic and OpenAI with the known models, and OpenRouter with the models given.
76
+ * Each is present only when its key is in env (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY`): without a
77
+ * key the provider and its models are absent from the catalog, and nothing fails at boot.
78
+ */
79
+ export function defaultProviderSpecs(options = {}) {
80
+ const specs = [
81
+ { kind: "anthropic", models: knownModelConfigs("anthropic") },
82
+ { kind: "openai", models: knownModelConfigs("openai") },
83
+ ];
84
+ if (options.openRouterModels !== undefined)
85
+ specs.push({ kind: "openrouter", models: options.openRouterModels });
86
+ return specs;
87
+ }
88
+ /** The chat models of a pool, one per provider serving them; a pool built from env holds only providers with keys. */
89
+ export function availableModels(config) {
90
+ const out = [];
91
+ for (const provider of config.providers) {
92
+ const kind = provider.kind ?? "openai-compatible";
93
+ for (const model of provider.models) {
94
+ const capabilities = model.capabilities ?? ["chat", "streaming", "tools", "json"];
95
+ if (!capabilities.includes("chat"))
96
+ continue;
97
+ out.push({
98
+ id: kind === "openrouter" ? `openrouter/${model.id.replace(/^openrouter\//, "")}` : model.id,
99
+ provider: provider.id,
100
+ label: KNOWN_MODELS.find((known) => known.id === model.id)?.label ?? model.id,
101
+ vision: capabilities.includes("vision"),
102
+ tools: capabilities.includes("tools"),
103
+ ...(model.contextWindow === undefined ? {} : { contextTokens: model.contextWindow }),
104
+ });
105
+ }
106
+ }
107
+ return out;
108
+ }
109
+ /**
110
+ * The pool with `auto-low`, `auto-medium`, `auto-high`, `auto-max` and `auto` (the medium one) profiles added: each tries
111
+ * the known models the pool serves, starting at its own effort tier, then the tiers below it from the nearest down, then
112
+ * those above from the nearest up. A profile with no known model to try is not added, and a profile the app declares is
113
+ * kept as it is. The pool holds only providers with keys, so `auto` never leaves them.
114
+ */
115
+ export function withAutoProfiles(config) {
116
+ const served = KNOWN_MODELS.filter((model) => matchesOf(config, { model: model.id }).length > 0);
117
+ const profiles = { ...(config.profiles ?? {}) };
118
+ const add = (name, tier) => {
119
+ if (name in profiles)
120
+ return;
121
+ const at = EFFORT_TIERS.indexOf(tier);
122
+ const distance = (model) => {
123
+ const gap = EFFORT_TIERS.indexOf(model.tier) - at;
124
+ return gap <= 0 ? -gap : 10 + gap;
125
+ };
126
+ const tiers = [];
127
+ let current;
128
+ for (const model of [...served].sort((a, b) => distance(a) - distance(b))) {
129
+ if (model.tier === current)
130
+ tiers[tiers.length - 1]?.push({ model: model.id });
131
+ else
132
+ tiers.push([{ model: model.id }]);
133
+ current = model.tier;
134
+ }
135
+ if (tiers.length > 0)
136
+ profiles[name] = { candidates: tiers };
137
+ };
138
+ for (const tier of EFFORT_TIERS)
139
+ add(`auto-${tier}`, tier);
140
+ add("auto", "medium");
141
+ return { ...config, profiles };
142
+ }
@@ -0,0 +1,137 @@
1
+ import { DomainError } from "@starci/common";
2
+ import type { ChatAnthropic } from "@langchain/anthropic";
3
+ import type { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai";
4
+ /**
5
+ * LangChain clients over the gateway (docs/design.md, "The LangChain entry"). An app holds one virtual key and names a
6
+ * pool model or a profile; it gets back the LangChain chat model that speaks the protocol the gateway serves that name
7
+ * through. The app never builds a provider client and never branches on a provider.
8
+ *
9
+ * The LangChain packages are optional peers, imported the first time a kind is asked for: an app that never asks for an
10
+ * `anthropic` model never needs `@langchain/anthropic`, and the router's core imports none of them.
11
+ */
12
+ /**
13
+ * The chat model of each kind. A kind is a wire protocol, never a model family: every OpenAI-compatible provider
14
+ * (DeepSeek, OpenRouter, Qwen, Groq, ...) is `openai`. A new kind is added by declaring its model here (module
15
+ * augmentation) and registering its factory with `registerChatKind`; nothing else changes.
16
+ */
17
+ export interface RoutedChatKinds {
18
+ openai: ChatOpenAI;
19
+ anthropic: ChatAnthropic;
20
+ }
21
+ /** A wire protocol the gateway serves a name through. */
22
+ export type RoutedKind = keyof RoutedChatKinds;
23
+ /** One chat model the gateway routes, with the protocol it speaks. The common path uses `model` and never reads `kind`. */
24
+ export type RoutedChat<K extends RoutedKind = RoutedKind> = {
25
+ [Kind in K]: {
26
+ readonly kind: Kind;
27
+ /** The pool model or profile this client names on every call. */
28
+ readonly name: string;
29
+ readonly model: RoutedChatKinds[Kind];
30
+ /**
31
+ * The same name through its other protocols, in the order the pool tries them: a profile whose candidates span two
32
+ * protocols is one client here and the next ones there. The router fails over inside a protocol on its own; moving
33
+ * to the next protocol is the caller's retry, which it already owns.
34
+ */
35
+ readonly fallbacks: readonly RoutedChat[];
36
+ };
37
+ }[K];
38
+ /** The embeddings client of a pinned profile or an embeddings model, with the length of its vectors. */
39
+ export interface RoutedEmbeddings {
40
+ readonly kind: "openai";
41
+ readonly name: string;
42
+ readonly embeddings: OpenAIEmbeddings;
43
+ /** The vector length the pool declares for it; undefined when the pool declares none. */
44
+ readonly dimensions: number | undefined;
45
+ }
46
+ /** A `fetch`-compatible function. */
47
+ export type RoutedFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
48
+ /** How the routed clients reach the gateway. */
49
+ export interface RoutedModelsOptions {
50
+ /** The gateway's origin, without `/v1` (`http://127.0.0.1:4100`). */
51
+ readonly gatewayUrl: string;
52
+ /** The app's one virtual key (`STARCI_AI_KEY`). */
53
+ readonly virtualKey: string;
54
+ /** The `fetch` every client and the catalog read use. Default: the global one. */
55
+ readonly fetch?: RoutedFetch;
56
+ /** The limit of one call in milliseconds, for every client unless a call to `chat` sets its own. */
57
+ readonly timeoutMs?: number;
58
+ /** Extra headers on every request to the gateway. */
59
+ readonly headers?: Readonly<Record<string, string>>;
60
+ }
61
+ /** What one `chat` call may set on the client it gets. */
62
+ export interface RoutedChatOptions {
63
+ /** The limit of one call in milliseconds. */
64
+ readonly timeoutMs?: number;
65
+ /** The most tokens an answer may hold. An `anthropic` client must send one: default 16000. */
66
+ readonly maxTokens?: number;
67
+ /** Extra constructor fields of the kind's own client, for a provider's own features; they win over the defaults. */
68
+ readonly fields?: Readonly<Partial<Record<RoutedKind, Readonly<Record<string, unknown>>>>>;
69
+ }
70
+ /** What a kind's factory is given to build its client. */
71
+ export interface RoutedChatContext {
72
+ readonly name: string;
73
+ /** The gateway's origin, without a trailing slash. */
74
+ readonly gatewayUrl: string;
75
+ readonly virtualKey: string;
76
+ readonly fetch: RoutedFetch | undefined;
77
+ readonly timeoutMs: number | undefined;
78
+ readonly maxTokens: number | undefined;
79
+ /** The headers every request carries: the caller's, and the one that keeps the call on this kind's providers. */
80
+ readonly headers: Readonly<Record<string, string>>;
81
+ /** The caller's own constructor fields for this kind. */
82
+ readonly fields: Readonly<Record<string, unknown>>;
83
+ }
84
+ /** Builds the chat model of one kind; it imports its LangChain package itself, so the package loads only when asked for. */
85
+ export type RoutedChatFactory<K extends RoutedKind = RoutedKind> = (context: RoutedChatContext) => Promise<RoutedChatKinds[K]>;
86
+ /** Why the routed models refused. */
87
+ export type RoutedModelsErrorCode = "ROUTED_CATALOG_UNAVAILABLE" | "ROUTED_NAME_UNKNOWN" | "ROUTED_KIND_UNSUPPORTED" | "ROUTED_NOT_CHAT" | "ROUTED_NOT_EMBEDDINGS" | "ROUTED_PACKAGE_MISSING";
88
+ /** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
89
+ export declare class RoutedModelsError extends DomainError<RoutedModelsErrorCode> {
90
+ constructor(code: RoutedModelsErrorCode, message: string, cause?: unknown);
91
+ }
92
+ /** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
93
+ export declare const ROUTED_ANTHROPIC_MAX_TOKENS = 16000;
94
+ /** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
95
+ export declare function registerChatKind<K extends RoutedKind>(kind: K, factory: RoutedChatFactory<K>): void;
96
+ /** The routed clients of one app. */
97
+ export interface RoutedModels {
98
+ /**
99
+ * The chat model of a pool model or profile, speaking the protocol of its first candidate. `createAgent({ model:
100
+ * (await ai.chat(name)).model })` works for every kind; `kind` is read only for a provider's own features.
101
+ */
102
+ chat(name: string, options?: RoutedChatOptions): Promise<RoutedChat>;
103
+ /** The embeddings client of a pinned profile or an embeddings model. It never falls back to another model identity. */
104
+ embeddings(name: string): Promise<RoutedEmbeddings>;
105
+ /** Reads the gateway's catalog again: after the pool's config changed. */
106
+ refresh(): Promise<void>;
107
+ }
108
+ /** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
109
+ export declare function createRoutedModels(options: RoutedModelsOptions): RoutedModels;
110
+ /** What one answer cost in tokens and who served it, the same for every kind: what credit billing reads. */
111
+ export interface RoutedUsage {
112
+ /** Prompt tokens billed at the full input price: the prompt without its cache reads and writes. */
113
+ readonly inputTokens: number;
114
+ /** Prompt tokens read from the provider's prompt cache. */
115
+ readonly cacheReadTokens: number;
116
+ /** Prompt tokens written to the provider's prompt cache. */
117
+ readonly cacheWriteTokens: number;
118
+ /** Completion tokens, thinking included. */
119
+ readonly outputTokens: number;
120
+ /** The part of `outputTokens` the model spent thinking; 0 when the provider does not count it. */
121
+ readonly reasoningTokens: number;
122
+ /** The pool model that answered, as the pool's config names it; undefined when the answer carried no usage. */
123
+ readonly model: string | undefined;
124
+ /** The pool provider that answered. */
125
+ readonly provider: string | undefined;
126
+ /** The cost the router computed from the model's prices; undefined when the model declares none or the answer was streamed natively. */
127
+ readonly cost: number | undefined;
128
+ }
129
+ /**
130
+ * The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
131
+ * in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
132
+ * gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
133
+ */
134
+ export declare function usageOf(reply: {
135
+ readonly usage_metadata?: unknown;
136
+ readonly response_metadata?: unknown;
137
+ }): RoutedUsage;
@@ -0,0 +1,182 @@
1
+ import { DomainError, isPlainRecord } from "@starci/common";
2
+ /** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
3
+ export class RoutedModelsError extends DomainError {
4
+ constructor(code, message, cause) {
5
+ super({ code, ...(cause === undefined ? {} : { cause }) });
6
+ this.message = message;
7
+ this.name = "RoutedModelsError";
8
+ }
9
+ }
10
+ /** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
11
+ export const ROUTED_ANTHROPIC_MAX_TOKENS = 16_000;
12
+ /** The header that keeps a call on the providers of one protocol. */
13
+ const KIND_HEADER = "x-ai-router-kind";
14
+ async function load(name, importer) {
15
+ try {
16
+ return await importer();
17
+ }
18
+ catch (error) {
19
+ throw new RoutedModelsError("ROUTED_PACKAGE_MISSING", `install ${name} to use this kind: it is an optional peer of @starci/ai-router/langchain`, error);
20
+ }
21
+ }
22
+ const factories = new Map();
23
+ /** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
24
+ export function registerChatKind(kind, factory) {
25
+ factories.set(kind, factory);
26
+ }
27
+ registerChatKind("openai", async (context) => {
28
+ const { ChatOpenAI: Model } = await load("@langchain/openai", () => import("@langchain/openai"));
29
+ return new Model({
30
+ model: context.name,
31
+ apiKey: context.virtualKey,
32
+ // The app owns its retries: the router already fails over between keys and providers of the kind.
33
+ maxRetries: 0,
34
+ streamUsage: true,
35
+ ...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }),
36
+ ...(context.maxTokens === undefined ? {} : { maxTokens: context.maxTokens }),
37
+ ...context.fields,
38
+ configuration: { baseURL: `${context.gatewayUrl}/v1`, defaultHeaders: { ...context.headers }, ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
39
+ });
40
+ });
41
+ registerChatKind("anthropic", async (context) => {
42
+ const { ChatAnthropic: Model } = await load("@langchain/anthropic", () => import("@langchain/anthropic"));
43
+ return new Model({
44
+ model: context.name,
45
+ apiKey: context.virtualKey,
46
+ // The Anthropic client appends /v1/messages to its base URL itself.
47
+ anthropicApiUrl: context.gatewayUrl,
48
+ maxRetries: 0,
49
+ streamUsage: true,
50
+ maxTokens: context.maxTokens ?? ROUTED_ANTHROPIC_MAX_TOKENS,
51
+ ...context.fields,
52
+ clientOptions: { defaultHeaders: { ...context.headers }, ...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }), ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
53
+ });
54
+ });
55
+ function strings(value) {
56
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
57
+ }
58
+ function listedOf(entry) {
59
+ if (!isPlainRecord(entry) || typeof entry.id !== "string")
60
+ return undefined;
61
+ return { id: entry.id, kinds: strings(entry.kinds), serves: strings(entry.serves), dimensions: typeof entry.dimensions === "number" ? entry.dimensions : undefined };
62
+ }
63
+ /** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
64
+ export function createRoutedModels(options) {
65
+ const gatewayUrl = options.gatewayUrl.replace(/\/+$/, "").replace(/\/v1$/, "");
66
+ const fetcher = options.fetch ?? ((input, init) => fetch(input, init));
67
+ let catalog;
68
+ async function read() {
69
+ let body;
70
+ try {
71
+ const response = await fetcher(`${gatewayUrl}/v1/models`, { headers: { authorization: `Bearer ${options.virtualKey}`, ...options.headers } });
72
+ if (!response.ok)
73
+ throw new Error(`HTTP ${String(response.status)}`);
74
+ body = await response.json();
75
+ }
76
+ catch (error) {
77
+ throw new RoutedModelsError("ROUTED_CATALOG_UNAVAILABLE", `the gateway at ${gatewayUrl} did not list its models: ${error instanceof Error ? error.message : "the request failed"}`, error);
78
+ }
79
+ const data = isPlainRecord(body) && Array.isArray(body.data) ? body.data : [];
80
+ return new Map(data.map(listedOf).filter((entry) => entry !== undefined).map((entry) => [entry.id, entry]));
81
+ }
82
+ function listed() {
83
+ catalog ??= read().catch((error) => {
84
+ // A failed read is not remembered: the next call asks again.
85
+ catalog = undefined;
86
+ throw error;
87
+ });
88
+ return catalog;
89
+ }
90
+ async function find(name) {
91
+ let entry = (await listed()).get(name);
92
+ if (entry === undefined) {
93
+ // The pool may have gained the name since the catalog was read.
94
+ catalog = undefined;
95
+ entry = (await listed()).get(name);
96
+ }
97
+ if (entry === undefined)
98
+ throw new RoutedModelsError("ROUTED_NAME_UNKNOWN", `the gateway serves no model or profile "${name}" to this key`);
99
+ return entry;
100
+ }
101
+ async function build(kind, name, call) {
102
+ const factory = factories.get(kind);
103
+ if (factory === undefined)
104
+ throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `"${name}" is served through the ${kind} protocol, which has no registered chat kind`);
105
+ const fields = call.fields?.[kind] ?? {};
106
+ const model = await factory({
107
+ name,
108
+ gatewayUrl,
109
+ virtualKey: options.virtualKey,
110
+ fetch: options.fetch,
111
+ timeoutMs: call.timeoutMs ?? options.timeoutMs,
112
+ maxTokens: call.maxTokens,
113
+ headers: { ...options.headers, [KIND_HEADER]: kind },
114
+ fields,
115
+ });
116
+ // The registry holds each kind's factory under its own key, so the model is the kind's.
117
+ return { kind, name, model, fallbacks: [] };
118
+ }
119
+ return {
120
+ async chat(name, call = {}) {
121
+ const entry = await find(name);
122
+ if (!entry.serves.includes("chat"))
123
+ throw new RoutedModelsError("ROUTED_NOT_CHAT", `"${name}" does not serve chat`);
124
+ const [first, ...rest] = entry.kinds;
125
+ if (first === undefined)
126
+ throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `the gateway names no protocol for "${name}"`);
127
+ const primary = await build(first, name, call);
128
+ const fallbacks = await Promise.all(rest.filter((kind) => factories.has(kind)).map((kind) => build(kind, name, call)));
129
+ return { ...primary, fallbacks };
130
+ },
131
+ async embeddings(name) {
132
+ const entry = await find(name);
133
+ if (!entry.serves.includes("embeddings"))
134
+ throw new RoutedModelsError("ROUTED_NOT_EMBEDDINGS", `"${name}" does not serve embeddings`);
135
+ const { OpenAIEmbeddings: Embeddings } = await load("@langchain/openai", () => import("@langchain/openai"));
136
+ const embeddings = new Embeddings({
137
+ model: name,
138
+ apiKey: options.virtualKey,
139
+ maxRetries: 0,
140
+ ...(options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }),
141
+ configuration: { baseURL: `${gatewayUrl}/v1`, defaultHeaders: { ...options.headers }, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) },
142
+ });
143
+ return { kind: "openai", name, embeddings, dimensions: entry.dimensions };
144
+ },
145
+ async refresh() {
146
+ catalog = undefined;
147
+ await listed();
148
+ },
149
+ };
150
+ }
151
+ function count(value) {
152
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
153
+ }
154
+ function record(value) {
155
+ return isPlainRecord(value) ? value : {};
156
+ }
157
+ /**
158
+ * The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
159
+ * in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
160
+ * gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
161
+ */
162
+ export function usageOf(reply) {
163
+ const usage = record(reply.usage_metadata);
164
+ const metadata = record(reply.response_metadata);
165
+ const raw = record(metadata.usage);
166
+ const routing = record(raw.served);
167
+ const inputDetails = record(usage.input_token_details);
168
+ const cacheReadTokens = count(inputDetails.cache_read);
169
+ const cacheWriteTokens = count(inputDetails.cache_creation);
170
+ // OpenAI's raw usage names thinking tokens `reasoning_tokens`, Anthropic's `thinking_tokens`.
171
+ const reasoningTokens = count(record(usage.output_token_details).reasoning) || count(record(raw.completion_tokens_details).reasoning_tokens) || count(record(raw.output_tokens_details).thinking_tokens);
172
+ return {
173
+ inputTokens: Math.max(0, count(usage.input_tokens) - cacheReadTokens - cacheWriteTokens),
174
+ cacheReadTokens,
175
+ cacheWriteTokens,
176
+ outputTokens: count(usage.output_tokens),
177
+ reasoningTokens,
178
+ model: typeof routing.model === "string" ? routing.model : undefined,
179
+ provider: typeof routing.provider === "string" ? routing.provider : undefined,
180
+ cost: typeof routing.cost === "number" ? routing.cost : undefined,
181
+ };
182
+ }
@@ -0,0 +1,7 @@
1
+ import type { ModelConfig } from "./types.js";
2
+ /**
3
+ * The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
4
+ * unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
5
+ * fields, and its forced choice is unforced in that shape.
6
+ */
7
+ export declare function shapeRequest(body: Record<string, unknown>, model: ModelConfig, native?: boolean): Record<string, unknown>;
@@ -0,0 +1,54 @@
1
+ import { isPlainRecord } from "@starci/common";
2
+ /**
3
+ * What one model needs of a chat request and another would refuse, applied from the model's own config just before the
4
+ * request is sent to its provider: default fields (`params`), fields never sent (`omit`) and a forced tool choice the
5
+ * model refuses (`forcedToolUse: false`). The app above sends one request shape whatever lane serves it.
6
+ */
7
+ const ANY_TOOL = "You must call one of the provided tools in this response.";
8
+ function namedTool(name) {
9
+ return `You must call the tool "${name}" in this response.`;
10
+ }
11
+ function isSystem(message) {
12
+ return isPlainRecord(message) && (message.role === "system" || message.role === "developer");
13
+ }
14
+ /** An OpenAI request's forced tool choice as `auto` plus a system instruction, placed after the leading system messages. */
15
+ function unforced(body) {
16
+ const choice = body.tool_choice;
17
+ const text = choice === "required" ? ANY_TOOL : isPlainRecord(choice) && isPlainRecord(choice.function) && typeof choice.function.name === "string" ? namedTool(choice.function.name) : undefined;
18
+ if (text === undefined)
19
+ return body;
20
+ const messages = Array.isArray(body.messages) ? body.messages : [];
21
+ const lead = messages.findIndex((message) => !isSystem(message));
22
+ const at = lead === -1 ? messages.length : lead;
23
+ return { ...body, tool_choice: "auto", messages: [...messages.slice(0, at), { role: "system", content: text }, ...messages.slice(at)] };
24
+ }
25
+ /** An Anthropic request's forced tool choice (`any`, `tool`) as `auto` plus an instruction at the end of its system prompt. */
26
+ function unforcedNative(body) {
27
+ const choice = body.tool_choice;
28
+ if (!isPlainRecord(choice))
29
+ return body;
30
+ const text = choice.type === "any" ? ANY_TOOL : choice.type === "tool" && typeof choice.name === "string" ? namedTool(choice.name) : undefined;
31
+ if (text === undefined)
32
+ return body;
33
+ const system = body.system;
34
+ const block = { type: "text", text };
35
+ return {
36
+ ...body,
37
+ tool_choice: { type: "auto", ...(choice.disable_parallel_tool_use === true ? { disable_parallel_tool_use: true } : {}) },
38
+ system: Array.isArray(system) ? [...system, block] : typeof system === "string" && system !== "" ? `${system}\n\n${text}` : text,
39
+ };
40
+ }
41
+ /**
42
+ * The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
43
+ * unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
44
+ * fields, and its forced choice is unforced in that shape.
45
+ */
46
+ export function shapeRequest(body, model, native = false) {
47
+ if (model.params === undefined && model.omit === undefined && model.forcedToolUse !== false)
48
+ return body;
49
+ const omitted = new Set(model.omit ?? []);
50
+ const out = Object.fromEntries(Object.entries({ ...model.params, ...body }).filter(([field]) => !omitted.has(field)));
51
+ if (model.forcedToolUse !== false)
52
+ return out;
53
+ return native ? unforcedNative(out) : unforced(out);
54
+ }
package/dist/pool.d.ts CHANGED
@@ -56,7 +56,11 @@ export interface Blocked {
56
56
  /** Set when every lane is out because its key is blacklisted or its provider's budget is spent. */
57
57
  readonly blacklisted: Blacklisted | null;
58
58
  }
59
- /** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
59
+ /**
60
+ * The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
61
+ * is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
62
+ * one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
63
+ */
60
64
  export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
61
65
  /**
62
66
  * The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency