@starci/ai-router 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +161 -21
- package/dist/ai-router.service.d.ts +5 -1
- package/dist/ai-router.service.js +8 -0
- package/dist/cache.d.ts +35 -0
- package/dist/cache.js +76 -0
- package/dist/config.js +31 -2
- package/dist/fake.d.ts +27 -0
- package/dist/fake.js +84 -15
- package/dist/gateway/config-file.js +9 -1
- package/dist/gateway/gateway.js +189 -17
- package/dist/langchain/index.d.ts +137 -0
- package/dist/langchain/index.js +182 -0
- package/dist/model-request.d.ts +7 -0
- package/dist/model-request.js +54 -0
- package/dist/pool.d.ts +5 -1
- package/dist/pool.js +17 -4
- package/dist/router.js +84 -21
- package/dist/testing.d.ts +1 -1
- package/dist/types.d.ts +130 -3
- package/dist/wire/anthropic.d.ts +18 -8
- package/dist/wire/anthropic.js +238 -58
- package/dist/wire/native.d.ts +29 -0
- package/dist/wire/native.js +56 -0
- package/dist/wire/thinking-memory.d.ts +32 -0
- package/dist/wire/thinking-memory.js +61 -0
- package/dist/wire/wire.d.ts +12 -7
- package/dist/wire/wire.js +10 -1
- package/docs/design.md +189 -19
- package/package.json +26 -6
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { DomainError } from "@starci/common";
|
|
2
|
+
import type { ChatAnthropic } from "@langchain/anthropic";
|
|
3
|
+
import type { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai";
|
|
4
|
+
/**
|
|
5
|
+
* LangChain clients over the gateway (docs/design.md, "The LangChain entry"). An app holds one virtual key and names a
|
|
6
|
+
* pool model or a profile; it gets back the LangChain chat model that speaks the protocol the gateway serves that name
|
|
7
|
+
* through. The app never builds a provider client and never branches on a provider.
|
|
8
|
+
*
|
|
9
|
+
* The LangChain packages are optional peers, imported the first time a kind is asked for: an app that never asks for an
|
|
10
|
+
* `anthropic` model never needs `@langchain/anthropic`, and the router's core imports none of them.
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* The chat model of each kind. A kind is a wire protocol, never a model family: every OpenAI-compatible provider
|
|
14
|
+
* (DeepSeek, OpenRouter, Qwen, Groq, ...) is `openai`. A new kind is added by declaring its model here (module
|
|
15
|
+
* augmentation) and registering its factory with `registerChatKind`; nothing else changes.
|
|
16
|
+
*/
|
|
17
|
+
export interface RoutedChatKinds {
|
|
18
|
+
openai: ChatOpenAI;
|
|
19
|
+
anthropic: ChatAnthropic;
|
|
20
|
+
}
|
|
21
|
+
/** A wire protocol the gateway serves a name through. */
|
|
22
|
+
export type RoutedKind = keyof RoutedChatKinds;
|
|
23
|
+
/** One chat model the gateway routes, with the protocol it speaks. The common path uses `model` and never reads `kind`. */
|
|
24
|
+
export type RoutedChat<K extends RoutedKind = RoutedKind> = {
|
|
25
|
+
[Kind in K]: {
|
|
26
|
+
readonly kind: Kind;
|
|
27
|
+
/** The pool model or profile this client names on every call. */
|
|
28
|
+
readonly name: string;
|
|
29
|
+
readonly model: RoutedChatKinds[Kind];
|
|
30
|
+
/**
|
|
31
|
+
* The same name through its other protocols, in the order the pool tries them: a profile whose candidates span two
|
|
32
|
+
* protocols is one client here and the next ones there. The router fails over inside a protocol on its own; moving
|
|
33
|
+
* to the next protocol is the caller's retry, which it already owns.
|
|
34
|
+
*/
|
|
35
|
+
readonly fallbacks: readonly RoutedChat[];
|
|
36
|
+
};
|
|
37
|
+
}[K];
|
|
38
|
+
/** The embeddings client of a pinned profile or an embeddings model, with the length of its vectors. */
|
|
39
|
+
export interface RoutedEmbeddings {
|
|
40
|
+
readonly kind: "openai";
|
|
41
|
+
readonly name: string;
|
|
42
|
+
readonly embeddings: OpenAIEmbeddings;
|
|
43
|
+
/** The vector length the pool declares for it; undefined when the pool declares none. */
|
|
44
|
+
readonly dimensions: number | undefined;
|
|
45
|
+
}
|
|
46
|
+
/** A `fetch`-compatible function. */
|
|
47
|
+
export type RoutedFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
48
|
+
/** How the routed clients reach the gateway. */
|
|
49
|
+
export interface RoutedModelsOptions {
|
|
50
|
+
/** The gateway's origin, without `/v1` (`http://127.0.0.1:4100`). */
|
|
51
|
+
readonly gatewayUrl: string;
|
|
52
|
+
/** The app's one virtual key (`STARCI_AI_KEY`). */
|
|
53
|
+
readonly virtualKey: string;
|
|
54
|
+
/** The `fetch` every client and the catalog read use. Default: the global one. */
|
|
55
|
+
readonly fetch?: RoutedFetch;
|
|
56
|
+
/** The limit of one call in milliseconds, for every client unless a call to `chat` sets its own. */
|
|
57
|
+
readonly timeoutMs?: number;
|
|
58
|
+
/** Extra headers on every request to the gateway. */
|
|
59
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
60
|
+
}
|
|
61
|
+
/** What one `chat` call may set on the client it gets. */
|
|
62
|
+
export interface RoutedChatOptions {
|
|
63
|
+
/** The limit of one call in milliseconds. */
|
|
64
|
+
readonly timeoutMs?: number;
|
|
65
|
+
/** The most tokens an answer may hold. An `anthropic` client must send one: default 16000. */
|
|
66
|
+
readonly maxTokens?: number;
|
|
67
|
+
/** Extra constructor fields of the kind's own client, for a provider's own features; they win over the defaults. */
|
|
68
|
+
readonly fields?: Readonly<Partial<Record<RoutedKind, Readonly<Record<string, unknown>>>>>;
|
|
69
|
+
}
|
|
70
|
+
/** What a kind's factory is given to build its client. */
|
|
71
|
+
export interface RoutedChatContext {
|
|
72
|
+
readonly name: string;
|
|
73
|
+
/** The gateway's origin, without a trailing slash. */
|
|
74
|
+
readonly gatewayUrl: string;
|
|
75
|
+
readonly virtualKey: string;
|
|
76
|
+
readonly fetch: RoutedFetch | undefined;
|
|
77
|
+
readonly timeoutMs: number | undefined;
|
|
78
|
+
readonly maxTokens: number | undefined;
|
|
79
|
+
/** The headers every request carries: the caller's, and the one that keeps the call on this kind's providers. */
|
|
80
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
81
|
+
/** The caller's own constructor fields for this kind. */
|
|
82
|
+
readonly fields: Readonly<Record<string, unknown>>;
|
|
83
|
+
}
|
|
84
|
+
/** Builds the chat model of one kind; it imports its LangChain package itself, so the package loads only when asked for. */
|
|
85
|
+
export type RoutedChatFactory<K extends RoutedKind = RoutedKind> = (context: RoutedChatContext) => Promise<RoutedChatKinds[K]>;
|
|
86
|
+
/** Why the routed models refused. */
|
|
87
|
+
export type RoutedModelsErrorCode = "ROUTED_CATALOG_UNAVAILABLE" | "ROUTED_NAME_UNKNOWN" | "ROUTED_KIND_UNSUPPORTED" | "ROUTED_NOT_CHAT" | "ROUTED_NOT_EMBEDDINGS" | "ROUTED_PACKAGE_MISSING";
|
|
88
|
+
/** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
|
|
89
|
+
export declare class RoutedModelsError extends DomainError<RoutedModelsErrorCode> {
|
|
90
|
+
constructor(code: RoutedModelsErrorCode, message: string, cause?: unknown);
|
|
91
|
+
}
|
|
92
|
+
/** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
|
|
93
|
+
export declare const ROUTED_ANTHROPIC_MAX_TOKENS = 16000;
|
|
94
|
+
/** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
|
|
95
|
+
export declare function registerChatKind<K extends RoutedKind>(kind: K, factory: RoutedChatFactory<K>): void;
|
|
96
|
+
/** The routed clients of one app. */
|
|
97
|
+
export interface RoutedModels {
|
|
98
|
+
/**
|
|
99
|
+
* The chat model of a pool model or profile, speaking the protocol of its first candidate. `createAgent({ model:
|
|
100
|
+
* (await ai.chat(name)).model })` works for every kind; `kind` is read only for a provider's own features.
|
|
101
|
+
*/
|
|
102
|
+
chat(name: string, options?: RoutedChatOptions): Promise<RoutedChat>;
|
|
103
|
+
/** The embeddings client of a pinned profile or an embeddings model. It never falls back to another model identity. */
|
|
104
|
+
embeddings(name: string): Promise<RoutedEmbeddings>;
|
|
105
|
+
/** Reads the gateway's catalog again: after the pool's config changed. */
|
|
106
|
+
refresh(): Promise<void>;
|
|
107
|
+
}
|
|
108
|
+
/** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
|
|
109
|
+
export declare function createRoutedModels(options: RoutedModelsOptions): RoutedModels;
|
|
110
|
+
/** What one answer cost in tokens and who served it, the same for every kind: what credit billing reads. */
|
|
111
|
+
export interface RoutedUsage {
|
|
112
|
+
/** Prompt tokens billed at the full input price: the prompt without its cache reads and writes. */
|
|
113
|
+
readonly inputTokens: number;
|
|
114
|
+
/** Prompt tokens read from the provider's prompt cache. */
|
|
115
|
+
readonly cacheReadTokens: number;
|
|
116
|
+
/** Prompt tokens written to the provider's prompt cache. */
|
|
117
|
+
readonly cacheWriteTokens: number;
|
|
118
|
+
/** Completion tokens, thinking included. */
|
|
119
|
+
readonly outputTokens: number;
|
|
120
|
+
/** The part of `outputTokens` the model spent thinking; 0 when the provider does not count it. */
|
|
121
|
+
readonly reasoningTokens: number;
|
|
122
|
+
/** The pool model that answered, as the pool's config names it; undefined when the answer carried no usage. */
|
|
123
|
+
readonly model: string | undefined;
|
|
124
|
+
/** The pool provider that answered. */
|
|
125
|
+
readonly provider: string | undefined;
|
|
126
|
+
/** The cost the router computed from the model's prices; undefined when the model declares none or the answer was streamed natively. */
|
|
127
|
+
readonly cost: number | undefined;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
|
|
131
|
+
* in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
|
|
132
|
+
* gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
|
|
133
|
+
*/
|
|
134
|
+
export declare function usageOf(reply: {
|
|
135
|
+
readonly usage_metadata?: unknown;
|
|
136
|
+
readonly response_metadata?: unknown;
|
|
137
|
+
}): RoutedUsage;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { DomainError, isPlainRecord } from "@starci/common";
|
|
2
|
+
/** The one error of this entry: the gateway's catalog could not be read, or it does not serve the name as asked. */
|
|
3
|
+
export class RoutedModelsError extends DomainError {
|
|
4
|
+
constructor(code, message, cause) {
|
|
5
|
+
super({ code, ...(cause === undefined ? {} : { cause }) });
|
|
6
|
+
this.message = message;
|
|
7
|
+
this.name = "RoutedModelsError";
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
/** The `max_tokens` an `anthropic` client sends when the caller names none: thinking counts toward it on current models. */
|
|
11
|
+
export const ROUTED_ANTHROPIC_MAX_TOKENS = 16_000;
|
|
12
|
+
/** The header that keeps a call on the providers of one protocol. */
|
|
13
|
+
const KIND_HEADER = "x-ai-router-kind";
|
|
14
|
+
async function load(name, importer) {
|
|
15
|
+
try {
|
|
16
|
+
return await importer();
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
throw new RoutedModelsError("ROUTED_PACKAGE_MISSING", `install ${name} to use this kind: it is an optional peer of @starci/ai-router/langchain`, error);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const factories = new Map();
|
|
23
|
+
/** Registers the factory of a kind; the built-in kinds are `openai` and `anthropic`. A later registration replaces an earlier one. */
|
|
24
|
+
export function registerChatKind(kind, factory) {
|
|
25
|
+
factories.set(kind, factory);
|
|
26
|
+
}
|
|
27
|
+
registerChatKind("openai", async (context) => {
|
|
28
|
+
const { ChatOpenAI: Model } = await load("@langchain/openai", () => import("@langchain/openai"));
|
|
29
|
+
return new Model({
|
|
30
|
+
model: context.name,
|
|
31
|
+
apiKey: context.virtualKey,
|
|
32
|
+
// The app owns its retries: the router already fails over between keys and providers of the kind.
|
|
33
|
+
maxRetries: 0,
|
|
34
|
+
streamUsage: true,
|
|
35
|
+
...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }),
|
|
36
|
+
...(context.maxTokens === undefined ? {} : { maxTokens: context.maxTokens }),
|
|
37
|
+
...context.fields,
|
|
38
|
+
configuration: { baseURL: `${context.gatewayUrl}/v1`, defaultHeaders: { ...context.headers }, ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
registerChatKind("anthropic", async (context) => {
|
|
42
|
+
const { ChatAnthropic: Model } = await load("@langchain/anthropic", () => import("@langchain/anthropic"));
|
|
43
|
+
return new Model({
|
|
44
|
+
model: context.name,
|
|
45
|
+
apiKey: context.virtualKey,
|
|
46
|
+
// The Anthropic client appends /v1/messages to its base URL itself.
|
|
47
|
+
anthropicApiUrl: context.gatewayUrl,
|
|
48
|
+
maxRetries: 0,
|
|
49
|
+
streamUsage: true,
|
|
50
|
+
maxTokens: context.maxTokens ?? ROUTED_ANTHROPIC_MAX_TOKENS,
|
|
51
|
+
...context.fields,
|
|
52
|
+
clientOptions: { defaultHeaders: { ...context.headers }, ...(context.timeoutMs === undefined ? {} : { timeout: context.timeoutMs }), ...(context.fetch === undefined ? {} : { fetch: context.fetch }) },
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
function strings(value) {
|
|
56
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
57
|
+
}
|
|
58
|
+
function listedOf(entry) {
|
|
59
|
+
if (!isPlainRecord(entry) || typeof entry.id !== "string")
|
|
60
|
+
return undefined;
|
|
61
|
+
return { id: entry.id, kinds: strings(entry.kinds), serves: strings(entry.serves), dimensions: typeof entry.dimensions === "number" ? entry.dimensions : undefined };
|
|
62
|
+
}
|
|
63
|
+
/** Creates the routed clients of one app over its gateway and its one virtual key. Nothing is read until the first call. */
|
|
64
|
+
export function createRoutedModels(options) {
|
|
65
|
+
const gatewayUrl = options.gatewayUrl.replace(/\/+$/, "").replace(/\/v1$/, "");
|
|
66
|
+
const fetcher = options.fetch ?? ((input, init) => fetch(input, init));
|
|
67
|
+
let catalog;
|
|
68
|
+
async function read() {
|
|
69
|
+
let body;
|
|
70
|
+
try {
|
|
71
|
+
const response = await fetcher(`${gatewayUrl}/v1/models`, { headers: { authorization: `Bearer ${options.virtualKey}`, ...options.headers } });
|
|
72
|
+
if (!response.ok)
|
|
73
|
+
throw new Error(`HTTP ${String(response.status)}`);
|
|
74
|
+
body = await response.json();
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
throw new RoutedModelsError("ROUTED_CATALOG_UNAVAILABLE", `the gateway at ${gatewayUrl} did not list its models: ${error instanceof Error ? error.message : "the request failed"}`, error);
|
|
78
|
+
}
|
|
79
|
+
const data = isPlainRecord(body) && Array.isArray(body.data) ? body.data : [];
|
|
80
|
+
return new Map(data.map(listedOf).filter((entry) => entry !== undefined).map((entry) => [entry.id, entry]));
|
|
81
|
+
}
|
|
82
|
+
function listed() {
|
|
83
|
+
catalog ??= read().catch((error) => {
|
|
84
|
+
// A failed read is not remembered: the next call asks again.
|
|
85
|
+
catalog = undefined;
|
|
86
|
+
throw error;
|
|
87
|
+
});
|
|
88
|
+
return catalog;
|
|
89
|
+
}
|
|
90
|
+
async function find(name) {
|
|
91
|
+
let entry = (await listed()).get(name);
|
|
92
|
+
if (entry === undefined) {
|
|
93
|
+
// The pool may have gained the name since the catalog was read.
|
|
94
|
+
catalog = undefined;
|
|
95
|
+
entry = (await listed()).get(name);
|
|
96
|
+
}
|
|
97
|
+
if (entry === undefined)
|
|
98
|
+
throw new RoutedModelsError("ROUTED_NAME_UNKNOWN", `the gateway serves no model or profile "${name}" to this key`);
|
|
99
|
+
return entry;
|
|
100
|
+
}
|
|
101
|
+
async function build(kind, name, call) {
|
|
102
|
+
const factory = factories.get(kind);
|
|
103
|
+
if (factory === undefined)
|
|
104
|
+
throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `"${name}" is served through the ${kind} protocol, which has no registered chat kind`);
|
|
105
|
+
const fields = call.fields?.[kind] ?? {};
|
|
106
|
+
const model = await factory({
|
|
107
|
+
name,
|
|
108
|
+
gatewayUrl,
|
|
109
|
+
virtualKey: options.virtualKey,
|
|
110
|
+
fetch: options.fetch,
|
|
111
|
+
timeoutMs: call.timeoutMs ?? options.timeoutMs,
|
|
112
|
+
maxTokens: call.maxTokens,
|
|
113
|
+
headers: { ...options.headers, [KIND_HEADER]: kind },
|
|
114
|
+
fields,
|
|
115
|
+
});
|
|
116
|
+
// The registry holds each kind's factory under its own key, so the model is the kind's.
|
|
117
|
+
return { kind, name, model, fallbacks: [] };
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
async chat(name, call = {}) {
|
|
121
|
+
const entry = await find(name);
|
|
122
|
+
if (!entry.serves.includes("chat"))
|
|
123
|
+
throw new RoutedModelsError("ROUTED_NOT_CHAT", `"${name}" does not serve chat`);
|
|
124
|
+
const [first, ...rest] = entry.kinds;
|
|
125
|
+
if (first === undefined)
|
|
126
|
+
throw new RoutedModelsError("ROUTED_KIND_UNSUPPORTED", `the gateway names no protocol for "${name}"`);
|
|
127
|
+
const primary = await build(first, name, call);
|
|
128
|
+
const fallbacks = await Promise.all(rest.filter((kind) => factories.has(kind)).map((kind) => build(kind, name, call)));
|
|
129
|
+
return { ...primary, fallbacks };
|
|
130
|
+
},
|
|
131
|
+
async embeddings(name) {
|
|
132
|
+
const entry = await find(name);
|
|
133
|
+
if (!entry.serves.includes("embeddings"))
|
|
134
|
+
throw new RoutedModelsError("ROUTED_NOT_EMBEDDINGS", `"${name}" does not serve embeddings`);
|
|
135
|
+
const { OpenAIEmbeddings: Embeddings } = await load("@langchain/openai", () => import("@langchain/openai"));
|
|
136
|
+
const embeddings = new Embeddings({
|
|
137
|
+
model: name,
|
|
138
|
+
apiKey: options.virtualKey,
|
|
139
|
+
maxRetries: 0,
|
|
140
|
+
...(options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }),
|
|
141
|
+
configuration: { baseURL: `${gatewayUrl}/v1`, defaultHeaders: { ...options.headers }, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) },
|
|
142
|
+
});
|
|
143
|
+
return { kind: "openai", name, embeddings, dimensions: entry.dimensions };
|
|
144
|
+
},
|
|
145
|
+
async refresh() {
|
|
146
|
+
catalog = undefined;
|
|
147
|
+
await listed();
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
function count(value) {
|
|
152
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
|
|
153
|
+
}
|
|
154
|
+
function record(value) {
|
|
155
|
+
return isPlainRecord(value) ? value : {};
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* The usage of a LangChain answer that came through the gateway (an `AIMessage`, or the chunks of a stream added up),
|
|
159
|
+
* in one shape for every kind. The token counts are LangChain's `usage_metadata`; the lane that answered is what the
|
|
160
|
+
* gateway wrote into the provider's raw usage, which LangChain keeps in `response_metadata.usage`.
|
|
161
|
+
*/
|
|
162
|
+
export function usageOf(reply) {
|
|
163
|
+
const usage = record(reply.usage_metadata);
|
|
164
|
+
const metadata = record(reply.response_metadata);
|
|
165
|
+
const raw = record(metadata.usage);
|
|
166
|
+
const routing = record(raw.served);
|
|
167
|
+
const inputDetails = record(usage.input_token_details);
|
|
168
|
+
const cacheReadTokens = count(inputDetails.cache_read);
|
|
169
|
+
const cacheWriteTokens = count(inputDetails.cache_creation);
|
|
170
|
+
// OpenAI's raw usage names thinking tokens `reasoning_tokens`, Anthropic's `thinking_tokens`.
|
|
171
|
+
const reasoningTokens = count(record(usage.output_token_details).reasoning) || count(record(raw.completion_tokens_details).reasoning_tokens) || count(record(raw.output_tokens_details).thinking_tokens);
|
|
172
|
+
return {
|
|
173
|
+
inputTokens: Math.max(0, count(usage.input_tokens) - cacheReadTokens - cacheWriteTokens),
|
|
174
|
+
cacheReadTokens,
|
|
175
|
+
cacheWriteTokens,
|
|
176
|
+
outputTokens: count(usage.output_tokens),
|
|
177
|
+
reasoningTokens,
|
|
178
|
+
model: typeof routing.model === "string" ? routing.model : undefined,
|
|
179
|
+
provider: typeof routing.provider === "string" ? routing.provider : undefined,
|
|
180
|
+
cost: typeof routing.cost === "number" ? routing.cost : undefined,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ModelConfig } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
|
|
4
|
+
* unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
|
|
5
|
+
* fields, and its forced choice is unforced in that shape.
|
|
6
|
+
*/
|
|
7
|
+
export declare function shapeRequest(body: Record<string, unknown>, model: ModelConfig, native?: boolean): Record<string, unknown>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { isPlainRecord } from "@starci/common";
|
|
2
|
+
/**
|
|
3
|
+
* What one model needs of a chat request and another would refuse, applied from the model's own config just before the
|
|
4
|
+
* request is sent to its provider: default fields (`params`), fields never sent (`omit`) and a forced tool choice the
|
|
5
|
+
* model refuses (`forcedToolUse: false`). The app above sends one request shape whatever lane serves it.
|
|
6
|
+
*/
|
|
7
|
+
const ANY_TOOL = "You must call one of the provided tools in this response.";
|
|
8
|
+
function namedTool(name) {
|
|
9
|
+
return `You must call the tool "${name}" in this response.`;
|
|
10
|
+
}
|
|
11
|
+
function isSystem(message) {
|
|
12
|
+
return isPlainRecord(message) && (message.role === "system" || message.role === "developer");
|
|
13
|
+
}
|
|
14
|
+
/** An OpenAI request's forced tool choice as `auto` plus a system instruction, placed after the leading system messages. */
|
|
15
|
+
function unforced(body) {
|
|
16
|
+
const choice = body.tool_choice;
|
|
17
|
+
const text = choice === "required" ? ANY_TOOL : isPlainRecord(choice) && isPlainRecord(choice.function) && typeof choice.function.name === "string" ? namedTool(choice.function.name) : undefined;
|
|
18
|
+
if (text === undefined)
|
|
19
|
+
return body;
|
|
20
|
+
const messages = Array.isArray(body.messages) ? body.messages : [];
|
|
21
|
+
const lead = messages.findIndex((message) => !isSystem(message));
|
|
22
|
+
const at = lead === -1 ? messages.length : lead;
|
|
23
|
+
return { ...body, tool_choice: "auto", messages: [...messages.slice(0, at), { role: "system", content: text }, ...messages.slice(at)] };
|
|
24
|
+
}
|
|
25
|
+
/** An Anthropic request's forced tool choice (`any`, `tool`) as `auto` plus an instruction at the end of its system prompt. */
|
|
26
|
+
function unforcedNative(body) {
|
|
27
|
+
const choice = body.tool_choice;
|
|
28
|
+
if (!isPlainRecord(choice))
|
|
29
|
+
return body;
|
|
30
|
+
const text = choice.type === "any" ? ANY_TOOL : choice.type === "tool" && typeof choice.name === "string" ? namedTool(choice.name) : undefined;
|
|
31
|
+
if (text === undefined)
|
|
32
|
+
return body;
|
|
33
|
+
const system = body.system;
|
|
34
|
+
const block = { type: "text", text };
|
|
35
|
+
return {
|
|
36
|
+
...body,
|
|
37
|
+
tool_choice: { type: "auto", ...(choice.disable_parallel_tool_use === true ? { disable_parallel_tool_use: true } : {}) },
|
|
38
|
+
system: Array.isArray(system) ? [...system, block] : typeof system === "string" && system !== "" ? `${system}\n\n${text}` : text,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The chat request a model is sent: its `params` under the request's own fields, without its `omit`, its tool choice
|
|
43
|
+
* unforced when it refuses one. A `native` request is in the Anthropic Messages shape: `params` and `omit` name its
|
|
44
|
+
* fields, and its forced choice is unforced in that shape.
|
|
45
|
+
*/
|
|
46
|
+
export function shapeRequest(body, model, native = false) {
|
|
47
|
+
if (model.params === undefined && model.omit === undefined && model.forcedToolUse !== false)
|
|
48
|
+
return body;
|
|
49
|
+
const omitted = new Set(model.omit ?? []);
|
|
50
|
+
const out = Object.fromEntries(Object.entries({ ...model.params, ...body }).filter(([field]) => !omitted.has(field)));
|
|
51
|
+
if (model.forcedToolUse !== false)
|
|
52
|
+
return out;
|
|
53
|
+
return native ? unforcedNative(out) : unforced(out);
|
|
54
|
+
}
|
package/dist/pool.d.ts
CHANGED
|
@@ -56,7 +56,11 @@ export interface Blocked {
|
|
|
56
56
|
/** Set when every lane is out because its key is blacklisted or its provider's budget is spent. */
|
|
57
57
|
readonly blacklisted: Blacklisted | null;
|
|
58
58
|
}
|
|
59
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
|
|
61
|
+
* is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
|
|
62
|
+
* one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
|
|
63
|
+
*/
|
|
60
64
|
export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
|
|
61
65
|
/**
|
|
62
66
|
* The pool's state: shared breaker, blacklist, headroom and budget records in the store; in-flight counts, latency
|
package/dist/pool.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { tokenParts } from "./cache.js";
|
|
1
2
|
import { windowEnd } from "./classify.js";
|
|
2
3
|
import { DEFAULT_CAPABILITIES, identityOf, modelMatches } from "./config.js";
|
|
3
4
|
import { presetOf } from "./providers.js";
|
|
@@ -13,12 +14,20 @@ function breakerState(record, now) {
|
|
|
13
14
|
return "closed";
|
|
14
15
|
return record.openUntil > now ? "open" : "half-open";
|
|
15
16
|
}
|
|
16
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
|
|
19
|
+
* is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
|
|
20
|
+
* one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
|
|
21
|
+
*/
|
|
17
22
|
export function costOf(model, usage) {
|
|
18
23
|
if (model.cost === undefined || usage === undefined)
|
|
19
24
|
return undefined;
|
|
20
|
-
const
|
|
21
|
-
|
|
25
|
+
const { input, output } = model.cost;
|
|
26
|
+
const cacheWrite = model.cost.cacheWrite ?? input;
|
|
27
|
+
const parts = tokenParts(usage);
|
|
28
|
+
const plain = usage.prompt_tokens - parts.cached - parts.cacheWrite;
|
|
29
|
+
const prompt = plain * input + parts.cached * (model.cost.cacheRead ?? input) + (parts.cacheWrite - parts.cacheWrite1h) * cacheWrite + parts.cacheWrite1h * (model.cost.cacheWrite1h ?? cacheWrite);
|
|
30
|
+
return (prompt + (usage.completion_tokens ?? 0) * output) / 1_000_000;
|
|
22
31
|
}
|
|
23
32
|
/** The UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`) of a time. */
|
|
24
33
|
function periods(now) {
|
|
@@ -343,7 +352,7 @@ export class PoolState {
|
|
|
343
352
|
#count(lane) {
|
|
344
353
|
let counters = this.#counters.get(lane.id);
|
|
345
354
|
if (counters === undefined) {
|
|
346
|
-
counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cost: 0 };
|
|
355
|
+
counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, cost: 0 };
|
|
347
356
|
this.#counters.set(lane.id, counters);
|
|
348
357
|
}
|
|
349
358
|
return counters;
|
|
@@ -427,6 +436,10 @@ export class PoolState {
|
|
|
427
436
|
const cost = costOf(lane.model, usage);
|
|
428
437
|
counters.promptTokens += usage?.prompt_tokens ?? 0;
|
|
429
438
|
counters.completionTokens += usage?.completion_tokens ?? 0;
|
|
439
|
+
const parts = tokenParts(usage);
|
|
440
|
+
counters.cachedTokens += parts.cached;
|
|
441
|
+
counters.cacheWriteTokens += parts.cacheWrite;
|
|
442
|
+
counters.reasoningTokens += parts.reasoning;
|
|
430
443
|
counters.cost += cost ?? 0;
|
|
431
444
|
await this.#spend(lane, cost, now);
|
|
432
445
|
return cost;
|
package/dist/router.js
CHANGED
|
@@ -10,8 +10,10 @@ import { baseUrlOf, presetOf } from "./providers.js";
|
|
|
10
10
|
import { resolveSecret } from "./secrets.js";
|
|
11
11
|
import { readChunk, readCompletion, readEmbeddings, readUsage, vectorLength } from "./shapes.js";
|
|
12
12
|
import { parseSse } from "./sse.js";
|
|
13
|
+
import { shapeRequest } from "./model-request.js";
|
|
13
14
|
import { MemoryStateStore } from "./store.js";
|
|
14
15
|
import { wireOf } from "./wire/index.js";
|
|
16
|
+
import { nativeItems, nativeRequest, nativeUsage } from "./wire/native.js";
|
|
15
17
|
const BUSY_POLL_MS = 250;
|
|
16
18
|
const ERROR_BODY_LIMIT = 4096;
|
|
17
19
|
/** One call's lifetime: its deadline, its cancellation, the pairs it tried and the failures it met. */
|
|
@@ -153,6 +155,21 @@ function chatNeed(request, streaming, policy) {
|
|
|
153
155
|
dimensions: null,
|
|
154
156
|
};
|
|
155
157
|
}
|
|
158
|
+
/** What a request in the Anthropic Messages shape needs of a lane. */
|
|
159
|
+
function messagesNeed(request, streaming, policy) {
|
|
160
|
+
const capabilities = new Set(["chat", ...policy.require]);
|
|
161
|
+
if (streaming)
|
|
162
|
+
capabilities.add("streaming");
|
|
163
|
+
if (request.tools !== undefined && request.tools.length > 0)
|
|
164
|
+
capabilities.add("tools");
|
|
165
|
+
return {
|
|
166
|
+
capabilities: [...capabilities],
|
|
167
|
+
promptTokens: estimateTokens(JSON.stringify([request.system ?? "", request.messages])),
|
|
168
|
+
outputTokens: request.max_tokens ?? 0,
|
|
169
|
+
oneIdentity: policy.pinned,
|
|
170
|
+
dimensions: null,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
156
173
|
function hasToken(chunk) {
|
|
157
174
|
return chunk.choices.some((choice) => Object.entries(choice.delta).some(([field, value]) => field !== "role" && value !== null && value !== undefined && value !== "" && !(Array.isArray(value) && value.length === 0)));
|
|
158
175
|
}
|
|
@@ -206,6 +223,12 @@ export function createRouter(config) {
|
|
|
206
223
|
const fetchImpl = config.fetch ?? ((input, init) => fetch(input, init));
|
|
207
224
|
const random = config.random ?? Math.random;
|
|
208
225
|
const catalog = buildCatalog(config);
|
|
226
|
+
const anthropicProviders = config.providers.filter((provider) => presetOf(provider).protocol === "anthropic").map((provider) => provider.id);
|
|
227
|
+
/** The call's options with its providers narrowed to those that speak the Anthropic protocol. */
|
|
228
|
+
function nativeOptions(options) {
|
|
229
|
+
const asked = options.policy?.providers;
|
|
230
|
+
return { ...options, policy: { ...options.policy, providers: asked === undefined ? anthropicProviders : anthropicProviders.filter((id) => asked.includes(id)) } };
|
|
231
|
+
}
|
|
209
232
|
// The policy is resolved at once, so a bad profile or model throws from the call itself; the clock starts at start().
|
|
210
233
|
function prepare(operation, model, options, need) {
|
|
211
234
|
const policy = resolvePolicy(config, options, model);
|
|
@@ -300,13 +323,20 @@ export function createRouter(config) {
|
|
|
300
323
|
"idempotency-key": call.requestId,
|
|
301
324
|
};
|
|
302
325
|
}
|
|
303
|
-
|
|
326
|
+
/** Sends one attempt; `context` is what the lane's adapter needs again to read the answer. */
|
|
327
|
+
async function open(call, pick, scope, operation, body, options, native = false) {
|
|
304
328
|
const secret = await resolveSecret(pick.lane.key);
|
|
305
329
|
if (secret === undefined)
|
|
306
330
|
throw new AttemptError("auth", `key ${pick.lane.key.alias} resolved to no value`, { missingKey: true });
|
|
307
331
|
scope.secret = secret;
|
|
308
332
|
const wire = wireOf(pick.lane.provider);
|
|
309
|
-
|
|
333
|
+
// A chat request is shaped for the model that serves this attempt: the next lane may need another shape.
|
|
334
|
+
const streaming = operation === "stream";
|
|
335
|
+
const request = operation === "embeddings"
|
|
336
|
+
? wire.embeddings(body, pick.lane.model.id)
|
|
337
|
+
: native
|
|
338
|
+
? nativeRequest(shapeRequest(body, pick.lane.model, true), pick.lane.model, streaming)
|
|
339
|
+
: wire.chat(shapeRequest(body, pick.lane.model), pick.lane.model, streaming, pick.lane.provider);
|
|
310
340
|
let response;
|
|
311
341
|
try {
|
|
312
342
|
response = await fetchImpl(`${baseUrlOf(pick.lane.provider)}${request.path}`, {
|
|
@@ -321,7 +351,7 @@ export function createRouter(config) {
|
|
|
321
351
|
}
|
|
322
352
|
if (!response.ok)
|
|
323
353
|
throw await errorOf(pick.lane, response);
|
|
324
|
-
return response;
|
|
354
|
+
return { response, context: request.context };
|
|
325
355
|
}
|
|
326
356
|
function begin(call, pick) {
|
|
327
357
|
call.attempts++;
|
|
@@ -403,7 +433,7 @@ export function createRouter(config) {
|
|
|
403
433
|
return error;
|
|
404
434
|
return call.error("network", error instanceof Error ? error.message : "the call failed");
|
|
405
435
|
}
|
|
406
|
-
async function unary(call, operation, body, options, parse) {
|
|
436
|
+
async function unary(call, operation, body, options, parse, native = false) {
|
|
407
437
|
try {
|
|
408
438
|
for (;;) {
|
|
409
439
|
const pick = await next(call);
|
|
@@ -411,7 +441,7 @@ export function createRouter(config) {
|
|
|
411
441
|
const scope = new AttemptScope(call, call.policy.timeoutMs);
|
|
412
442
|
const started = Date.now();
|
|
413
443
|
try {
|
|
414
|
-
const response = await open(call, pick, scope, operation, body, options);
|
|
444
|
+
const { response, context } = await open(call, pick, scope, operation, body, options, native);
|
|
415
445
|
let json;
|
|
416
446
|
try {
|
|
417
447
|
json = await response.json();
|
|
@@ -424,7 +454,7 @@ export function createRouter(config) {
|
|
|
424
454
|
// An error inside a 200 body (OpenRouter does this) is classified like an error status.
|
|
425
455
|
if (!isPlainRecord(json) || (json.error !== undefined && json.error !== null))
|
|
426
456
|
throw failureOf(pick.lane, undefined, json, null, "the provider answered an error");
|
|
427
|
-
const parsed = parse(json, pick.lane);
|
|
457
|
+
const parsed = parse(json, pick.lane, context);
|
|
428
458
|
const latencyMs = Date.now() - started;
|
|
429
459
|
const routing = await succeed(call, pick, readRateLimit(response.headers, Date.now()), latencyMs, parsed.usage, { request: body, response: json });
|
|
430
460
|
settle(call, routing, null);
|
|
@@ -452,7 +482,19 @@ export function createRouter(config) {
|
|
|
452
482
|
throw routerError;
|
|
453
483
|
}
|
|
454
484
|
}
|
|
455
|
-
|
|
485
|
+
/** A provider stream as OpenAI chunks: the lane's adapter translates, and a chunk with content is the first token. */
|
|
486
|
+
async function* chatItems(lane, events, context) {
|
|
487
|
+
for await (const event of wireOf(lane.provider).chunks(events, context)) {
|
|
488
|
+
if (event.kind !== "chunk") {
|
|
489
|
+
yield event;
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
const chunk = readChunk(event.chunk);
|
|
493
|
+
if (chunk !== undefined)
|
|
494
|
+
yield { kind: "item", item: chunk, token: hasToken(chunk), usage: readUsage(chunk.usage) };
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
async function* streamCall(call, body, options, items, native = false) {
|
|
456
498
|
let settled = false;
|
|
457
499
|
try {
|
|
458
500
|
for (;;) {
|
|
@@ -466,29 +508,30 @@ export function createRouter(config) {
|
|
|
466
508
|
// "pending" in the finally block means the consumer stopped reading: no catch ran and no success did.
|
|
467
509
|
let outcome = "pending";
|
|
468
510
|
try {
|
|
469
|
-
const response = await open(call, pick, scope, "stream", body, options);
|
|
511
|
+
const { response, context } = await open(call, pick, scope, "stream", body, options, native);
|
|
470
512
|
info = readRateLimit(response.headers, Date.now());
|
|
471
513
|
if (response.body === null)
|
|
472
514
|
throw new AttemptError("transient", "the provider answered no stream body");
|
|
473
515
|
const held = [];
|
|
474
|
-
for await (const event of
|
|
516
|
+
for await (const event of items(pick.lane, parseSse(response.body), context)) {
|
|
517
|
+
// The provider is at work (a ping, hidden thinking) with nothing for the caller yet: the idle timeout starts over.
|
|
518
|
+
if (event.kind === "alive") {
|
|
519
|
+
scope.arm(call.policy.timeoutMs);
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
475
522
|
if (event.kind === "malformed")
|
|
476
523
|
throw new AttemptError("stream", "the provider streamed an event that is not JSON");
|
|
477
524
|
if (event.kind === "error")
|
|
478
525
|
throw failureOf(pick.lane, undefined, event.body, null, "the provider streamed an error");
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
continue;
|
|
482
|
-
const chunkUsage = readUsage(chunk.usage);
|
|
483
|
-
if (chunkUsage !== undefined)
|
|
484
|
-
usage = chunkUsage;
|
|
526
|
+
if (event.usage !== undefined)
|
|
527
|
+
usage = event.usage;
|
|
485
528
|
scope.arm(call.policy.timeoutMs);
|
|
486
529
|
if (firstToken) {
|
|
487
|
-
yield
|
|
530
|
+
yield event.item;
|
|
488
531
|
continue;
|
|
489
532
|
}
|
|
490
|
-
held.push(
|
|
491
|
-
if (
|
|
533
|
+
held.push(event.item);
|
|
534
|
+
if (event.token) {
|
|
492
535
|
firstToken = true;
|
|
493
536
|
for (const early of held.splice(0))
|
|
494
537
|
yield early;
|
|
@@ -543,8 +586,8 @@ export function createRouter(config) {
|
|
|
543
586
|
const body = { ...request };
|
|
544
587
|
delete body.stream;
|
|
545
588
|
const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy));
|
|
546
|
-
const { value, routing } = await unary(start(), "chat", body, options, (json, lane) => {
|
|
547
|
-
const completion = readCompletion(wireOf(lane.provider).chatResponse(json));
|
|
589
|
+
const { value, routing } = await unary(start(), "chat", body, options, (json, lane, context) => {
|
|
590
|
+
const completion = readCompletion(wireOf(lane.provider).chatResponse(json, context));
|
|
548
591
|
if (completion === undefined)
|
|
549
592
|
throw new AttemptError("transient", "the provider answered a completion without choices");
|
|
550
593
|
return { value: completion, usage: completion.usage };
|
|
@@ -557,7 +600,27 @@ export function createRouter(config) {
|
|
|
557
600
|
const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy));
|
|
558
601
|
return {
|
|
559
602
|
requestId,
|
|
560
|
-
[Symbol.asyncIterator]: () => streamCall(start(), body, options),
|
|
603
|
+
[Symbol.asyncIterator]: () => streamCall(start(), body, options, chatItems),
|
|
604
|
+
};
|
|
605
|
+
},
|
|
606
|
+
async messages(request, options = {}) {
|
|
607
|
+
const native = nativeOptions(options);
|
|
608
|
+
const { start } = prepare("chat", request.model, native, (policy) => messagesNeed(request, false, policy));
|
|
609
|
+
const { value, routing } = await unary(start(), "chat", { ...request }, native, (json) => {
|
|
610
|
+
if (json.type !== "message" || !Array.isArray(json.content))
|
|
611
|
+
throw new AttemptError("transient", "the provider answered a body that is not a message");
|
|
612
|
+
return { value: json, usage: nativeUsage(json.usage) };
|
|
613
|
+
}, true);
|
|
614
|
+
const usage = nativeUsage(value.usage);
|
|
615
|
+
return { message: value, ...(usage === undefined ? {} : { usage }), routing };
|
|
616
|
+
},
|
|
617
|
+
messagesStream(request, options = {}) {
|
|
618
|
+
const native = nativeOptions(options);
|
|
619
|
+
const { requestId, start } = prepare("stream", request.model, native, (policy) => messagesNeed(request, true, policy));
|
|
620
|
+
const items = (lane, events) => nativeItems(lane.provider, lane.model, events);
|
|
621
|
+
return {
|
|
622
|
+
requestId,
|
|
623
|
+
[Symbol.asyncIterator]: () => streamCall(start(), { ...request }, native, items, true),
|
|
561
624
|
};
|
|
562
625
|
},
|
|
563
626
|
async embeddings(request, options = {}) {
|