@starci/ai-router 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +170 -30
- package/dist/ai-router.service.d.ts +5 -1
- package/dist/ai-router.service.js +8 -0
- package/dist/cache.d.ts +35 -0
- package/dist/cache.js +76 -0
- package/dist/config.js +31 -2
- package/dist/fake.d.ts +27 -0
- package/dist/fake.js +84 -15
- package/dist/gateway/config-file.js +9 -1
- package/dist/gateway/gateway.js +189 -17
- package/dist/langchain/index.d.ts +137 -0
- package/dist/langchain/index.js +182 -0
- package/dist/model-request.d.ts +7 -0
- package/dist/model-request.js +54 -0
- package/dist/pool.d.ts +5 -1
- package/dist/pool.js +17 -4
- package/dist/router.js +84 -21
- package/dist/testing.d.ts +1 -1
- package/dist/types.d.ts +130 -3
- package/dist/wire/anthropic.d.ts +18 -8
- package/dist/wire/anthropic.js +238 -58
- package/dist/wire/native.d.ts +29 -0
- package/dist/wire/native.js +56 -0
- package/dist/wire/thinking-memory.d.ts +32 -0
- package/dist/wire/thinking-memory.js +61 -0
- package/dist/wire/wire.d.ts +12 -7
- package/dist/wire/wire.js +10 -1
- package/docs/design.md +189 -19
- package/package.json +26 -6
package/dist/testing.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { AiRouterTestingModule } from "./ai-router-testing.module.js";
|
|
2
2
|
export type { AiRouterTestingOptions } from "./ai-router-testing.module.js";
|
|
3
3
|
export { createFakeProvider, PROVIDER_ERRORS } from "./fake.js";
|
|
4
|
-
export type { FakeCall, FakeErrorCase, FakeMatch, FakeProvider, FakeReply, FakeStream, FakeToolCall } from "./fake.js";
|
|
4
|
+
export type { FakeCall, FakeErrorCase, FakeMatch, FakeCacheUsage, FakeProvider, FakeReply, FakeStream, FakeThinking, FakeToolCall } from "./fake.js";
|
package/dist/types.d.ts
CHANGED
|
@@ -3,10 +3,26 @@ import type { StateStore } from "./store.js";
|
|
|
3
3
|
export type MaybePromise<T> = T | Promise<T>;
|
|
4
4
|
/** What a model can do; a request needs every capability its shape implies. */
|
|
5
5
|
export type Capability = "chat" | "streaming" | "tools" | "json" | "embeddings" | "vision";
|
|
6
|
-
/**
|
|
6
|
+
/**
|
|
7
|
+
* The price of a model per one million tokens, in whatever currency the pool uses. The cache prices default to `input`:
|
|
8
|
+
* a provider that bills a cached prompt less (or a cache write more) declares them, and the cost of a call counts each
|
|
9
|
+
* part of the prompt at its own price.
|
|
10
|
+
*/
|
|
7
11
|
export interface ModelCost {
|
|
8
12
|
readonly input: number;
|
|
9
13
|
readonly output: number;
|
|
14
|
+
/** Prompt tokens read from the provider's prompt cache (`usage.prompt_tokens_details.cached_tokens`). Default `input`. */
|
|
15
|
+
readonly cacheRead?: number;
|
|
16
|
+
/** Prompt tokens written to the cache (`cache_write_tokens`); Anthropic's 5-minute write. Default `input`. */
|
|
17
|
+
readonly cacheWrite?: number;
|
|
18
|
+
/** Prompt tokens written to a one-hour cache (`cache_write_1h_tokens`). Default `cacheWrite`. */
|
|
19
|
+
readonly cacheWrite1h?: number;
|
|
20
|
+
}
|
|
21
|
+
/** A prompt-cache breakpoint as a request marks it: on a content part, on a message, on a tool, or at the top level. */
|
|
22
|
+
export interface CacheControl {
|
|
23
|
+
readonly type: "ephemeral";
|
|
24
|
+
/** How long the entry lives; the provider's default (five minutes at Anthropic) when absent. */
|
|
25
|
+
readonly ttl?: "5m" | "1h";
|
|
10
26
|
}
|
|
11
27
|
/** One model a provider serves. `id` is the name sent to the provider; `aliases` are names a caller may use instead. */
|
|
12
28
|
export interface ModelConfig {
|
|
@@ -25,6 +41,25 @@ export interface ModelConfig {
|
|
|
25
41
|
readonly identity?: string;
|
|
26
42
|
/** The length of the vectors an embeddings model answers; a response of another length fails the attempt. */
|
|
27
43
|
readonly dimensions?: number;
|
|
44
|
+
/**
|
|
45
|
+
* Request fields sent to this model when a chat request does not set them (`{ max_tokens: 32000, output_config:
|
|
46
|
+
* { effort: "medium" } }`): what one model needs and another would refuse lives here, never in the app. Shallow: a
|
|
47
|
+
* field the request sets replaces the whole default.
|
|
48
|
+
*/
|
|
49
|
+
readonly params?: Readonly<Record<string, unknown>>;
|
|
50
|
+
/** Request fields never sent to this model (`["temperature", "top_p"]` for a model that refuses sampling parameters). */
|
|
51
|
+
readonly omit?: readonly string[];
|
|
52
|
+
/**
|
|
53
|
+
* False for a model that refuses a forced tool choice (Claude Opus 5.5, Sonnet 5.5, Fable 5.1 and later): `tool_choice`
|
|
54
|
+
* `required` or a named function is sent as `auto` with a system instruction to make the call. `auto` does not
|
|
55
|
+
* guarantee a call, so the caller checks one was made. Default true: the choice is sent as asked.
|
|
56
|
+
*/
|
|
57
|
+
readonly forcedToolUse?: boolean;
|
|
58
|
+
/**
|
|
59
|
+
* Whether this model's provider takes `cache_control` breakpoints. Default: true on an `anthropic` provider and for
|
|
60
|
+
* `anthropic/*` models on `openrouter`, false elsewhere, where the markers are removed before the request is sent.
|
|
61
|
+
*/
|
|
62
|
+
readonly cacheControl?: boolean;
|
|
28
63
|
}
|
|
29
64
|
/** Returns the secret of a key at the moment of an attempt; an empty or missing answer disables the key. */
|
|
30
65
|
export type KeyResolver = () => MaybePromise<string | undefined>;
|
|
@@ -67,8 +102,15 @@ export interface ProviderConfig {
|
|
|
67
102
|
readonly weight?: number;
|
|
68
103
|
/** The most requests this provider carries at once in this process, across its keys. */
|
|
69
104
|
readonly maxConcurrency?: number;
|
|
70
|
-
/** The `max_tokens` sent to a provider that requires one (Anthropic) when the request sets none. Default
|
|
105
|
+
/** The `max_tokens` sent to a provider that requires one (Anthropic) when the request sets none. Default 16000. */
|
|
71
106
|
readonly defaultMaxTokens?: number;
|
|
107
|
+
/**
|
|
108
|
+
* What an Anthropic provider does with the thinking blocks of an answer that calls tools. `drop` (the default) never
|
|
109
|
+
* sends them back: the next request carries the turn without its reasoning, which the API accepts. `replay` keeps
|
|
110
|
+
* each such turn in this process and sends it back exactly as it was received while the conversation before it is
|
|
111
|
+
* unchanged, so the model keeps its reasoning across a tool loop whatever the client above forwards.
|
|
112
|
+
*/
|
|
113
|
+
readonly thinking?: "drop" | "replay";
|
|
72
114
|
readonly budget?: ProviderBudget;
|
|
73
115
|
readonly keys: readonly KeyConfig[];
|
|
74
116
|
readonly models: readonly ModelConfig[];
|
|
@@ -189,6 +231,8 @@ export interface PoolConfig<Profiles extends string = string> {
|
|
|
189
231
|
export interface ContentPart {
|
|
190
232
|
readonly type: string;
|
|
191
233
|
readonly text?: string;
|
|
234
|
+
/** Marks a prompt-cache breakpoint after this part. */
|
|
235
|
+
readonly cache_control?: CacheControl;
|
|
192
236
|
readonly [key: string]: unknown;
|
|
193
237
|
}
|
|
194
238
|
/** One tool call an assistant message carries. */
|
|
@@ -207,6 +251,8 @@ export interface ChatMessage {
|
|
|
207
251
|
readonly name?: string;
|
|
208
252
|
readonly tool_calls?: readonly ToolCall[];
|
|
209
253
|
readonly tool_call_id?: string;
|
|
254
|
+
/** Marks a prompt-cache breakpoint after this message's last part. */
|
|
255
|
+
readonly cache_control?: CacheControl;
|
|
210
256
|
}
|
|
211
257
|
/** An OpenAI chat-completions request; unknown fields pass through to an OpenAI-compatible provider. */
|
|
212
258
|
export interface ChatCompletionRequest {
|
|
@@ -230,13 +276,36 @@ export interface ChatCompletionRequest {
|
|
|
230
276
|
readonly stream_options?: {
|
|
231
277
|
readonly include_usage?: boolean;
|
|
232
278
|
};
|
|
279
|
+
/** Automatic prompt caching: the provider places one breakpoint on the last cacheable block. */
|
|
280
|
+
readonly cache_control?: CacheControl;
|
|
281
|
+
readonly [key: string]: unknown;
|
|
282
|
+
}
|
|
283
|
+
/** How the prompt tokens split between the provider's prompt cache and full-price input. */
|
|
284
|
+
export interface PromptTokensDetails {
|
|
285
|
+
/** Tokens read from the cache. */
|
|
286
|
+
readonly cached_tokens?: number;
|
|
287
|
+
/** Tokens written to the cache, every lifetime together. */
|
|
288
|
+
readonly cache_write_tokens?: number;
|
|
289
|
+
/** The part of `cache_write_tokens` written to a one-hour cache. */
|
|
290
|
+
readonly cache_write_1h_tokens?: number;
|
|
291
|
+
readonly [key: string]: unknown;
|
|
292
|
+
}
|
|
293
|
+
/** How the completion tokens split. */
|
|
294
|
+
export interface CompletionTokensDetails {
|
|
295
|
+
/** Tokens the model spent thinking; they are part of `completion_tokens`. */
|
|
296
|
+
readonly reasoning_tokens?: number;
|
|
233
297
|
readonly [key: string]: unknown;
|
|
234
298
|
}
|
|
235
|
-
/**
|
|
299
|
+
/**
|
|
300
|
+
* Token counts a provider reports. `prompt_tokens` is the whole prompt: cache reads and writes are parts of it, listed
|
|
301
|
+
* in `prompt_tokens_details`, whatever the provider's own usage shape.
|
|
302
|
+
*/
|
|
236
303
|
export interface Usage {
|
|
237
304
|
readonly prompt_tokens: number;
|
|
238
305
|
readonly completion_tokens?: number;
|
|
239
306
|
readonly total_tokens: number;
|
|
307
|
+
readonly prompt_tokens_details?: PromptTokensDetails;
|
|
308
|
+
readonly completion_tokens_details?: CompletionTokensDetails;
|
|
240
309
|
readonly [key: string]: unknown;
|
|
241
310
|
}
|
|
242
311
|
/** Where a successful call went and what it cost. */
|
|
@@ -337,6 +406,39 @@ export interface EmbeddingsBody {
|
|
|
337
406
|
export interface EmbeddingsResponse extends EmbeddingsBody {
|
|
338
407
|
readonly routing: RoutingInfo;
|
|
339
408
|
}
|
|
409
|
+
/**
|
|
410
|
+
* A request in the Anthropic Messages shape, for a model an Anthropic provider serves. Every field passes to the
|
|
411
|
+
* provider as it is (cache breakpoints, thinking, tools); `model` names a pool model or is left to the policy.
|
|
412
|
+
*/
|
|
413
|
+
export interface AnthropicMessagesRequest {
|
|
414
|
+
readonly model?: string;
|
|
415
|
+
readonly messages: readonly unknown[];
|
|
416
|
+
readonly max_tokens?: number;
|
|
417
|
+
readonly system?: unknown;
|
|
418
|
+
readonly tools?: readonly unknown[];
|
|
419
|
+
readonly [key: string]: unknown;
|
|
420
|
+
}
|
|
421
|
+
/** An Anthropic message as the provider answered it, with the usage the router read from it and the routing. */
|
|
422
|
+
export interface AnthropicMessageResult {
|
|
423
|
+
/** The provider's answer, untouched. */
|
|
424
|
+
readonly message: Readonly<Record<string, unknown>>;
|
|
425
|
+
/** Its usage in the router's shape. */
|
|
426
|
+
readonly usage?: Usage;
|
|
427
|
+
readonly routing: RoutingInfo;
|
|
428
|
+
}
|
|
429
|
+
/** One server-sent event of an Anthropic stream as the provider sent it, and the lane that serves the stream. */
|
|
430
|
+
export interface AnthropicStreamEvent {
|
|
431
|
+
/** The event's type (`message_start`, `content_block_delta`, ...). */
|
|
432
|
+
readonly event: string;
|
|
433
|
+
/** Its JSON data, untouched. */
|
|
434
|
+
readonly data: string;
|
|
435
|
+
readonly provider: string;
|
|
436
|
+
readonly model: string;
|
|
437
|
+
}
|
|
438
|
+
/** A stream of Anthropic events plus the id that names the call in telemetry. */
|
|
439
|
+
export interface AnthropicStream extends AsyncIterable<AnthropicStreamEvent> {
|
|
440
|
+
readonly requestId: string;
|
|
441
|
+
}
|
|
340
442
|
/** The per-call options of `chat`, `stream` and `embeddings`. */
|
|
341
443
|
export interface CallOptions<Profiles extends string = string> {
|
|
342
444
|
readonly profile?: Profiles;
|
|
@@ -476,6 +578,12 @@ export interface LaneStats {
|
|
|
476
578
|
readonly failures: Readonly<Partial<Record<ErrorClass, number>>>;
|
|
477
579
|
readonly promptTokens: number;
|
|
478
580
|
readonly completionTokens: number;
|
|
581
|
+
/** The part of `promptTokens` read from the provider's prompt cache. */
|
|
582
|
+
readonly cachedTokens: number;
|
|
583
|
+
/** The part of `promptTokens` written to the provider's prompt cache. */
|
|
584
|
+
readonly cacheWriteTokens: number;
|
|
585
|
+
/** The part of `completionTokens` the model spent thinking. */
|
|
586
|
+
readonly reasoningTokens: number;
|
|
479
587
|
readonly cost: number;
|
|
480
588
|
}
|
|
481
589
|
/** Counters of this process since the router was created. */
|
|
@@ -499,6 +607,10 @@ export interface AiRouter<Profiles extends string = string> {
|
|
|
499
607
|
stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
|
|
500
608
|
/** Sends an embeddings request through the pool. */
|
|
501
609
|
embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
|
|
610
|
+
/** Sends a request in the Anthropic Messages shape, untranslated, through the pool's Anthropic providers only. */
|
|
611
|
+
messages(request: AnthropicMessagesRequest, options?: CallOptions<Profiles>): Promise<AnthropicMessageResult>;
|
|
612
|
+
/** Streams an Anthropic Messages request as the provider's own events; fails over only before the first content. */
|
|
613
|
+
messagesStream(request: AnthropicMessagesRequest, options?: CallOptions<Profiles>): AnthropicStream;
|
|
502
614
|
/** Reads every breaker, blacklisted key, headroom and budget from the store. */
|
|
503
615
|
health(): Promise<HealthSnapshot>;
|
|
504
616
|
/** Returns this process's counters. */
|
|
@@ -515,9 +627,19 @@ export interface CatalogModel {
|
|
|
515
627
|
readonly id: string;
|
|
516
628
|
readonly aliases: readonly string[];
|
|
517
629
|
readonly providers: readonly string[];
|
|
630
|
+
/** The wire protocols of those providers, each once, in provider order: `openai`, `anthropic`. */
|
|
631
|
+
readonly protocols: readonly ("openai" | "anthropic")[];
|
|
518
632
|
readonly capabilities: readonly Capability[];
|
|
519
633
|
readonly contextWindow?: number;
|
|
520
634
|
readonly cost?: ModelCost;
|
|
635
|
+
/** The vector length of an embeddings model, when it declares one. */
|
|
636
|
+
readonly dimensions?: number;
|
|
637
|
+
}
|
|
638
|
+
/** One provider as the catalog lists it: its id, its kind and the wire protocol it speaks. No key. */
|
|
639
|
+
export interface CatalogProvider {
|
|
640
|
+
readonly id: string;
|
|
641
|
+
readonly kind: ProviderKind;
|
|
642
|
+
readonly protocol: "openai" | "anthropic";
|
|
521
643
|
}
|
|
522
644
|
/** One named profile as the catalog lists it. */
|
|
523
645
|
export interface CatalogProfile {
|
|
@@ -530,6 +652,10 @@ export interface CatalogProfile {
|
|
|
530
652
|
readonly pinned: boolean;
|
|
531
653
|
/** The operations its first model serves: `chat`, `embeddings` or both. */
|
|
532
654
|
readonly serves: readonly ("chat" | "embeddings")[];
|
|
655
|
+
/** The wire protocols of its candidates' providers, each once, in candidate order: the first is its first candidate's. */
|
|
656
|
+
readonly protocols: readonly ("openai" | "anthropic")[];
|
|
657
|
+
/** The vector length of its first model, when that model declares one. */
|
|
658
|
+
readonly dimensions?: number;
|
|
533
659
|
}
|
|
534
660
|
/** The names a caller may route by: the defaults' fallbacks, every profile and every model. */
|
|
535
661
|
export interface RouterCatalog {
|
|
@@ -537,6 +663,7 @@ export interface RouterCatalog {
|
|
|
537
663
|
readonly model?: string;
|
|
538
664
|
readonly fallbacks: readonly string[];
|
|
539
665
|
};
|
|
666
|
+
readonly providers: readonly CatalogProvider[];
|
|
540
667
|
readonly profiles: readonly CatalogProfile[];
|
|
541
668
|
readonly models: readonly CatalogModel[];
|
|
542
669
|
}
|
package/dist/wire/anthropic.d.ts
CHANGED
|
@@ -1,21 +1,31 @@
|
|
|
1
1
|
import type { ProviderConfig } from "../types.js";
|
|
2
2
|
import type { Wire } from "./wire.js";
|
|
3
3
|
/**
|
|
4
|
-
* The Anthropic Messages API (`POST /v1/messages
|
|
5
|
-
* /build-with-claude/streaming
|
|
6
|
-
*
|
|
4
|
+
* The Anthropic Messages API (`POST /v1/messages`) behind the OpenAI chat-completions shape. Read at
|
|
5
|
+
* platform.claude.com/docs/en: /api/messages and /build-with-claude/streaming on 2026-10-31; /build-with-claude/
|
|
6
|
+
* prompt-caching, /thinking, /thinking-tool-workflows and /preserved-thinking on 2026-11-10. docs/design.md, "The
|
|
7
|
+
* Anthropic adapter", maps every field; what has no counterpart is refused (`n` above 1) or dropped (seed, penalties,
|
|
8
|
+
* logprobs).
|
|
7
9
|
*/
|
|
8
10
|
/** The `anthropic-version` sent unless the provider's headers set one. */
|
|
9
11
|
export declare const ANTHROPIC_VERSION = "2023-06-01";
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* The `max_tokens` sent when neither the request nor the provider's `defaultMaxTokens` gives one: Anthropic requires it,
|
|
14
|
+
* and on current models thinking counts toward it, so a low cap cuts an answer before its first visible token.
|
|
15
|
+
*/
|
|
16
|
+
export declare const ANTHROPIC_DEFAULT_MAX_TOKENS = 16000;
|
|
12
17
|
/** Builds the Anthropic Messages body of an OpenAI chat request. */
|
|
13
18
|
export declare function toAnthropicRequest(body: Record<string, unknown>, model: string, stream: boolean, provider: ProviderConfig): Record<string, unknown>;
|
|
14
19
|
/** Anthropic's stop reason as OpenAI's finish reason. */
|
|
15
20
|
export declare function finishReason(reason: unknown): string | null;
|
|
16
|
-
/**
|
|
17
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Anthropic usage as OpenAI usage. Anthropic lists the uncached input, the cache reads and the cache writes apart; the
|
|
23
|
+
* prompt here is their sum, with the reads and writes named in `prompt_tokens_details` (`cached_tokens`,
|
|
24
|
+
* `cache_write_tokens`, and `cache_write_1h_tokens` when the answer splits writes by lifetime). Thinking tokens, when
|
|
25
|
+
* the answer counts them, are `completion_tokens_details.reasoning_tokens`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function usageOf(usage: unknown): Record<string, unknown> | undefined;
|
|
18
28
|
/** An Anthropic message as an OpenAI `chat.completion`. */
|
|
19
|
-
export declare function fromAnthropicMessage(json: Record<string, unknown
|
|
29
|
+
export declare function fromAnthropicMessage(json: Record<string, unknown>, context?: unknown): Record<string, unknown>;
|
|
20
30
|
/** The Anthropic Messages API behind the OpenAI shape: `x-api-key` auth, `/messages`, and its SSE events as OpenAI chunks. */
|
|
21
31
|
export declare const anthropicWire: Wire;
|