@starci/ai-router 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +176 -21
- package/dist/ai-router.service.d.ts +8 -1
- package/dist/ai-router.service.js +12 -0
- package/dist/cache.d.ts +35 -0
- package/dist/cache.js +76 -0
- package/dist/config.js +31 -2
- package/dist/fake.d.ts +27 -0
- package/dist/fake.js +84 -15
- package/dist/gateway/config-file.js +9 -1
- package/dist/gateway/gateway.js +189 -17
- package/dist/gateway/index.d.ts +2 -0
- package/dist/gateway/index.js +1 -0
- package/dist/known-models.d.ts +81 -0
- package/dist/known-models.js +142 -0
- package/dist/langchain/index.d.ts +137 -0
- package/dist/langchain/index.js +182 -0
- package/dist/model-request.d.ts +7 -0
- package/dist/model-request.js +54 -0
- package/dist/pool.d.ts +5 -1
- package/dist/pool.js +17 -4
- package/dist/router.js +86 -21
- package/dist/testing.d.ts +1 -1
- package/dist/types.d.ts +133 -3
- package/dist/wire/anthropic.d.ts +18 -8
- package/dist/wire/anthropic.js +238 -58
- package/dist/wire/native.d.ts +29 -0
- package/dist/wire/native.js +56 -0
- package/dist/wire/thinking-memory.d.ts +32 -0
- package/dist/wire/thinking-memory.js +61 -0
- package/dist/wire/wire.d.ts +12 -7
- package/dist/wire/wire.js +10 -1
- package/docs/design.md +189 -19
- package/package.json +26 -6
package/dist/pool.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { tokenParts } from "./cache.js";
|
|
1
2
|
import { windowEnd } from "./classify.js";
|
|
2
3
|
import { DEFAULT_CAPABILITIES, identityOf, modelMatches } from "./config.js";
|
|
3
4
|
import { presetOf } from "./providers.js";
|
|
@@ -13,12 +14,20 @@ function breakerState(record, now) {
|
|
|
13
14
|
return "closed";
|
|
14
15
|
return record.openUntil > now ? "open" : "half-open";
|
|
15
16
|
}
|
|
16
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. The prompt
|
|
19
|
+
* is priced in its parts: what was read from the cache at `cacheRead`, what was written to it at `cacheWrite` (its
|
|
20
|
+
* one-hour part at `cacheWrite1h`), the rest at `input`; a cache price the model does not declare is `input`.
|
|
21
|
+
*/
|
|
17
22
|
export function costOf(model, usage) {
|
|
18
23
|
if (model.cost === undefined || usage === undefined)
|
|
19
24
|
return undefined;
|
|
20
|
-
const
|
|
21
|
-
|
|
25
|
+
const { input, output } = model.cost;
|
|
26
|
+
const cacheWrite = model.cost.cacheWrite ?? input;
|
|
27
|
+
const parts = tokenParts(usage);
|
|
28
|
+
const plain = usage.prompt_tokens - parts.cached - parts.cacheWrite;
|
|
29
|
+
const prompt = plain * input + parts.cached * (model.cost.cacheRead ?? input) + (parts.cacheWrite - parts.cacheWrite1h) * cacheWrite + parts.cacheWrite1h * (model.cost.cacheWrite1h ?? cacheWrite);
|
|
30
|
+
return (prompt + (usage.completion_tokens ?? 0) * output) / 1_000_000;
|
|
22
31
|
}
|
|
23
32
|
/** The UTC day (`YYYY-MM-DD`) and month (`YYYY-MM`) of a time. */
|
|
24
33
|
function periods(now) {
|
|
@@ -343,7 +352,7 @@ export class PoolState {
|
|
|
343
352
|
#count(lane) {
|
|
344
353
|
let counters = this.#counters.get(lane.id);
|
|
345
354
|
if (counters === undefined) {
|
|
346
|
-
counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cost: 0 };
|
|
355
|
+
counters = { attempts: 0, successes: 0, failures: {}, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, cost: 0 };
|
|
347
356
|
this.#counters.set(lane.id, counters);
|
|
348
357
|
}
|
|
349
358
|
return counters;
|
|
@@ -427,6 +436,10 @@ export class PoolState {
|
|
|
427
436
|
const cost = costOf(lane.model, usage);
|
|
428
437
|
counters.promptTokens += usage?.prompt_tokens ?? 0;
|
|
429
438
|
counters.completionTokens += usage?.completion_tokens ?? 0;
|
|
439
|
+
const parts = tokenParts(usage);
|
|
440
|
+
counters.cachedTokens += parts.cached;
|
|
441
|
+
counters.cacheWriteTokens += parts.cacheWrite;
|
|
442
|
+
counters.reasoningTokens += parts.reasoning;
|
|
430
443
|
counters.cost += cost ?? 0;
|
|
431
444
|
await this.#spend(lane, cost, now);
|
|
432
445
|
return cost;
|
package/dist/router.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isPlainRecord } from "@starci/common";
|
|
2
2
|
import { AttemptError } from "./attempt.js";
|
|
3
3
|
import { classifyFailure } from "./classify.js";
|
|
4
|
+
import { availableModels } from "./known-models.js";
|
|
4
5
|
import { buildCatalog, resolvePolicy, resolveSettings, validateConfig } from "./config.js";
|
|
5
6
|
import { AiRouterError, isRetryable, scrub } from "./errors.js";
|
|
6
7
|
import { Emitter } from "./events.js";
|
|
@@ -10,8 +11,10 @@ import { baseUrlOf, presetOf } from "./providers.js";
|
|
|
10
11
|
import { resolveSecret } from "./secrets.js";
|
|
11
12
|
import { readChunk, readCompletion, readEmbeddings, readUsage, vectorLength } from "./shapes.js";
|
|
12
13
|
import { parseSse } from "./sse.js";
|
|
14
|
+
import { shapeRequest } from "./model-request.js";
|
|
13
15
|
import { MemoryStateStore } from "./store.js";
|
|
14
16
|
import { wireOf } from "./wire/index.js";
|
|
17
|
+
import { nativeItems, nativeRequest, nativeUsage } from "./wire/native.js";
|
|
15
18
|
const BUSY_POLL_MS = 250;
|
|
16
19
|
const ERROR_BODY_LIMIT = 4096;
|
|
17
20
|
/** One call's lifetime: its deadline, its cancellation, the pairs it tried and the failures it met. */
|
|
@@ -153,6 +156,21 @@ function chatNeed(request, streaming, policy) {
|
|
|
153
156
|
dimensions: null,
|
|
154
157
|
};
|
|
155
158
|
}
|
|
159
|
+
/** What a request in the Anthropic Messages shape needs of a lane. */
|
|
160
|
+
function messagesNeed(request, streaming, policy) {
|
|
161
|
+
const capabilities = new Set(["chat", ...policy.require]);
|
|
162
|
+
if (streaming)
|
|
163
|
+
capabilities.add("streaming");
|
|
164
|
+
if (request.tools !== undefined && request.tools.length > 0)
|
|
165
|
+
capabilities.add("tools");
|
|
166
|
+
return {
|
|
167
|
+
capabilities: [...capabilities],
|
|
168
|
+
promptTokens: estimateTokens(JSON.stringify([request.system ?? "", request.messages])),
|
|
169
|
+
outputTokens: request.max_tokens ?? 0,
|
|
170
|
+
oneIdentity: policy.pinned,
|
|
171
|
+
dimensions: null,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
156
174
|
function hasToken(chunk) {
|
|
157
175
|
return chunk.choices.some((choice) => Object.entries(choice.delta).some(([field, value]) => field !== "role" && value !== null && value !== undefined && value !== "" && !(Array.isArray(value) && value.length === 0)));
|
|
158
176
|
}
|
|
@@ -206,6 +224,12 @@ export function createRouter(config) {
|
|
|
206
224
|
const fetchImpl = config.fetch ?? ((input, init) => fetch(input, init));
|
|
207
225
|
const random = config.random ?? Math.random;
|
|
208
226
|
const catalog = buildCatalog(config);
|
|
227
|
+
const anthropicProviders = config.providers.filter((provider) => presetOf(provider).protocol === "anthropic").map((provider) => provider.id);
|
|
228
|
+
/** The call's options with its providers narrowed to those that speak the Anthropic protocol. */
|
|
229
|
+
function nativeOptions(options) {
|
|
230
|
+
const asked = options.policy?.providers;
|
|
231
|
+
return { ...options, policy: { ...options.policy, providers: asked === undefined ? anthropicProviders : anthropicProviders.filter((id) => asked.includes(id)) } };
|
|
232
|
+
}
|
|
209
233
|
// The policy is resolved at once, so a bad profile or model throws from the call itself; the clock starts at start().
|
|
210
234
|
function prepare(operation, model, options, need) {
|
|
211
235
|
const policy = resolvePolicy(config, options, model);
|
|
@@ -300,13 +324,20 @@ export function createRouter(config) {
|
|
|
300
324
|
"idempotency-key": call.requestId,
|
|
301
325
|
};
|
|
302
326
|
}
|
|
303
|
-
|
|
327
|
+
/** Sends one attempt; `context` is what the lane's adapter needs again to read the answer. */
|
|
328
|
+
async function open(call, pick, scope, operation, body, options, native = false) {
|
|
304
329
|
const secret = await resolveSecret(pick.lane.key);
|
|
305
330
|
if (secret === undefined)
|
|
306
331
|
throw new AttemptError("auth", `key ${pick.lane.key.alias} resolved to no value`, { missingKey: true });
|
|
307
332
|
scope.secret = secret;
|
|
308
333
|
const wire = wireOf(pick.lane.provider);
|
|
309
|
-
|
|
334
|
+
// A chat request is shaped for the model that serves this attempt: the next lane may need another shape.
|
|
335
|
+
const streaming = operation === "stream";
|
|
336
|
+
const request = operation === "embeddings"
|
|
337
|
+
? wire.embeddings(body, pick.lane.model.id)
|
|
338
|
+
: native
|
|
339
|
+
? nativeRequest(shapeRequest(body, pick.lane.model, true), pick.lane.model, streaming)
|
|
340
|
+
: wire.chat(shapeRequest(body, pick.lane.model), pick.lane.model, streaming, pick.lane.provider);
|
|
310
341
|
let response;
|
|
311
342
|
try {
|
|
312
343
|
response = await fetchImpl(`${baseUrlOf(pick.lane.provider)}${request.path}`, {
|
|
@@ -321,7 +352,7 @@ export function createRouter(config) {
|
|
|
321
352
|
}
|
|
322
353
|
if (!response.ok)
|
|
323
354
|
throw await errorOf(pick.lane, response);
|
|
324
|
-
return response;
|
|
355
|
+
return { response, context: request.context };
|
|
325
356
|
}
|
|
326
357
|
function begin(call, pick) {
|
|
327
358
|
call.attempts++;
|
|
@@ -403,7 +434,7 @@ export function createRouter(config) {
|
|
|
403
434
|
return error;
|
|
404
435
|
return call.error("network", error instanceof Error ? error.message : "the call failed");
|
|
405
436
|
}
|
|
406
|
-
async function unary(call, operation, body, options, parse) {
|
|
437
|
+
async function unary(call, operation, body, options, parse, native = false) {
|
|
407
438
|
try {
|
|
408
439
|
for (;;) {
|
|
409
440
|
const pick = await next(call);
|
|
@@ -411,7 +442,7 @@ export function createRouter(config) {
|
|
|
411
442
|
const scope = new AttemptScope(call, call.policy.timeoutMs);
|
|
412
443
|
const started = Date.now();
|
|
413
444
|
try {
|
|
414
|
-
const response = await open(call, pick, scope, operation, body, options);
|
|
445
|
+
const { response, context } = await open(call, pick, scope, operation, body, options, native);
|
|
415
446
|
let json;
|
|
416
447
|
try {
|
|
417
448
|
json = await response.json();
|
|
@@ -424,7 +455,7 @@ export function createRouter(config) {
|
|
|
424
455
|
// An error inside a 200 body (OpenRouter does this) is classified like an error status.
|
|
425
456
|
if (!isPlainRecord(json) || (json.error !== undefined && json.error !== null))
|
|
426
457
|
throw failureOf(pick.lane, undefined, json, null, "the provider answered an error");
|
|
427
|
-
const parsed = parse(json, pick.lane);
|
|
458
|
+
const parsed = parse(json, pick.lane, context);
|
|
428
459
|
const latencyMs = Date.now() - started;
|
|
429
460
|
const routing = await succeed(call, pick, readRateLimit(response.headers, Date.now()), latencyMs, parsed.usage, { request: body, response: json });
|
|
430
461
|
settle(call, routing, null);
|
|
@@ -452,7 +483,19 @@ export function createRouter(config) {
|
|
|
452
483
|
throw routerError;
|
|
453
484
|
}
|
|
454
485
|
}
|
|
455
|
-
|
|
486
|
+
/** A provider stream as OpenAI chunks: the lane's adapter translates, and a chunk with content is the first token. */
|
|
487
|
+
async function* chatItems(lane, events, context) {
|
|
488
|
+
for await (const event of wireOf(lane.provider).chunks(events, context)) {
|
|
489
|
+
if (event.kind !== "chunk") {
|
|
490
|
+
yield event;
|
|
491
|
+
continue;
|
|
492
|
+
}
|
|
493
|
+
const chunk = readChunk(event.chunk);
|
|
494
|
+
if (chunk !== undefined)
|
|
495
|
+
yield { kind: "item", item: chunk, token: hasToken(chunk), usage: readUsage(chunk.usage) };
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
async function* streamCall(call, body, options, items, native = false) {
|
|
456
499
|
let settled = false;
|
|
457
500
|
try {
|
|
458
501
|
for (;;) {
|
|
@@ -466,29 +509,30 @@ export function createRouter(config) {
|
|
|
466
509
|
// "pending" in the finally block means the consumer stopped reading: no catch ran and no success did.
|
|
467
510
|
let outcome = "pending";
|
|
468
511
|
try {
|
|
469
|
-
const response = await open(call, pick, scope, "stream", body, options);
|
|
512
|
+
const { response, context } = await open(call, pick, scope, "stream", body, options, native);
|
|
470
513
|
info = readRateLimit(response.headers, Date.now());
|
|
471
514
|
if (response.body === null)
|
|
472
515
|
throw new AttemptError("transient", "the provider answered no stream body");
|
|
473
516
|
const held = [];
|
|
474
|
-
for await (const event of
|
|
517
|
+
for await (const event of items(pick.lane, parseSse(response.body), context)) {
|
|
518
|
+
// The provider is at work (a ping, hidden thinking) with nothing for the caller yet: the idle timeout starts over.
|
|
519
|
+
if (event.kind === "alive") {
|
|
520
|
+
scope.arm(call.policy.timeoutMs);
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
475
523
|
if (event.kind === "malformed")
|
|
476
524
|
throw new AttemptError("stream", "the provider streamed an event that is not JSON");
|
|
477
525
|
if (event.kind === "error")
|
|
478
526
|
throw failureOf(pick.lane, undefined, event.body, null, "the provider streamed an error");
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
continue;
|
|
482
|
-
const chunkUsage = readUsage(chunk.usage);
|
|
483
|
-
if (chunkUsage !== undefined)
|
|
484
|
-
usage = chunkUsage;
|
|
527
|
+
if (event.usage !== undefined)
|
|
528
|
+
usage = event.usage;
|
|
485
529
|
scope.arm(call.policy.timeoutMs);
|
|
486
530
|
if (firstToken) {
|
|
487
|
-
yield
|
|
531
|
+
yield event.item;
|
|
488
532
|
continue;
|
|
489
533
|
}
|
|
490
|
-
held.push(
|
|
491
|
-
if (
|
|
534
|
+
held.push(event.item);
|
|
535
|
+
if (event.token) {
|
|
492
536
|
firstToken = true;
|
|
493
537
|
for (const early of held.splice(0))
|
|
494
538
|
yield early;
|
|
@@ -543,8 +587,8 @@ export function createRouter(config) {
|
|
|
543
587
|
const body = { ...request };
|
|
544
588
|
delete body.stream;
|
|
545
589
|
const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy));
|
|
546
|
-
const { value, routing } = await unary(start(), "chat", body, options, (json, lane) => {
|
|
547
|
-
const completion = readCompletion(wireOf(lane.provider).chatResponse(json));
|
|
590
|
+
const { value, routing } = await unary(start(), "chat", body, options, (json, lane, context) => {
|
|
591
|
+
const completion = readCompletion(wireOf(lane.provider).chatResponse(json, context));
|
|
548
592
|
if (completion === undefined)
|
|
549
593
|
throw new AttemptError("transient", "the provider answered a completion without choices");
|
|
550
594
|
return { value: completion, usage: completion.usage };
|
|
@@ -557,7 +601,27 @@ export function createRouter(config) {
|
|
|
557
601
|
const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy));
|
|
558
602
|
return {
|
|
559
603
|
requestId,
|
|
560
|
-
[Symbol.asyncIterator]: () => streamCall(start(), body, options),
|
|
604
|
+
[Symbol.asyncIterator]: () => streamCall(start(), body, options, chatItems),
|
|
605
|
+
};
|
|
606
|
+
},
|
|
607
|
+
async messages(request, options = {}) {
|
|
608
|
+
const native = nativeOptions(options);
|
|
609
|
+
const { start } = prepare("chat", request.model, native, (policy) => messagesNeed(request, false, policy));
|
|
610
|
+
const { value, routing } = await unary(start(), "chat", { ...request }, native, (json) => {
|
|
611
|
+
if (json.type !== "message" || !Array.isArray(json.content))
|
|
612
|
+
throw new AttemptError("transient", "the provider answered a body that is not a message");
|
|
613
|
+
return { value: json, usage: nativeUsage(json.usage) };
|
|
614
|
+
}, true);
|
|
615
|
+
const usage = nativeUsage(value.usage);
|
|
616
|
+
return { message: value, ...(usage === undefined ? {} : { usage }), routing };
|
|
617
|
+
},
|
|
618
|
+
messagesStream(request, options = {}) {
|
|
619
|
+
const native = nativeOptions(options);
|
|
620
|
+
const { requestId, start } = prepare("stream", request.model, native, (policy) => messagesNeed(request, true, policy));
|
|
621
|
+
const items = (lane, events) => nativeItems(lane.provider, lane.model, events);
|
|
622
|
+
return {
|
|
623
|
+
requestId,
|
|
624
|
+
[Symbol.asyncIterator]: () => streamCall(start(), { ...request }, native, items, true),
|
|
561
625
|
};
|
|
562
626
|
},
|
|
563
627
|
async embeddings(request, options = {}) {
|
|
@@ -583,6 +647,7 @@ export function createRouter(config) {
|
|
|
583
647
|
return { ...value, routing };
|
|
584
648
|
},
|
|
585
649
|
catalog: () => catalog,
|
|
650
|
+
availableModels: () => availableModels(config),
|
|
586
651
|
health: () => state.health(),
|
|
587
652
|
stats: () => state.stats(),
|
|
588
653
|
on: (event, handler) => emitter.on(event, handler),
|
package/dist/testing.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { AiRouterTestingModule } from "./ai-router-testing.module.js";
|
|
2
2
|
export type { AiRouterTestingOptions } from "./ai-router-testing.module.js";
|
|
3
3
|
export { createFakeProvider, PROVIDER_ERRORS } from "./fake.js";
|
|
4
|
-
export type { FakeCall, FakeErrorCase, FakeMatch, FakeProvider, FakeReply, FakeStream, FakeToolCall } from "./fake.js";
|
|
4
|
+
export type { FakeCall, FakeErrorCase, FakeMatch, FakeCacheUsage, FakeProvider, FakeReply, FakeStream, FakeThinking, FakeToolCall } from "./fake.js";
|
package/dist/types.d.ts
CHANGED
|
@@ -1,12 +1,29 @@
|
|
|
1
|
+
import type { AvailableModel } from "./known-models.js";
|
|
1
2
|
import type { StateStore } from "./store.js";
|
|
2
3
|
/** A value or a promise of it: store adapters and key resolvers may answer either way. */
|
|
3
4
|
export type MaybePromise<T> = T | Promise<T>;
|
|
4
5
|
/** What a model can do; a request needs every capability its shape implies. */
|
|
5
6
|
export type Capability = "chat" | "streaming" | "tools" | "json" | "embeddings" | "vision";
|
|
6
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* The price of a model per one million tokens, in whatever currency the pool uses. The cache prices default to `input`:
|
|
9
|
+
* a provider that bills a cached prompt less (or a cache write more) declares them, and the cost of a call counts each
|
|
10
|
+
* part of the prompt at its own price.
|
|
11
|
+
*/
|
|
7
12
|
export interface ModelCost {
|
|
8
13
|
readonly input: number;
|
|
9
14
|
readonly output: number;
|
|
15
|
+
/** Prompt tokens read from the provider's prompt cache (`usage.prompt_tokens_details.cached_tokens`). Default `input`. */
|
|
16
|
+
readonly cacheRead?: number;
|
|
17
|
+
/** Prompt tokens written to the cache (`cache_write_tokens`); Anthropic's 5-minute write. Default `input`. */
|
|
18
|
+
readonly cacheWrite?: number;
|
|
19
|
+
/** Prompt tokens written to a one-hour cache (`cache_write_1h_tokens`). Default `cacheWrite`. */
|
|
20
|
+
readonly cacheWrite1h?: number;
|
|
21
|
+
}
|
|
22
|
+
/** A prompt-cache breakpoint as a request marks it: on a content part, on a message, on a tool, or at the top level. */
|
|
23
|
+
export interface CacheControl {
|
|
24
|
+
readonly type: "ephemeral";
|
|
25
|
+
/** How long the entry lives; the provider's default (five minutes at Anthropic) when absent. */
|
|
26
|
+
readonly ttl?: "5m" | "1h";
|
|
10
27
|
}
|
|
11
28
|
/** One model a provider serves. `id` is the name sent to the provider; `aliases` are names a caller may use instead. */
|
|
12
29
|
export interface ModelConfig {
|
|
@@ -25,6 +42,25 @@ export interface ModelConfig {
|
|
|
25
42
|
readonly identity?: string;
|
|
26
43
|
/** The length of the vectors an embeddings model answers; a response of another length fails the attempt. */
|
|
27
44
|
readonly dimensions?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Request fields sent to this model when a chat request does not set them (`{ max_tokens: 32000, output_config:
|
|
47
|
+
* { effort: "medium" } }`): what one model needs and another would refuse lives here, never in the app. Shallow: a
|
|
48
|
+
* field the request sets replaces the whole default.
|
|
49
|
+
*/
|
|
50
|
+
readonly params?: Readonly<Record<string, unknown>>;
|
|
51
|
+
/** Request fields never sent to this model (`["temperature", "top_p"]` for a model that refuses sampling parameters). */
|
|
52
|
+
readonly omit?: readonly string[];
|
|
53
|
+
/**
|
|
54
|
+
* False for a model that refuses a forced tool choice (Claude Opus 5.5, Sonnet 5.5, Fable 5.1 and later): `tool_choice`
|
|
55
|
+
* `required` or a named function is sent as `auto` with a system instruction to make the call. `auto` does not
|
|
56
|
+
* guarantee a call, so the caller checks one was made. Default true: the choice is sent as asked.
|
|
57
|
+
*/
|
|
58
|
+
readonly forcedToolUse?: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Whether this model's provider takes `cache_control` breakpoints. Default: true on an `anthropic` provider and for
|
|
61
|
+
* `anthropic/*` models on `openrouter`, false elsewhere, where the markers are removed before the request is sent.
|
|
62
|
+
*/
|
|
63
|
+
readonly cacheControl?: boolean;
|
|
28
64
|
}
|
|
29
65
|
/** Returns the secret of a key at the moment of an attempt; an empty or missing answer disables the key. */
|
|
30
66
|
export type KeyResolver = () => MaybePromise<string | undefined>;
|
|
@@ -67,8 +103,15 @@ export interface ProviderConfig {
|
|
|
67
103
|
readonly weight?: number;
|
|
68
104
|
/** The most requests this provider carries at once in this process, across its keys. */
|
|
69
105
|
readonly maxConcurrency?: number;
|
|
70
|
-
/** The `max_tokens` sent to a provider that requires one (Anthropic) when the request sets none. Default
|
|
106
|
+
/** The `max_tokens` sent to a provider that requires one (Anthropic) when the request sets none. Default 16000. */
|
|
71
107
|
readonly defaultMaxTokens?: number;
|
|
108
|
+
/**
|
|
109
|
+
* What an Anthropic provider does with the thinking blocks of an answer that calls tools. `drop` (the default) never
|
|
110
|
+
* sends them back: the next request carries the turn without its reasoning, which the API accepts. `replay` keeps
|
|
111
|
+
* each such turn in this process and sends it back exactly as it was received while the conversation before it is
|
|
112
|
+
* unchanged, so the model keeps its reasoning across a tool loop whatever the client above forwards.
|
|
113
|
+
*/
|
|
114
|
+
readonly thinking?: "drop" | "replay";
|
|
72
115
|
readonly budget?: ProviderBudget;
|
|
73
116
|
readonly keys: readonly KeyConfig[];
|
|
74
117
|
readonly models: readonly ModelConfig[];
|
|
@@ -189,6 +232,8 @@ export interface PoolConfig<Profiles extends string = string> {
|
|
|
189
232
|
export interface ContentPart {
|
|
190
233
|
readonly type: string;
|
|
191
234
|
readonly text?: string;
|
|
235
|
+
/** Marks a prompt-cache breakpoint after this part. */
|
|
236
|
+
readonly cache_control?: CacheControl;
|
|
192
237
|
readonly [key: string]: unknown;
|
|
193
238
|
}
|
|
194
239
|
/** One tool call an assistant message carries. */
|
|
@@ -207,6 +252,8 @@ export interface ChatMessage {
|
|
|
207
252
|
readonly name?: string;
|
|
208
253
|
readonly tool_calls?: readonly ToolCall[];
|
|
209
254
|
readonly tool_call_id?: string;
|
|
255
|
+
/** Marks a prompt-cache breakpoint after this message's last part. */
|
|
256
|
+
readonly cache_control?: CacheControl;
|
|
210
257
|
}
|
|
211
258
|
/** An OpenAI chat-completions request; unknown fields pass through to an OpenAI-compatible provider. */
|
|
212
259
|
export interface ChatCompletionRequest {
|
|
@@ -230,13 +277,36 @@ export interface ChatCompletionRequest {
|
|
|
230
277
|
readonly stream_options?: {
|
|
231
278
|
readonly include_usage?: boolean;
|
|
232
279
|
};
|
|
280
|
+
/** Automatic prompt caching: the provider places one breakpoint on the last cacheable block. */
|
|
281
|
+
readonly cache_control?: CacheControl;
|
|
282
|
+
readonly [key: string]: unknown;
|
|
283
|
+
}
|
|
284
|
+
/** How the prompt tokens split between the provider's prompt cache and full-price input. */
|
|
285
|
+
export interface PromptTokensDetails {
|
|
286
|
+
/** Tokens read from the cache. */
|
|
287
|
+
readonly cached_tokens?: number;
|
|
288
|
+
/** Tokens written to the cache, every lifetime together. */
|
|
289
|
+
readonly cache_write_tokens?: number;
|
|
290
|
+
/** The part of `cache_write_tokens` written to a one-hour cache. */
|
|
291
|
+
readonly cache_write_1h_tokens?: number;
|
|
292
|
+
readonly [key: string]: unknown;
|
|
293
|
+
}
|
|
294
|
+
/** How the completion tokens split. */
|
|
295
|
+
export interface CompletionTokensDetails {
|
|
296
|
+
/** Tokens the model spent thinking; they are part of `completion_tokens`. */
|
|
297
|
+
readonly reasoning_tokens?: number;
|
|
233
298
|
readonly [key: string]: unknown;
|
|
234
299
|
}
|
|
235
|
-
/**
|
|
300
|
+
/**
|
|
301
|
+
* Token counts a provider reports. `prompt_tokens` is the whole prompt: cache reads and writes are parts of it, listed
|
|
302
|
+
* in `prompt_tokens_details`, whatever the provider's own usage shape.
|
|
303
|
+
*/
|
|
236
304
|
export interface Usage {
|
|
237
305
|
readonly prompt_tokens: number;
|
|
238
306
|
readonly completion_tokens?: number;
|
|
239
307
|
readonly total_tokens: number;
|
|
308
|
+
readonly prompt_tokens_details?: PromptTokensDetails;
|
|
309
|
+
readonly completion_tokens_details?: CompletionTokensDetails;
|
|
240
310
|
readonly [key: string]: unknown;
|
|
241
311
|
}
|
|
242
312
|
/** Where a successful call went and what it cost. */
|
|
@@ -337,6 +407,39 @@ export interface EmbeddingsBody {
|
|
|
337
407
|
export interface EmbeddingsResponse extends EmbeddingsBody {
|
|
338
408
|
readonly routing: RoutingInfo;
|
|
339
409
|
}
|
|
410
|
+
/**
|
|
411
|
+
* A request in the Anthropic Messages shape, for a model an Anthropic provider serves. Every field passes to the
|
|
412
|
+
* provider as it is (cache breakpoints, thinking, tools); `model` names a pool model or is left to the policy.
|
|
413
|
+
*/
|
|
414
|
+
export interface AnthropicMessagesRequest {
|
|
415
|
+
readonly model?: string;
|
|
416
|
+
readonly messages: readonly unknown[];
|
|
417
|
+
readonly max_tokens?: number;
|
|
418
|
+
readonly system?: unknown;
|
|
419
|
+
readonly tools?: readonly unknown[];
|
|
420
|
+
readonly [key: string]: unknown;
|
|
421
|
+
}
|
|
422
|
+
/** An Anthropic message as the provider answered it, with the usage the router read from it and the routing. */
|
|
423
|
+
export interface AnthropicMessageResult {
|
|
424
|
+
/** The provider's answer, untouched. */
|
|
425
|
+
readonly message: Readonly<Record<string, unknown>>;
|
|
426
|
+
/** Its usage in the router's shape. */
|
|
427
|
+
readonly usage?: Usage;
|
|
428
|
+
readonly routing: RoutingInfo;
|
|
429
|
+
}
|
|
430
|
+
/** One server-sent event of an Anthropic stream as the provider sent it, and the lane that serves the stream. */
|
|
431
|
+
export interface AnthropicStreamEvent {
|
|
432
|
+
/** The event's type (`message_start`, `content_block_delta`, ...). */
|
|
433
|
+
readonly event: string;
|
|
434
|
+
/** Its JSON data, untouched. */
|
|
435
|
+
readonly data: string;
|
|
436
|
+
readonly provider: string;
|
|
437
|
+
readonly model: string;
|
|
438
|
+
}
|
|
439
|
+
/** A stream of Anthropic events plus the id that names the call in telemetry. */
|
|
440
|
+
export interface AnthropicStream extends AsyncIterable<AnthropicStreamEvent> {
|
|
441
|
+
readonly requestId: string;
|
|
442
|
+
}
|
|
340
443
|
/** The per-call options of `chat`, `stream` and `embeddings`. */
|
|
341
444
|
export interface CallOptions<Profiles extends string = string> {
|
|
342
445
|
readonly profile?: Profiles;
|
|
@@ -476,6 +579,12 @@ export interface LaneStats {
|
|
|
476
579
|
readonly failures: Readonly<Partial<Record<ErrorClass, number>>>;
|
|
477
580
|
readonly promptTokens: number;
|
|
478
581
|
readonly completionTokens: number;
|
|
582
|
+
/** The part of `promptTokens` read from the provider's prompt cache. */
|
|
583
|
+
readonly cachedTokens: number;
|
|
584
|
+
/** The part of `promptTokens` written to the provider's prompt cache. */
|
|
585
|
+
readonly cacheWriteTokens: number;
|
|
586
|
+
/** The part of `completionTokens` the model spent thinking. */
|
|
587
|
+
readonly reasoningTokens: number;
|
|
479
588
|
readonly cost: number;
|
|
480
589
|
}
|
|
481
590
|
/** Counters of this process since the router was created. */
|
|
@@ -499,6 +608,10 @@ export interface AiRouter<Profiles extends string = string> {
|
|
|
499
608
|
stream(request: ChatCompletionRequest, options?: CallOptions<Profiles>): ChatStream;
|
|
500
609
|
/** Sends an embeddings request through the pool. */
|
|
501
610
|
embeddings(request: EmbeddingsRequest, options?: CallOptions<Profiles>): Promise<EmbeddingsResponse>;
|
|
611
|
+
/** Sends a request in the Anthropic Messages shape, untranslated, through the pool's Anthropic providers only. */
|
|
612
|
+
messages(request: AnthropicMessagesRequest, options?: CallOptions<Profiles>): Promise<AnthropicMessageResult>;
|
|
613
|
+
/** Streams an Anthropic Messages request as the provider's own events; fails over only before the first content. */
|
|
614
|
+
messagesStream(request: AnthropicMessagesRequest, options?: CallOptions<Profiles>): AnthropicStream;
|
|
502
615
|
/** Reads every breaker, blacklisted key, headroom and budget from the store. */
|
|
503
616
|
health(): Promise<HealthSnapshot>;
|
|
504
617
|
/** Returns this process's counters. */
|
|
@@ -509,15 +622,27 @@ export interface AiRouter<Profiles extends string = string> {
|
|
|
509
622
|
reset(target?: ResetTarget): Promise<void>;
|
|
510
623
|
/** Lists the pool's profiles and models by name, with no key: what a caller may ask for. */
|
|
511
624
|
catalog(): RouterCatalog;
|
|
625
|
+
/** The chat models a caller may choose, one per provider serving them: only providers in the pool, which has the ones with keys. */
|
|
626
|
+
availableModels(): AvailableModel[];
|
|
512
627
|
}
|
|
513
628
|
/** One model of the pool as the catalog lists it: one entry per configured id, merged across the providers serving it. */
|
|
514
629
|
export interface CatalogModel {
|
|
515
630
|
readonly id: string;
|
|
516
631
|
readonly aliases: readonly string[];
|
|
517
632
|
readonly providers: readonly string[];
|
|
633
|
+
/** The wire protocols of those providers, each once, in provider order: `openai`, `anthropic`. */
|
|
634
|
+
readonly protocols: readonly ("openai" | "anthropic")[];
|
|
518
635
|
readonly capabilities: readonly Capability[];
|
|
519
636
|
readonly contextWindow?: number;
|
|
520
637
|
readonly cost?: ModelCost;
|
|
638
|
+
/** The vector length of an embeddings model, when it declares one. */
|
|
639
|
+
readonly dimensions?: number;
|
|
640
|
+
}
|
|
641
|
+
/** One provider as the catalog lists it: its id, its kind and the wire protocol it speaks. No key. */
|
|
642
|
+
export interface CatalogProvider {
|
|
643
|
+
readonly id: string;
|
|
644
|
+
readonly kind: ProviderKind;
|
|
645
|
+
readonly protocol: "openai" | "anthropic";
|
|
521
646
|
}
|
|
522
647
|
/** One named profile as the catalog lists it. */
|
|
523
648
|
export interface CatalogProfile {
|
|
@@ -530,6 +655,10 @@ export interface CatalogProfile {
|
|
|
530
655
|
readonly pinned: boolean;
|
|
531
656
|
/** The operations its first model serves: `chat`, `embeddings` or both. */
|
|
532
657
|
readonly serves: readonly ("chat" | "embeddings")[];
|
|
658
|
+
/** The wire protocols of its candidates' providers, each once, in candidate order: the first is its first candidate's. */
|
|
659
|
+
readonly protocols: readonly ("openai" | "anthropic")[];
|
|
660
|
+
/** The vector length of its first model, when that model declares one. */
|
|
661
|
+
readonly dimensions?: number;
|
|
533
662
|
}
|
|
534
663
|
/** The names a caller may route by: the defaults' fallbacks, every profile and every model. */
|
|
535
664
|
export interface RouterCatalog {
|
|
@@ -537,6 +666,7 @@ export interface RouterCatalog {
|
|
|
537
666
|
readonly model?: string;
|
|
538
667
|
readonly fallbacks: readonly string[];
|
|
539
668
|
};
|
|
669
|
+
readonly providers: readonly CatalogProvider[];
|
|
540
670
|
readonly profiles: readonly CatalogProfile[];
|
|
541
671
|
readonly models: readonly CatalogModel[];
|
|
542
672
|
}
|
package/dist/wire/anthropic.d.ts
CHANGED
|
@@ -1,21 +1,31 @@
|
|
|
1
1
|
import type { ProviderConfig } from "../types.js";
|
|
2
2
|
import type { Wire } from "./wire.js";
|
|
3
3
|
/**
|
|
4
|
-
* The Anthropic Messages API (`POST /v1/messages
|
|
5
|
-
* /build-with-claude/streaming
|
|
6
|
-
*
|
|
4
|
+
* The Anthropic Messages API (`POST /v1/messages`) behind the OpenAI chat-completions shape. Read at
|
|
5
|
+
* platform.claude.com/docs/en: /api/messages and /build-with-claude/streaming on 2026-10-31; /build-with-claude/
|
|
6
|
+
* prompt-caching, /thinking, /thinking-tool-workflows and /preserved-thinking on 2026-11-10. docs/design.md, "The
|
|
7
|
+
* Anthropic adapter", maps every field; what has no counterpart is refused (`n` above 1) or dropped (seed, penalties,
|
|
8
|
+
* logprobs).
|
|
7
9
|
*/
|
|
8
10
|
/** The `anthropic-version` sent unless the provider's headers set one. */
|
|
9
11
|
export declare const ANTHROPIC_VERSION = "2023-06-01";
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* The `max_tokens` sent when neither the request nor the provider's `defaultMaxTokens` gives one: Anthropic requires it,
|
|
14
|
+
* and on current models thinking counts toward it, so a low cap cuts an answer before its first visible token.
|
|
15
|
+
*/
|
|
16
|
+
export declare const ANTHROPIC_DEFAULT_MAX_TOKENS = 16000;
|
|
12
17
|
/** Builds the Anthropic Messages body of an OpenAI chat request. */
|
|
13
18
|
export declare function toAnthropicRequest(body: Record<string, unknown>, model: string, stream: boolean, provider: ProviderConfig): Record<string, unknown>;
|
|
14
19
|
/** Anthropic's stop reason as OpenAI's finish reason. */
|
|
15
20
|
export declare function finishReason(reason: unknown): string | null;
|
|
16
|
-
/**
|
|
17
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Anthropic usage as OpenAI usage. Anthropic lists the uncached input, the cache reads and the cache writes apart; the
|
|
23
|
+
* prompt here is their sum, with the reads and writes named in `prompt_tokens_details` (`cached_tokens`,
|
|
24
|
+
* `cache_write_tokens`, and `cache_write_1h_tokens` when the answer splits writes by lifetime). Thinking tokens, when
|
|
25
|
+
* the answer counts them, are `completion_tokens_details.reasoning_tokens`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function usageOf(usage: unknown): Record<string, unknown> | undefined;
|
|
18
28
|
/** An Anthropic message as an OpenAI `chat.completion`. */
|
|
19
|
-
export declare function fromAnthropicMessage(json: Record<string, unknown
|
|
29
|
+
export declare function fromAnthropicMessage(json: Record<string, unknown>, context?: unknown): Record<string, unknown>;
|
|
20
30
|
/** The Anthropic Messages API behind the OpenAI shape: `x-api-key` auth, `/messages`, and its SSE events as OpenAI chunks. */
|
|
21
31
|
export declare const anthropicWire: Wire;
|