@providerkit/core 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -5
- package/THIRD_PARTY_NOTICES +144 -0
- package/dist/auth/chatgpt-models.js.map +1 -1
- package/dist/auth/oauth.d.ts.map +1 -1
- package/dist/auth/oauth.js.map +1 -1
- package/dist/capability.js.map +1 -1
- package/dist/context.d.ts +4 -3
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +19 -5
- package/dist/context.js.map +1 -1
- package/dist/decisions.js.map +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/fallback.d.ts +3 -0
- package/dist/fallback.d.ts.map +1 -1
- package/dist/fallback.js +9 -5
- package/dist/fallback.js.map +1 -1
- package/dist/gate.js.map +1 -1
- package/dist/hosted/anthropic.d.ts +30 -0
- package/dist/hosted/anthropic.d.ts.map +1 -0
- package/dist/hosted/anthropic.js +465 -0
- package/dist/hosted/anthropic.js.map +1 -0
- package/dist/hosted/gemini.d.ts +22 -0
- package/dist/hosted/gemini.d.ts.map +1 -0
- package/dist/hosted/gemini.js +440 -0
- package/dist/hosted/gemini.js.map +1 -0
- package/dist/hosted/openai.d.ts +33 -0
- package/dist/hosted/openai.d.ts.map +1 -0
- package/dist/hosted/openai.js +107 -0
- package/dist/hosted/openai.js.map +1 -0
- package/dist/hosted/responses.d.ts +28 -0
- package/dist/hosted/responses.d.ts.map +1 -0
- package/dist/hosted/responses.js +358 -0
- package/dist/hosted/responses.js.map +1 -0
- package/dist/hosted/shared.d.ts +107 -0
- package/dist/hosted/shared.d.ts.map +1 -0
- package/dist/hosted/shared.js +264 -0
- package/dist/hosted/shared.js.map +1 -0
- package/dist/hosted/types.d.ts +191 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/hosted/types.js +10 -0
- package/dist/hosted/types.js.map +1 -0
- package/dist/hosted.d.ts +33 -0
- package/dist/hosted.d.ts.map +1 -0
- package/dist/hosted.js +61 -0
- package/dist/hosted.js.map +1 -0
- package/dist/index.d.ts +8 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -7
- package/dist/index.js.map +1 -1
- package/dist/key-pool.d.ts.map +1 -1
- package/dist/key-pool.js.map +1 -1
- package/dist/otel.d.ts +26 -0
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +263 -0
- package/dist/otel.js.map +1 -0
- package/dist/presets.d.ts +2 -2
- package/dist/presets.d.ts.map +1 -1
- package/dist/providers/anthropic.d.ts +75 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +207 -135
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/factory.d.ts +10 -0
- package/dist/providers/factory.d.ts.map +1 -1
- package/dist/providers/factory.js +31 -23
- package/dist/providers/factory.js.map +1 -1
- package/dist/providers/gemini.d.ts +123 -1
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +114 -54
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/hosted-codecs.d.ts +10 -0
- package/dist/providers/hosted-codecs.d.ts.map +1 -0
- package/dist/providers/hosted-codecs.js +8 -0
- package/dist/providers/hosted-codecs.js.map +1 -0
- package/dist/providers/openai.d.ts +69 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +38 -14
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/responses.d.ts +123 -0
- package/dist/providers/responses.d.ts.map +1 -1
- package/dist/providers/responses.js +57 -22
- package/dist/providers/responses.js.map +1 -1
- package/dist/providers/speech-factory.js.map +1 -1
- package/dist/registry.d.ts +3 -0
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +8 -4
- package/dist/registry.js.map +1 -1
- package/dist/retry.js.map +1 -1
- package/dist/speech-core.d.ts.map +1 -1
- package/dist/speech-core.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js.map +1 -1
- package/dist/transcription-core.d.ts.map +1 -1
- package/dist/transcription-core.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/dist/turn.d.ts +10 -1
- package/dist/turn.d.ts.map +1 -1
- package/dist/turn.js +53 -1
- package/dist/turn.js.map +1 -1
- package/dist/types.d.ts +51 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/usage.d.ts +10 -1
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +40 -2
- package/dist/usage.js.map +1 -1
- package/dist/watchdog.d.ts +6 -0
- package/dist/watchdog.d.ts.map +1 -1
- package/dist/watchdog.js +10 -2
- package/dist/watchdog.js.map +1 -1
- package/package.json +19 -5
- package/src/context.ts +21 -5
- package/src/fallback.ts +21 -6
- package/src/hosted/anthropic.ts +512 -0
- package/src/hosted/gemini.ts +510 -0
- package/src/hosted/openai.ts +137 -0
- package/src/hosted/responses.ts +420 -0
- package/src/hosted/shared.ts +341 -0
- package/src/hosted/types.ts +214 -0
- package/src/hosted.ts +101 -0
- package/src/index.ts +64 -7
- package/src/otel.ts +306 -0
- package/src/providers/anthropic.ts +252 -143
- package/src/providers/factory.ts +127 -82
- package/src/providers/gemini.ts +184 -60
- package/src/providers/hosted-codecs.ts +29 -0
- package/src/providers/openai.ts +52 -15
- package/src/providers/responses.ts +93 -24
- package/src/registry.ts +14 -4
- package/src/turn.ts +55 -1
- package/src/types.ts +62 -1
- package/src/usage.ts +32 -2
- package/src/watchdog.ts +12 -2
- package/dist/jev.d.ts +0 -123
- package/dist/jev.d.ts.map +0 -1
- package/dist/jev.js +0 -278
- package/dist/jev.js.map +0 -1
|
@@ -10,6 +10,11 @@
|
|
|
10
10
|
// The event names arrive on the SSE `event:` line and are repeated inside each
|
|
11
11
|
// payload's own `type`. The transport yields only `data:` payloads, so this
|
|
12
12
|
// adapter reads `type` — which is what survives, and what gateways agree on.
|
|
13
|
+
//
|
|
14
|
+
// Hosted tool shapes, include values and item fields follow Vercel's AI SDK
|
|
15
|
+
// (Apache-2.0, Copyright 2023 Vercel, Inc.: openai-responses-prepare-tools.ts,
|
|
16
|
+
// openai-responses-language-model.ts, map-openai-responses-annotation.ts); the
|
|
17
|
+
// marker-span check follows PydanticAI (MIT). See THIRD_PARTY_NOTICES.
|
|
13
18
|
import { repairHistory } from "../context.ts";
|
|
14
19
|
import {
|
|
15
20
|
fileRefused,
|
|
@@ -18,6 +23,8 @@ import {
|
|
|
18
23
|
streamError,
|
|
19
24
|
validateMaxAttempts,
|
|
20
25
|
} from "../errors.ts";
|
|
26
|
+
import type { HostedToolType } from "../hosted/types.ts";
|
|
27
|
+
import { refuseHostedTools, type HostedCodecs } from "./hosted-codecs.ts";
|
|
21
28
|
import { streamSse, apiUrl, parseFrame, type FetchLike } from "../transport.ts";
|
|
22
29
|
import { parseRateLimitReset } from "../rate-limit.ts";
|
|
23
30
|
import { settleToolCalls } from "../tool-args.ts";
|
|
@@ -35,7 +42,7 @@ import type {
|
|
|
35
42
|
} from "../types.ts";
|
|
36
43
|
import { toDataUri } from "../types.ts";
|
|
37
44
|
import { isStrictSchema, withoutPatterns } from "../schema.ts";
|
|
38
|
-
import {
|
|
45
|
+
import { configuredFallbacks, type ProviderFallbackConfig } from "../fallback.ts";
|
|
39
46
|
import { gateImages, type ImageLimits } from "../image.ts";
|
|
40
47
|
|
|
41
48
|
export interface ResponsesConfig extends ProviderFallbackConfig {
|
|
@@ -83,6 +90,14 @@ export interface ResponsesConfig extends ProviderFallbackConfig {
|
|
|
83
90
|
replayReasoning?: boolean;
|
|
84
91
|
/** Default `service_tier` for every call; `StreamOptions.serviceTier` wins. */
|
|
85
92
|
serviceTier?: ServiceTier;
|
|
93
|
+
/**
|
|
94
|
+
* For providers imported from `@providerkit/core/hosted`.
|
|
95
|
+
* The hosted tools this endpoint runs. Defaults to web search, code
|
|
96
|
+
* execution, file search and image generation on api.openai.com, and to
|
|
97
|
+
* none anywhere else, the ChatGPT backend included, until someone checks
|
|
98
|
+
* that endpoint. One that ignores a tool answers unsearched without saying so.
|
|
99
|
+
*/
|
|
100
|
+
servesHostedTools?: readonly HostedToolType[];
|
|
86
101
|
}
|
|
87
102
|
|
|
88
103
|
const DEFAULT_BASE_URL = "https://api.openai.com";
|
|
@@ -275,7 +290,7 @@ interface ResponsesUsage {
|
|
|
275
290
|
output_tokens_details?: { reasoning_tokens?: number };
|
|
276
291
|
}
|
|
277
292
|
|
|
278
|
-
interface ResponsesItem {
|
|
293
|
+
export interface ResponsesItem {
|
|
279
294
|
type?: string;
|
|
280
295
|
/** The output item's own id (`fc_…`), which the argument deltas reference. */
|
|
281
296
|
id?: string;
|
|
@@ -284,12 +299,36 @@ interface ResponsesItem {
|
|
|
284
299
|
name?: string;
|
|
285
300
|
arguments?: string;
|
|
286
301
|
encrypted_content?: string;
|
|
302
|
+
/** A hosted item's state: `in_progress`, `searching`, `completed`, `failed`… */
|
|
303
|
+
status?: string;
|
|
304
|
+
error?: { code?: string; message?: string } | null;
|
|
305
|
+
/** `web_search_call`: what the search did, sources included when asked for. */
|
|
306
|
+
action?: {
|
|
307
|
+
type?: string;
|
|
308
|
+
query?: string | null;
|
|
309
|
+
queries?: string[] | null;
|
|
310
|
+
url?: string | null;
|
|
311
|
+
sources?: { type?: string; url?: string }[] | null;
|
|
312
|
+
} | null;
|
|
313
|
+
/** `code_interpreter_call` */
|
|
314
|
+
code?: string | null;
|
|
315
|
+
container_id?: string;
|
|
316
|
+
outputs?: { type?: string; logs?: string; url?: string }[] | null;
|
|
317
|
+
/** `file_search_call` */
|
|
318
|
+
queries?: string[] | null;
|
|
319
|
+
results?: { file_id?: string; filename?: string; text?: string }[] | null;
|
|
320
|
+
/** `image_generation_call`: the image, as base64. */
|
|
321
|
+
result?: string | null;
|
|
322
|
+
revised_prompt?: string | null;
|
|
287
323
|
}
|
|
288
324
|
|
|
289
|
-
interface ResponsesEvent {
|
|
325
|
+
export interface ResponsesEvent {
|
|
290
326
|
type?: string;
|
|
291
327
|
delta?: string;
|
|
292
328
|
item_id?: string;
|
|
329
|
+
/** Which part of a message item a text delta or annotation belongs to. */
|
|
330
|
+
content_index?: number;
|
|
331
|
+
annotation?: import("../hosted/responses.ts").ResponsesAnnotation;
|
|
293
332
|
/** xAI keys argument events by the CALL id instead of the item id. */
|
|
294
333
|
call_id?: string;
|
|
295
334
|
/** The whole argument string, on `function_call_arguments.done`. */
|
|
@@ -373,7 +412,12 @@ function codexHeaders(limits: CodexRateLimits): Headers {
|
|
|
373
412
|
return headers;
|
|
374
413
|
}
|
|
375
414
|
|
|
376
|
-
|
|
415
|
+
/** The usage record, with the hosted calls the stream showed: this shape bills
|
|
416
|
+
* them per call and states no count of its own. */
|
|
417
|
+
function usageChunk(
|
|
418
|
+
usage: ResponsesUsage,
|
|
419
|
+
hostedCalls?: Partial<Record<HostedToolType, number>>,
|
|
420
|
+
): ProviderChunk {
|
|
377
421
|
return {
|
|
378
422
|
type: "usage",
|
|
379
423
|
usage: {
|
|
@@ -386,6 +430,7 @@ function usageChunk(usage: ResponsesUsage): ProviderChunk {
|
|
|
386
430
|
...(usage.output_tokens_details?.reasoning_tokens !== undefined
|
|
387
431
|
? { reasoningTokens: usage.output_tokens_details.reasoning_tokens }
|
|
388
432
|
: {}),
|
|
433
|
+
...(hostedCalls ? { hostedCalls } : {}),
|
|
389
434
|
},
|
|
390
435
|
};
|
|
391
436
|
}
|
|
@@ -410,6 +455,11 @@ function unsentTail(snapshot: string, streamed: string): string {
|
|
|
410
455
|
}
|
|
411
456
|
|
|
412
457
|
export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
458
|
+
return responsesProvider(config);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** @internal Shared by the plain and hosted entry points. */
|
|
462
|
+
export function responsesProvider(config: ResponsesConfig, codecs?: HostedCodecs): Provider {
|
|
413
463
|
const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
|
|
414
464
|
const id = config.id ?? "openai-responses";
|
|
415
465
|
const instanceSessionId = config.sessionHeader
|
|
@@ -426,6 +476,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
426
476
|
opts: StreamOptions = {},
|
|
427
477
|
): AsyncIterable<ProviderChunk> {
|
|
428
478
|
validateMaxAttempts(opts.maxAttempts);
|
|
479
|
+
if (!codecs) refuseHostedTools(id, opts, messages);
|
|
429
480
|
const effort = opts.effort ?? config.effort;
|
|
430
481
|
const model = opts.model ?? config.model;
|
|
431
482
|
const museContributor = isMuseSparkContributor(model);
|
|
@@ -448,7 +499,16 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
448
499
|
store: false,
|
|
449
500
|
};
|
|
450
501
|
if (instructions) request.instructions = instructions;
|
|
451
|
-
|
|
502
|
+
// Hosted items are never sent back: `store` is off, so the backend has
|
|
503
|
+
// no copy to reference, and their results ride in the answer instead.
|
|
504
|
+
const hosted = codecs?.responses.plan(config, baseUrl, id, opts);
|
|
505
|
+
// Merged, not overwritten: the reasoning items and each hosted tool's
|
|
506
|
+
// results are asked for in the same list.
|
|
507
|
+
const include = [
|
|
508
|
+
...(config.replayReasoning ? ["reasoning.encrypted_content"] : []),
|
|
509
|
+
...(hosted?.include ?? []),
|
|
510
|
+
];
|
|
511
|
+
if (include.length > 0) request.include = include;
|
|
452
512
|
const maxTokens = opts.maxTokens ?? config.maxTokens;
|
|
453
513
|
if (maxTokens !== undefined) request.max_output_tokens = maxTokens;
|
|
454
514
|
if (opts.temperature !== undefined) request.temperature = opts.temperature;
|
|
@@ -465,22 +525,27 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
465
525
|
request.reasoning =
|
|
466
526
|
level === "none" ? { effort: "none" } : { effort: level, summary: "auto" };
|
|
467
527
|
}
|
|
468
|
-
if (requestTools.length > 0) {
|
|
528
|
+
if (requestTools.length > 0 || (hosted?.tools.length ?? 0) > 0) {
|
|
469
529
|
// Flat here — no nested `function` envelope, unlike chat/completions.
|
|
470
|
-
request.tools =
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
530
|
+
request.tools = [
|
|
531
|
+
...requestTools.map((tool) => ({
|
|
532
|
+
type: "function",
|
|
533
|
+
name: tool.name,
|
|
534
|
+
description: tool.description,
|
|
535
|
+
parameters: withoutPatterns(tool.inputSchema),
|
|
536
|
+
// Said either way, never left to the default. Strict demands every
|
|
537
|
+
// property required and every object closed; on a schema that is not
|
|
538
|
+
// already shaped that way the backend makes optional arguments
|
|
539
|
+
// mandatory, and the model starts filling in parameters nobody asked
|
|
540
|
+
// for (cc-proxy, 0.1.15). Where the schema already qualifies, the
|
|
541
|
+
// enforcement is free.
|
|
542
|
+
strict: isStrictSchema(tool.inputSchema),
|
|
543
|
+
})),
|
|
544
|
+
...(hosted?.tools ?? []),
|
|
545
|
+
];
|
|
546
|
+
if (requestTools.length > 0 && opts.parallelToolCalls === false) {
|
|
547
|
+
request.parallel_tool_calls = false;
|
|
548
|
+
}
|
|
484
549
|
}
|
|
485
550
|
// The cache affinity key. Without one the backend spreads a
|
|
486
551
|
// conversation's turns across cache shards, and a re-sent prefix misses.
|
|
@@ -528,7 +593,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
528
593
|
reasoningItems.length
|
|
529
594
|
? [{ type: "delta", reasoningItems: { provider: id, items: reasoningItems } }]
|
|
530
595
|
: [];
|
|
531
|
-
|
|
596
|
+
const hostedStream = codecs?.responses.stream(hosted?.hosted ?? []);
|
|
532
597
|
for await (const data of streamSse({
|
|
533
598
|
url: apiUrl(baseUrl, config.path ?? DEFAULT_PATH),
|
|
534
599
|
headers: {
|
|
@@ -550,6 +615,8 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
550
615
|
produced = true;
|
|
551
616
|
}
|
|
552
617
|
|
|
618
|
+
if (hostedStream) yield* hostedStream.event(event);
|
|
619
|
+
|
|
553
620
|
switch (event.type) {
|
|
554
621
|
case "response.output_text.delta":
|
|
555
622
|
if (event.delta) yield { type: "delta", content: event.delta };
|
|
@@ -716,8 +783,9 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
716
783
|
const minted = settleToolCalls(id, calls);
|
|
717
784
|
if (minted.length > 0) yield { type: "delta", toolCalls: minted };
|
|
718
785
|
}
|
|
786
|
+
if (hostedStream) yield* hostedStream.close();
|
|
719
787
|
yield* reasoningChunk();
|
|
720
|
-
if (response?.usage) yield usageChunk(response.usage);
|
|
788
|
+
if (response?.usage) yield usageChunk(response.usage, hostedStream?.counts());
|
|
721
789
|
yield { type: "finish", finishReason };
|
|
722
790
|
return;
|
|
723
791
|
}
|
|
@@ -730,9 +798,10 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
730
798
|
// A turn that ran out of output tokens still billed for its input,
|
|
731
799
|
// and this event carries usage in the same shape as `completed`.
|
|
732
800
|
const finishReason = incompleteFinish(id, event.response?.incomplete_details?.reason);
|
|
801
|
+
if (hostedStream) yield* hostedStream.close();
|
|
733
802
|
yield* reasoningChunk();
|
|
734
803
|
const usage = event.response?.usage;
|
|
735
|
-
if (usage) yield usageChunk(usage);
|
|
804
|
+
if (usage) yield usageChunk(usage, hostedStream?.counts());
|
|
736
805
|
yield { type: "finish", finishReason };
|
|
737
806
|
return;
|
|
738
807
|
}
|
|
@@ -764,5 +833,5 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
|
|
|
764
833
|
},
|
|
765
834
|
};
|
|
766
835
|
|
|
767
|
-
return
|
|
836
|
+
return configuredFallbacks(provider, config, codecs);
|
|
768
837
|
}
|
package/src/registry.ts
CHANGED
|
@@ -19,7 +19,8 @@ import { withFallbackProviders, type Cooldown, type FallbackOptions } from "./fa
|
|
|
19
19
|
import { createGate, withGate, type Gate } from "./gate.ts";
|
|
20
20
|
import { KeyPool, withKeyPool, type KeyPoolOptions, type KeyTier } from "./key-pool.ts";
|
|
21
21
|
import { PROVIDER_PRESETS, type ProviderPreset, type ProviderPresetId } from "./presets.ts";
|
|
22
|
-
import {
|
|
22
|
+
import { buildPresetAdapter, type PresetAdapterConfig } from "./providers/factory.ts";
|
|
23
|
+
import type { HostedCodecs } from "./providers/hosted-codecs.ts";
|
|
23
24
|
import type { RateLimitWindow } from "./rate-limit.ts";
|
|
24
25
|
import type { Effort, Provider, ProviderChunk } from "./types.ts";
|
|
25
26
|
import { withWatchdog, type WatchdogOptions } from "./watchdog.ts";
|
|
@@ -130,6 +131,11 @@ interface Member {
|
|
|
130
131
|
}
|
|
131
132
|
|
|
132
133
|
export function createRegistry(options: RegistryOptions): Registry {
|
|
134
|
+
return providerRegistry(options);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** @internal The same registry, with codecs bound to every rebuilt adapter. */
|
|
138
|
+
export function providerRegistry(options: RegistryOptions, codecs?: HostedCodecs): Registry {
|
|
133
139
|
const connections = new Map<string, Connection>();
|
|
134
140
|
const chainRefs = new Map(Object.entries(options.chains ?? {}));
|
|
135
141
|
/** Built chains by ref, with the connections each one uses. */
|
|
@@ -264,15 +270,19 @@ export function createRegistry(options: RegistryOptions): Registry {
|
|
|
264
270
|
// The credential's host and headers win: Copilot gives each plan its
|
|
265
271
|
// own host, and the Codex backend refuses a call without its account id.
|
|
266
272
|
provider = withToken(config.token, (auth) =>
|
|
267
|
-
|
|
273
|
+
buildPresetAdapter(
|
|
274
|
+
preset,
|
|
275
|
+
{ ...base, ...auth, headers: { ...headers, ...auth.headers } },
|
|
276
|
+
codecs,
|
|
277
|
+
),
|
|
268
278
|
);
|
|
269
279
|
} else if (config.keys) {
|
|
270
280
|
// `open` builds the pool whenever `keys` is set.
|
|
271
281
|
provider = withKeyPool(keyPool!, (key) =>
|
|
272
|
-
|
|
282
|
+
buildPresetAdapter(preset, { ...base, headers, apiKey: key }, codecs),
|
|
273
283
|
);
|
|
274
284
|
} else {
|
|
275
|
-
provider =
|
|
285
|
+
provider = buildPresetAdapter(preset, { ...base, headers, apiKey: config.apiKey }, codecs);
|
|
276
286
|
}
|
|
277
287
|
// On unless turned off, where `createPresetProvider` adds one only when
|
|
278
288
|
// asked. Handing apps the whole composition is the point of a registry:
|
package/src/turn.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
// turn that said nothing is not judged here: that is `requireContent`'s call
|
|
10
10
|
// (invariant 13), made where the retry that can fix it still can.
|
|
11
11
|
import { ProviderError } from "./errors.ts";
|
|
12
|
+
import type { Citation, HostedCall, HostedItems, HostedToolType } from "./hosted/types.ts";
|
|
12
13
|
import {
|
|
13
14
|
EMPTY_USAGE,
|
|
14
15
|
type ChatMessage,
|
|
@@ -19,6 +20,9 @@ import {
|
|
|
19
20
|
type ToolCall,
|
|
20
21
|
} from "./types.ts";
|
|
21
22
|
|
|
23
|
+
/** A hosted call while it assembles: its type may not be stated yet. */
|
|
24
|
+
type AssemblingCall = Omit<HostedCall, "type"> & { type?: HostedToolType };
|
|
25
|
+
|
|
22
26
|
/** One drained turn: the message a loop appends, whose `toolCalls` are the
|
|
23
27
|
* calls it runs, and what the turn cost. */
|
|
24
28
|
export interface Turn {
|
|
@@ -43,8 +47,12 @@ export interface Turn {
|
|
|
43
47
|
* for (const call of turn.message.toolCalls ?? []) history.push(await run(call));
|
|
44
48
|
* ```
|
|
45
49
|
*
|
|
50
|
+
* Hosted calls (a vendor's search, fetch or code run) land in `hostedCalls`,
|
|
51
|
+
* never in `toolCalls`: there is nothing for the loop to run.
|
|
52
|
+
*
|
|
46
53
|
* @throws {ProviderError} `overload` when a tool call ends the stream with no
|
|
47
|
-
* name. Whatever the stream itself throws
|
|
54
|
+
* name, or a hosted call with no type. Whatever the stream itself throws
|
|
55
|
+
* passes through unchanged.
|
|
48
56
|
*/
|
|
49
57
|
export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise<Turn> {
|
|
50
58
|
let content = "";
|
|
@@ -56,6 +64,9 @@ export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise
|
|
|
56
64
|
let reasoningDetails: unknown[] | undefined;
|
|
57
65
|
let reasoningItems: ReasoningItems | undefined;
|
|
58
66
|
const calls = new Map<number, ToolCall>();
|
|
67
|
+
const hosted = new Map<number, AssemblingCall>();
|
|
68
|
+
const citations: Citation[] = [];
|
|
69
|
+
let hostedItems: HostedItems | undefined;
|
|
59
70
|
// Cumulative on some providers and final-only on others: the LAST one wins.
|
|
60
71
|
let usage: TokenUsage = EMPTY_USAGE;
|
|
61
72
|
let finishReason: FinishReason | null = null;
|
|
@@ -85,6 +96,27 @@ export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise
|
|
|
85
96
|
call.arguments += delta.arguments ?? "";
|
|
86
97
|
calls.set(delta.index, call);
|
|
87
98
|
}
|
|
99
|
+
for (const delta of chunk.hosted ?? []) {
|
|
100
|
+
// The same rule as a tool call's identity: the last value stated wins,
|
|
101
|
+
// so a skeleton's blanks never erase what the closing event says. A
|
|
102
|
+
// call's place in the answer is where the text was when it first
|
|
103
|
+
// appeared, unless the adapter said otherwise (Gemini's grounding record
|
|
104
|
+
// arrives after the answer and covers it from 0).
|
|
105
|
+
const call = hosted.get(delta.index) ?? { id: "", status: "running", at: content.length };
|
|
106
|
+
if (delta.id) call.id = delta.id;
|
|
107
|
+
if (delta.type) call.type = delta.type;
|
|
108
|
+
if (delta.status) call.status = delta.status;
|
|
109
|
+
if (delta.input) call.input = { ...call.input, ...delta.input };
|
|
110
|
+
if (delta.codeDelta) {
|
|
111
|
+
call.input = { ...call.input, code: (call.input?.code ?? "") + delta.codeDelta };
|
|
112
|
+
}
|
|
113
|
+
if (delta.output) call.output = delta.output;
|
|
114
|
+
if (delta.error) call.error = delta.error;
|
|
115
|
+
if (delta.at !== undefined) call.at = delta.at;
|
|
116
|
+
hosted.set(delta.index, call);
|
|
117
|
+
}
|
|
118
|
+
if (chunk.citations?.length) citations.push(...chunk.citations);
|
|
119
|
+
if (chunk.hostedItems) hostedItems = chunk.hostedItems;
|
|
88
120
|
if (chunk.usage) usage = chunk.usage;
|
|
89
121
|
if (chunk.finishReason) finishReason = chunk.finishReason;
|
|
90
122
|
}
|
|
@@ -110,6 +142,19 @@ export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise
|
|
|
110
142
|
);
|
|
111
143
|
}
|
|
112
144
|
const toolCalls = indexed.map(([, call]) => call);
|
|
145
|
+
const hostedCalls = [...hosted]
|
|
146
|
+
.sort(([a], [b]) => a - b)
|
|
147
|
+
.map(([index, { type, ...call }]): HostedCall => {
|
|
148
|
+
// A record that never says what ran can't be shown or counted: the
|
|
149
|
+
// vendor's frame was broken, as with a nameless tool call above.
|
|
150
|
+
if (type) return { ...call, type };
|
|
151
|
+
const provider = source?.provider ?? "turn";
|
|
152
|
+
throw new ProviderError(
|
|
153
|
+
provider,
|
|
154
|
+
"overload",
|
|
155
|
+
`${provider}: hosted call ${call.id || `at index ${index}`} never said which tool ran`,
|
|
156
|
+
);
|
|
157
|
+
});
|
|
113
158
|
|
|
114
159
|
return {
|
|
115
160
|
message: {
|
|
@@ -119,6 +164,9 @@ export async function collectTurn(stream: AsyncIterable<ProviderChunk>): Promise
|
|
|
119
164
|
...(reasoningDetails ? { reasoningDetails } : {}),
|
|
120
165
|
...(reasoningItems ? { reasoningItems } : {}),
|
|
121
166
|
...(toolCalls.length > 0 ? { toolCalls } : {}),
|
|
167
|
+
...(hostedCalls.length > 0 ? { hostedCalls } : {}),
|
|
168
|
+
...(citations.length > 0 ? { citations } : {}),
|
|
169
|
+
...(hostedItems ? { hostedItems } : {}),
|
|
122
170
|
},
|
|
123
171
|
usage,
|
|
124
172
|
finishReason,
|
|
@@ -139,6 +187,10 @@ export interface Completion {
|
|
|
139
187
|
* for none. Before this field a drained turn that called a tool read as a
|
|
140
188
|
* turn that said nothing. */
|
|
141
189
|
toolCalls: ToolCall[];
|
|
190
|
+
/** Present only when the vendor ran its own tools. See ChatMessage.hostedCalls. */
|
|
191
|
+
hostedCalls?: HostedCall[];
|
|
192
|
+
/** Present only when the answer cites sources. See ChatMessage.citations. */
|
|
193
|
+
citations?: Citation[];
|
|
142
194
|
usage: TokenUsage;
|
|
143
195
|
finishReason: FinishReason | null;
|
|
144
196
|
model: string;
|
|
@@ -158,6 +210,8 @@ export async function drainStream(
|
|
|
158
210
|
...(message.reasoningDetails ? { reasoningDetails: message.reasoningDetails } : {}),
|
|
159
211
|
...(message.reasoningItems ? { reasoningItems: message.reasoningItems } : {}),
|
|
160
212
|
toolCalls: message.toolCalls ?? [],
|
|
213
|
+
...(message.hostedCalls ? { hostedCalls: message.hostedCalls } : {}),
|
|
214
|
+
...(message.citations ? { citations: message.citations } : {}),
|
|
161
215
|
usage,
|
|
162
216
|
finishReason,
|
|
163
217
|
model: source?.model ?? model,
|
package/src/types.ts
CHANGED
|
@@ -6,6 +6,14 @@
|
|
|
6
6
|
// the lingua franca forces the other adapters to round-trip through a dialect
|
|
7
7
|
// that isn't theirs, and every quirk of that dialect then leaks into callers
|
|
8
8
|
// who never asked for it.
|
|
9
|
+
import type {
|
|
10
|
+
Citation,
|
|
11
|
+
HostedCall,
|
|
12
|
+
HostedCallDelta,
|
|
13
|
+
HostedItems,
|
|
14
|
+
HostedTool,
|
|
15
|
+
HostedToolType,
|
|
16
|
+
} from "./hosted/types.ts";
|
|
9
17
|
import type { RateLimitReading } from "./rate-limit.ts";
|
|
10
18
|
|
|
11
19
|
/**
|
|
@@ -46,7 +54,10 @@ export interface ImagePart {
|
|
|
46
54
|
/** A file's media type: anything but an image's, because an image is an
|
|
47
55
|
* `ImagePart`, which every adapter can send. */
|
|
48
56
|
export type FileMimeType =
|
|
49
|
-
|
|
57
|
+
| `application/${string}`
|
|
58
|
+
| `audio/${string}`
|
|
59
|
+
| `video/${string}`
|
|
60
|
+
| `text/${string}`;
|
|
50
61
|
|
|
51
62
|
/**
|
|
52
63
|
* A file the model reads (a PDF, a recording, a video) as base64 bytes,
|
|
@@ -119,6 +130,19 @@ export type ChatMessage =
|
|
|
119
130
|
*/
|
|
120
131
|
reasoningItems?: ReasoningItems;
|
|
121
132
|
toolCalls?: ToolCall[];
|
|
133
|
+
/**
|
|
134
|
+
* What the vendor's own tools did in this turn (a search, a fetch, a code
|
|
135
|
+
* run), in the order they ran. Never function calls: nothing here is
|
|
136
|
+
* run by the loop or answered with a `tool` message. Display data, with
|
|
137
|
+
* the sensitivity of `content`.
|
|
138
|
+
*/
|
|
139
|
+
hostedCalls?: HostedCall[];
|
|
140
|
+
/** The sources the answer cites, with spans into `content`. */
|
|
141
|
+
citations?: Citation[];
|
|
142
|
+
/** The vendor's own record of its hosted tools, replayed only to the
|
|
143
|
+
* provider that made it. Provider-owned like `reasoningItems`: never
|
|
144
|
+
* echo it to a browser. See {@link HostedItems}. */
|
|
145
|
+
hostedItems?: HostedItems;
|
|
122
146
|
}
|
|
123
147
|
| {
|
|
124
148
|
role: "tool";
|
|
@@ -189,6 +213,20 @@ export interface TokenUsage {
|
|
|
189
213
|
* `costUsd` and `UsageTracker` bill it over any rate.
|
|
190
214
|
*/
|
|
191
215
|
reportedCostUsd?: number;
|
|
216
|
+
/**
|
|
217
|
+
* The hosted tool calls this turn ran, by type: as the vendor states them
|
|
218
|
+
* (Anthropic, OpenRouter), or as the stream showed them. Absent when none
|
|
219
|
+
* ran. Vendors bill these per call, beside the tokens; `ModelRate.hosted`
|
|
220
|
+
* prices them.
|
|
221
|
+
*/
|
|
222
|
+
hostedCalls?: Partial<Record<HostedToolType, number>>;
|
|
223
|
+
/**
|
|
224
|
+
* The prompt of the turn's LAST model pass, cache included, where the
|
|
225
|
+
* vendor states it (Anthropic's `usage.iterations`). A hosted tool loop runs
|
|
226
|
+
* several passes inside one turn and bills their sum, so `inputTokens` can
|
|
227
|
+
* read far larger than the window ever held. Compaction reads this first.
|
|
228
|
+
*/
|
|
229
|
+
contextTokens?: number;
|
|
192
230
|
}
|
|
193
231
|
|
|
194
232
|
export const EMPTY_USAGE: TokenUsage = {
|
|
@@ -211,6 +249,13 @@ export interface ProviderChunk {
|
|
|
211
249
|
/** Once per turn, where a provider replays its reasoning. See ChatMessage.reasoningItems. */
|
|
212
250
|
reasoningItems?: ReasoningItems;
|
|
213
251
|
toolCalls?: ToolCallDelta[];
|
|
252
|
+
/** Hosted call fragments, assembled by `index`. One commits the stream:
|
|
253
|
+
* past it, a retry would rerun and rebill the vendor's tool. */
|
|
254
|
+
hosted?: HostedCallDelta[];
|
|
255
|
+
/** Citations, appended in order. Their spans index the turn's content. */
|
|
256
|
+
citations?: Citation[];
|
|
257
|
+
/** Once per turn, where a provider replays its hosted tools. See ChatMessage.hostedItems. */
|
|
258
|
+
hostedItems?: HostedItems;
|
|
214
259
|
usage?: TokenUsage;
|
|
215
260
|
finishReason?: FinishReason;
|
|
216
261
|
}
|
|
@@ -437,6 +482,22 @@ export interface StreamOptions {
|
|
|
437
482
|
* Gemini has no such switch. Overrides the provider's own setting.
|
|
438
483
|
*/
|
|
439
484
|
serviceTier?: ServiceTier;
|
|
485
|
+
/**
|
|
486
|
+
* Tools the model vendor runs inside its own turn: web search, web fetch,
|
|
487
|
+
* code execution, file search, image generation. Each adapter sends them in
|
|
488
|
+
* its dialect, or refuses one it can't send with an `invalid` error before
|
|
489
|
+
* any request, unless the tool is `optional`. `toolChoice: "none"` sends
|
|
490
|
+
* none; a forced choice with any is refused.
|
|
491
|
+
*
|
|
492
|
+
* Import the provider constructor from `@providerkit/core/hosted` first.
|
|
493
|
+
* Plain constructors refuse this option before any request, even with
|
|
494
|
+
* `optional` or `toolChoice: "none"`; they include no hosted codecs.
|
|
495
|
+
*
|
|
496
|
+
* A custom Provider that can't run them must refuse them the same way
|
|
497
|
+
* (`planHostedTools` from `@providerkit/core/hosted` does it): ignored, they
|
|
498
|
+
* become an answer that never searched, with nothing to say so.
|
|
499
|
+
*/
|
|
500
|
+
hostedTools?: readonly HostedTool[];
|
|
440
501
|
}
|
|
441
502
|
|
|
442
503
|
// ponytail: Codex's `ultrafast` tier (gpt-6-astra only) is the next value to add.
|
package/src/usage.ts
CHANGED
|
@@ -12,9 +12,22 @@
|
|
|
12
12
|
// go stale, and where one model id is served by several hosts at different
|
|
13
13
|
// prices it is the only figure that names the host that answered. When it is
|
|
14
14
|
// present it wins; the caller's rate prices everything else.
|
|
15
|
+
import { HOSTED_TOOL_TYPES, type HostedToolType } from "./hosted/types.ts";
|
|
15
16
|
import type { TokenUsage } from "./types.ts";
|
|
16
17
|
import { EMPTY_USAGE } from "./types.ts";
|
|
17
18
|
|
|
19
|
+
type HostedCounts = Partial<Record<HostedToolType, number>>;
|
|
20
|
+
|
|
21
|
+
/** Two count records combined type by type, each clamped at zero. */
|
|
22
|
+
function combineCounts(a: HostedCounts, b: HostedCounts | undefined, sign: 1 | -1): HostedCounts {
|
|
23
|
+
const out: HostedCounts = { ...a };
|
|
24
|
+
for (const type of HOSTED_TOOL_TYPES) {
|
|
25
|
+
const count = b?.[type];
|
|
26
|
+
if (count !== undefined) out[type] = Math.max(0, (out[type] ?? 0) + sign * count);
|
|
27
|
+
}
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
|
|
18
31
|
/** USD per million tokens, as every vendor prints its price sheet, so a row
|
|
19
32
|
* can be checked against one without arithmetic. */
|
|
20
33
|
export interface ModelRate {
|
|
@@ -30,12 +43,17 @@ export interface ModelRate {
|
|
|
30
43
|
* (1.25×); the OpenAI-shape auto-cachers bill it at the input rate.
|
|
31
44
|
* Defaults to `input` when a vendor does not price it separately. */
|
|
32
45
|
cacheWrite?: number;
|
|
46
|
+
/** USD per hosted tool call (a search, a fetch, a code run), by type, as
|
|
47
|
+
* the vendor's price sheet states it. Billed beside the tokens; a type
|
|
48
|
+
* left out bills nothing. */
|
|
49
|
+
hosted?: Partial<Record<HostedToolType, number>>;
|
|
33
50
|
}
|
|
34
51
|
|
|
35
52
|
/** Add two usage records. The sum carries no `reportedCostUsd`: a call
|
|
36
53
|
* priced by its provider plus one priced by a rate is not one bill, and
|
|
37
54
|
* `costUsd` would price the whole sum at the one figure. Accumulate cost in
|
|
38
|
-
* a `UsageTracker`, which prices each call as it arrives.
|
|
55
|
+
* a `UsageTracker`, which prices each call as it arrives. Nor
|
|
56
|
+
* `contextTokens`: two calls' contexts do not add up to a window. */
|
|
39
57
|
export function addUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
|
|
40
58
|
return {
|
|
41
59
|
inputTokens: a.inputTokens + b.inputTokens,
|
|
@@ -45,6 +63,9 @@ export function addUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
|
|
|
45
63
|
...(a.reasoningTokens !== undefined || b.reasoningTokens !== undefined
|
|
46
64
|
? { reasoningTokens: (a.reasoningTokens ?? 0) + (b.reasoningTokens ?? 0) }
|
|
47
65
|
: {}),
|
|
66
|
+
...(a.hostedCalls || b.hostedCalls
|
|
67
|
+
? { hostedCalls: combineCounts(a.hostedCalls ?? {}, b.hostedCalls, 1) }
|
|
68
|
+
: {}),
|
|
48
69
|
};
|
|
49
70
|
}
|
|
50
71
|
|
|
@@ -61,6 +82,7 @@ export function subtractUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
|
|
|
61
82
|
...(a.reasoningTokens !== undefined
|
|
62
83
|
? { reasoningTokens: Math.max(0, a.reasoningTokens - (b.reasoningTokens ?? 0)) }
|
|
63
84
|
: {}),
|
|
85
|
+
...(a.hostedCalls ? { hostedCalls: combineCounts(a.hostedCalls, b.hostedCalls, -1) } : {}),
|
|
64
86
|
};
|
|
65
87
|
}
|
|
66
88
|
|
|
@@ -75,6 +97,9 @@ export function subtractUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
|
|
|
75
97
|
* part at the write rate. `cached` is clamped to `input` so an over-reporting
|
|
76
98
|
* provider can neither drive the miss count negative nor bill for more prompt
|
|
77
99
|
* than it was sent.
|
|
100
|
+
*
|
|
101
|
+
* Hosted tool calls bill per call at `rate.hosted`. A reported cost already
|
|
102
|
+
* includes them (OpenRouter's search fee is inside its `cost`).
|
|
78
103
|
*/
|
|
79
104
|
export function costUsd(usage: TokenUsage, rate: ModelRate): number {
|
|
80
105
|
if (usage.reportedCostUsd !== undefined) return usage.reportedCostUsd;
|
|
@@ -85,12 +110,17 @@ export function costUsd(usage: TokenUsage, rate: ModelRate): number {
|
|
|
85
110
|
// once at the input rate, again at the write rate: 2.25× on Claude.
|
|
86
111
|
const miss = Math.max(0, input - cached - written);
|
|
87
112
|
const writeRate = rate.cacheWrite ?? rate.input;
|
|
113
|
+
let hosted = 0;
|
|
114
|
+
for (const type of HOSTED_TOOL_TYPES) {
|
|
115
|
+
hosted += Math.max(0, usage.hostedCalls?.[type] ?? 0) * (rate.hosted?.[type] ?? 0);
|
|
116
|
+
}
|
|
88
117
|
return (
|
|
89
118
|
(miss * rate.input +
|
|
90
119
|
cached * rate.cacheRead +
|
|
91
120
|
written * writeRate +
|
|
92
121
|
Math.max(0, usage.outputTokens) * rate.output) /
|
|
93
|
-
|
|
122
|
+
1_000_000 +
|
|
123
|
+
hosted
|
|
94
124
|
);
|
|
95
125
|
}
|
|
96
126
|
|
package/src/watchdog.ts
CHANGED
|
@@ -208,6 +208,12 @@ export async function* watchChunks<T>(
|
|
|
208
208
|
* answers) and backup-eligible (a model that keeps doing it should be walked
|
|
209
209
|
* away from). Until something streams, the empty frames are held back, so the
|
|
210
210
|
* retry rule still holds: retry only while nothing was emitted.
|
|
211
|
+
*
|
|
212
|
+
* A hosted call streams the moment it appears, like reasoning, and is not an
|
|
213
|
+
* answer either: a search alone has said nothing. Forwarding it commits the
|
|
214
|
+
* stream, which is the point, because a retry past it would rerun and rebill
|
|
215
|
+
* the vendor's tool and show the reader the same search twice. The one hosted
|
|
216
|
+
* result that IS an answer is a generated image.
|
|
211
217
|
*/
|
|
212
218
|
export async function* requireContent<T extends ProviderChunk>(
|
|
213
219
|
provider: string,
|
|
@@ -223,9 +229,13 @@ export async function* requireContent<T extends ProviderChunk>(
|
|
|
223
229
|
for await (const chunk of chunks) {
|
|
224
230
|
if (chunk.type === "finish") finish = chunk.finishReason;
|
|
225
231
|
if (chunk.usage) usage = chunk.usage;
|
|
226
|
-
answered ||= Boolean(
|
|
232
|
+
answered ||= Boolean(
|
|
233
|
+
chunk.content ||
|
|
234
|
+
chunk.toolCalls?.length ||
|
|
235
|
+
chunk.hosted?.some((call) => call.status !== "failed" && call.output?.images?.length),
|
|
236
|
+
);
|
|
227
237
|
if (!streaming) {
|
|
228
|
-
streaming = answered || Boolean(chunk.reasoning);
|
|
238
|
+
streaming = answered || Boolean(chunk.reasoning || chunk.hosted?.length);
|
|
229
239
|
// Held rather than forwarded: once a chunk is out, the stream is
|
|
230
240
|
// committed and the retry this guard exists to trigger can no longer
|
|
231
241
|
// fire. Nothing has arrived worth showing yet, so there is nothing to
|