@providerkit/core 0.29.1 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/README.md +58 -9
  2. package/THIRD_PARTY_NOTICES +144 -0
  3. package/dist/auth/chatgpt-models.js.map +1 -1
  4. package/dist/auth/oauth.d.ts.map +1 -1
  5. package/dist/auth/oauth.js.map +1 -1
  6. package/dist/capability.js.map +1 -1
  7. package/dist/context.d.ts +4 -3
  8. package/dist/context.d.ts.map +1 -1
  9. package/dist/context.js +19 -5
  10. package/dist/context.js.map +1 -1
  11. package/dist/decisions.js.map +1 -1
  12. package/dist/errors.d.ts.map +1 -1
  13. package/dist/errors.js +29 -2
  14. package/dist/errors.js.map +1 -1
  15. package/dist/fallback.d.ts +3 -0
  16. package/dist/fallback.d.ts.map +1 -1
  17. package/dist/fallback.js +9 -5
  18. package/dist/fallback.js.map +1 -1
  19. package/dist/gate.js.map +1 -1
  20. package/dist/hosted/anthropic.d.ts +30 -0
  21. package/dist/hosted/anthropic.d.ts.map +1 -0
  22. package/dist/hosted/anthropic.js +465 -0
  23. package/dist/hosted/anthropic.js.map +1 -0
  24. package/dist/hosted/gemini.d.ts +22 -0
  25. package/dist/hosted/gemini.d.ts.map +1 -0
  26. package/dist/hosted/gemini.js +440 -0
  27. package/dist/hosted/gemini.js.map +1 -0
  28. package/dist/hosted/openai.d.ts +33 -0
  29. package/dist/hosted/openai.d.ts.map +1 -0
  30. package/dist/hosted/openai.js +107 -0
  31. package/dist/hosted/openai.js.map +1 -0
  32. package/dist/hosted/responses.d.ts +28 -0
  33. package/dist/hosted/responses.d.ts.map +1 -0
  34. package/dist/hosted/responses.js +358 -0
  35. package/dist/hosted/responses.js.map +1 -0
  36. package/dist/hosted/shared.d.ts +107 -0
  37. package/dist/hosted/shared.d.ts.map +1 -0
  38. package/dist/hosted/shared.js +264 -0
  39. package/dist/hosted/shared.js.map +1 -0
  40. package/dist/hosted/types.d.ts +191 -0
  41. package/dist/hosted/types.d.ts.map +1 -0
  42. package/dist/hosted/types.js +10 -0
  43. package/dist/hosted/types.js.map +1 -0
  44. package/dist/hosted.d.ts +33 -0
  45. package/dist/hosted.d.ts.map +1 -0
  46. package/dist/hosted.js +61 -0
  47. package/dist/hosted.js.map +1 -0
  48. package/dist/index.d.ts +8 -7
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +8 -7
  51. package/dist/index.js.map +1 -1
  52. package/dist/key-pool.d.ts.map +1 -1
  53. package/dist/key-pool.js.map +1 -1
  54. package/dist/presets.d.ts +36 -5
  55. package/dist/presets.d.ts.map +1 -1
  56. package/dist/presets.js +32 -8
  57. package/dist/presets.js.map +1 -1
  58. package/dist/providers/anthropic.d.ts +75 -1
  59. package/dist/providers/anthropic.d.ts.map +1 -1
  60. package/dist/providers/anthropic.js +207 -135
  61. package/dist/providers/anthropic.js.map +1 -1
  62. package/dist/providers/deepgram-transcription.d.ts +24 -0
  63. package/dist/providers/deepgram-transcription.d.ts.map +1 -0
  64. package/dist/providers/deepgram-transcription.js +84 -0
  65. package/dist/providers/deepgram-transcription.js.map +1 -0
  66. package/dist/providers/factory.d.ts +10 -0
  67. package/dist/providers/factory.d.ts.map +1 -1
  68. package/dist/providers/factory.js +31 -23
  69. package/dist/providers/factory.js.map +1 -1
  70. package/dist/providers/gemini.d.ts +123 -1
  71. package/dist/providers/gemini.d.ts.map +1 -1
  72. package/dist/providers/gemini.js +114 -54
  73. package/dist/providers/gemini.js.map +1 -1
  74. package/dist/providers/hosted-codecs.d.ts +10 -0
  75. package/dist/providers/hosted-codecs.d.ts.map +1 -0
  76. package/dist/providers/hosted-codecs.js +8 -0
  77. package/dist/providers/hosted-codecs.js.map +1 -0
  78. package/dist/providers/openai-transcription.d.ts +21 -0
  79. package/dist/providers/openai-transcription.d.ts.map +1 -0
  80. package/dist/providers/openai-transcription.js +144 -0
  81. package/dist/providers/openai-transcription.js.map +1 -0
  82. package/dist/providers/openai.d.ts +69 -0
  83. package/dist/providers/openai.d.ts.map +1 -1
  84. package/dist/providers/openai.js +38 -14
  85. package/dist/providers/openai.js.map +1 -1
  86. package/dist/providers/responses.d.ts +123 -0
  87. package/dist/providers/responses.d.ts.map +1 -1
  88. package/dist/providers/responses.js +57 -22
  89. package/dist/providers/responses.js.map +1 -1
  90. package/dist/providers/speech-factory.js.map +1 -1
  91. package/dist/providers/transcription-factory.d.ts +30 -0
  92. package/dist/providers/transcription-factory.d.ts.map +1 -0
  93. package/dist/providers/transcription-factory.js +38 -0
  94. package/dist/providers/transcription-factory.js.map +1 -0
  95. package/dist/registry.d.ts +3 -0
  96. package/dist/registry.d.ts.map +1 -1
  97. package/dist/registry.js +8 -4
  98. package/dist/registry.js.map +1 -1
  99. package/dist/retry.js.map +1 -1
  100. package/dist/speech-core.d.ts +2 -17
  101. package/dist/speech-core.d.ts.map +1 -1
  102. package/dist/speech-core.js +10 -50
  103. package/dist/speech-core.js.map +1 -1
  104. package/dist/tools.d.ts.map +1 -1
  105. package/dist/tools.js.map +1 -1
  106. package/dist/transcription-core.d.ts +146 -0
  107. package/dist/transcription-core.d.ts.map +1 -0
  108. package/dist/transcription-core.js +193 -0
  109. package/dist/transcription-core.js.map +1 -0
  110. package/dist/transcription.d.ts +7 -0
  111. package/dist/transcription.d.ts.map +1 -0
  112. package/dist/transcription.js +12 -0
  113. package/dist/transcription.js.map +1 -0
  114. package/dist/transport.d.ts +26 -0
  115. package/dist/transport.d.ts.map +1 -1
  116. package/dist/transport.js +57 -2
  117. package/dist/transport.js.map +1 -1
  118. package/dist/turn.d.ts +10 -1
  119. package/dist/turn.d.ts.map +1 -1
  120. package/dist/turn.js +53 -1
  121. package/dist/turn.js.map +1 -1
  122. package/dist/types.d.ts +51 -0
  123. package/dist/types.d.ts.map +1 -1
  124. package/dist/types.js.map +1 -1
  125. package/dist/usage.d.ts +10 -1
  126. package/dist/usage.d.ts.map +1 -1
  127. package/dist/usage.js +40 -2
  128. package/dist/usage.js.map +1 -1
  129. package/dist/watchdog.d.ts +6 -0
  130. package/dist/watchdog.d.ts.map +1 -1
  131. package/dist/watchdog.js +10 -2
  132. package/dist/watchdog.js.map +1 -1
  133. package/package.json +14 -5
  134. package/src/context.ts +21 -5
  135. package/src/errors.ts +35 -2
  136. package/src/fallback.ts +21 -6
  137. package/src/hosted/anthropic.ts +512 -0
  138. package/src/hosted/gemini.ts +510 -0
  139. package/src/hosted/openai.ts +137 -0
  140. package/src/hosted/responses.ts +420 -0
  141. package/src/hosted/shared.ts +341 -0
  142. package/src/hosted/types.ts +214 -0
  143. package/src/hosted.ts +101 -0
  144. package/src/index.ts +64 -7
  145. package/src/presets.ts +54 -8
  146. package/src/providers/anthropic.ts +252 -143
  147. package/src/providers/deepgram-transcription.ts +109 -0
  148. package/src/providers/factory.ts +127 -82
  149. package/src/providers/gemini.ts +184 -60
  150. package/src/providers/hosted-codecs.ts +29 -0
  151. package/src/providers/openai-transcription.ts +177 -0
  152. package/src/providers/openai.ts +52 -15
  153. package/src/providers/responses.ts +93 -24
  154. package/src/providers/transcription-factory.ts +68 -0
  155. package/src/registry.ts +14 -4
  156. package/src/speech-core.ts +9 -47
  157. package/src/transcription-core.ts +345 -0
  158. package/src/transcription.ts +24 -0
  159. package/src/transport.ts +77 -3
  160. package/src/turn.ts +55 -1
  161. package/src/types.ts +62 -1
  162. package/src/usage.ts +32 -2
  163. package/src/watchdog.ts +12 -2
@@ -5,6 +5,11 @@
5
5
  // its own chain of thought on the next turn. Written straight against the REST
6
6
  // wire rather than @google/genai, because this package ships zero dependencies
7
7
  // and the SDK is a Node-shaped one.
8
+ //
9
+ // Built-in tool shapes, parts and grounding fields follow Vercel's AI SDK
10
+ // (Apache-2.0, Copyright 2023 Vercel, Inc.: google-prepare-tools.ts,
11
+ // google-language-model.ts, convert-to-google-messages.ts); the byte offsets
12
+ // and segment checks follow PydanticAI (MIT). See THIRD_PARTY_NOTICES.
8
13
  import { repairHistory } from "../context.ts";
9
14
  import {
10
15
  ProviderError,
@@ -13,10 +18,12 @@ import {
13
18
  validateMaxAttempts,
14
19
  type ErrorKind,
15
20
  } from "../errors.ts";
21
+ import type { HostedToolType } from "../hosted/types.ts";
22
+ import { refuseHostedTools, type HostedCodecs } from "./hosted-codecs.ts";
16
23
  import { schemaPrompt, toAntigravityToolSchema, toGeminiToolSchema } from "../schema.ts";
17
24
  import { parseToolArgs, settleToolCalls } from "../tool-args.ts";
18
25
  import { streamSse, apiUrl, type FetchLike } from "../transport.ts";
19
- import { withConfiguredFallbacks, type ProviderFallbackConfig } from "../fallback.ts";
26
+ import { configuredFallbacks, type ProviderFallbackConfig } from "../fallback.ts";
20
27
  import type {
21
28
  ChatMessage,
22
29
  Effort,
@@ -25,6 +32,8 @@ import type {
25
32
  Provider,
26
33
  ProviderChunk,
27
34
  StreamOptions,
35
+ TokenUsage,
36
+ ToolCall,
28
37
  ToolCallDelta,
29
38
  ToolChoice,
30
39
  ToolDefinition,
@@ -64,6 +73,15 @@ export interface GeminiConfig extends ProviderFallbackConfig {
64
73
  maxTokens?: number;
65
74
  fetchImpl?: FetchLike;
66
75
  headers?: Record<string, string>;
76
+ /**
77
+ * For providers imported from `@providerkit/core/hosted`.
78
+ * The hosted tools this endpoint runs. Defaults to web search, web fetch,
79
+ * code execution and file search on the Gemini API
80
+ * (generativelanguage.googleapis.com), and to none anywhere else. Vertex
81
+ * and the gateways differ, and one that ignores a tool answers unsearched
82
+ * without saying so. Antigravity runs none.
83
+ */
84
+ servesHostedTools?: readonly HostedToolType[];
67
85
  }
68
86
 
69
87
  const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com";
@@ -193,6 +211,16 @@ function parseArgs(json: string): Record<string, unknown> {
193
211
  return parseToolArgs(json);
194
212
  }
195
213
 
214
+ function functionCallPart(call: ToolCall): unknown {
215
+ return {
216
+ functionCall: { id: call.id, name: call.name, args: parseArgs(call.arguments) },
217
+ // Replayed verbatim: this is the model's own reasoning token, and without
218
+ // it the next turn starts from a chain of thought that no longer includes
219
+ // the call it just made.
220
+ ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}),
221
+ };
222
+ }
223
+
196
224
  /** Same rule for a tool RESULT, which `functionResponse.response` also requires
197
225
  * as an object. Plain text (the common case) rides under `output`. */
198
226
  function toResponseObject(content: string): Record<string, unknown> {
@@ -219,8 +247,25 @@ function toResponseObject(content: string): Record<string, unknown> {
219
247
  * that the response count does not match the call count. The responses go
220
248
  * first, in call order, and the run's images follow them all, the way the
221
249
  * OpenAI dialect places them.
250
+ *
251
+ * A model turn that ran built-in tools goes back to `provider`, the one that
252
+ * made it, as the parts it came in: the code and its result, the server
253
+ * tool's call and response with their thought signatures, which Gemini needs
254
+ * returned unchanged when built-ins and functions share a turn.
222
255
  */
223
- export function toGeminiContents(messages: readonly ChatMessage[]): {
256
+ export function toGeminiContents(
257
+ messages: readonly ChatMessage[],
258
+ provider?: string,
259
+ ): { system?: string; contents: GeminiContent[] } {
260
+ return geminiContents(messages, provider);
261
+ }
262
+
263
+ /** @internal Hosted replay stays outside the plain converter. */
264
+ export function geminiContents(
265
+ messages: readonly ChatMessage[],
266
+ provider?: string,
267
+ replay?: HostedCodecs["gemini"]["replay"],
268
+ ): {
224
269
  system?: string;
225
270
  contents: GeminiContent[];
226
271
  } {
@@ -253,17 +298,15 @@ export function toGeminiContents(messages: readonly ChatMessage[]): {
253
298
  break;
254
299
 
255
300
  case "assistant": {
301
+ const replayed =
302
+ provider === undefined ? undefined : replay?.(message, provider, functionCallPart);
303
+ if (replayed) {
304
+ contents.push({ role: "model", parts: replayed });
305
+ break;
306
+ }
256
307
  const parts: unknown[] = [];
257
308
  if (message.content) parts.push({ text: message.content });
258
- for (const call of message.toolCalls ?? []) {
259
- parts.push({
260
- functionCall: { id: call.id, name: call.name, args: parseArgs(call.arguments) },
261
- // Replayed verbatim: this is the model's own reasoning token, and
262
- // without it the next turn starts from a chain of thought that
263
- // no longer includes the call it just made.
264
- ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}),
265
- });
266
- }
309
+ for (const call of message.toolCalls ?? []) parts.push(functionCallPart(call));
267
310
  // An assistant turn with neither text nor calls has no representation
268
311
  // here, and an empty `parts` is a 400.
269
312
  if (parts.length > 0) contents.push({ role: "model", parts });
@@ -369,12 +412,39 @@ function mapFinishReason(provider: string, reason: string): FinishReason {
369
412
  }
370
413
  }
371
414
 
372
- interface GeminiPart {
415
+ export interface GeminiPart {
373
416
  text?: string;
374
417
  /** Marks the parts that are the model's reasoning. Absent on the answer. */
375
418
  thought?: boolean;
376
419
  thoughtSignature?: string;
377
420
  functionCall?: { id?: string; name?: string; args?: Record<string, unknown> };
421
+ /** Code the code execution tool ran, and what came of it, paired by order. */
422
+ executableCode?: { language?: string; code?: string };
423
+ codeExecutionResult?: { outcome?: string; output?: string };
424
+ /** A built-in tool's call and response, on Gemini 3 with function tools. */
425
+ toolCall?: { id?: string; toolType?: string; args?: Record<string, unknown> };
426
+ toolResponse?: { id?: string; toolType?: string; response?: unknown };
427
+ }
428
+
429
+ export interface GroundingSegment {
430
+ /** Offsets count UTF-8 bytes in the named Part, or the answer when absent. */
431
+ partIndex?: number;
432
+ startIndex?: number;
433
+ endIndex?: number;
434
+ text?: string;
435
+ }
436
+
437
+ export interface GroundingChunk {
438
+ web?: { uri?: string; title?: string };
439
+ retrievedContext?: { uri?: string; title?: string; text?: string };
440
+ }
441
+
442
+ /** What a grounded answer read, and which of its sentences each source backs. */
443
+ export interface GroundingMetadata {
444
+ webSearchQueries?: string[];
445
+ groundingChunks?: GroundingChunk[];
446
+ groundingSupports?: { segment?: GroundingSegment; groundingChunkIndices?: number[] }[];
447
+ searchEntryPoint?: { renderedContent?: string };
378
448
  }
379
449
 
380
450
  /** Google's `google.rpc.Status`: an HTTP `code`, the canonical `status` name,
@@ -386,16 +456,20 @@ interface GeminiStatus {
386
456
  details?: unknown;
387
457
  }
388
458
 
389
- interface GeminiResponse {
459
+ export interface GeminiResponse {
390
460
  candidates?: {
391
461
  content?: { parts?: GeminiPart[] };
392
462
  finishReason?: string;
463
+ groundingMetadata?: GroundingMetadata;
464
+ urlContextMetadata?: { urlMetadata?: { retrievedUrl?: string; urlRetrievalStatus?: string }[] };
393
465
  }[];
394
466
  usageMetadata?: {
395
467
  promptTokenCount?: number;
396
468
  candidatesTokenCount?: number;
397
469
  thoughtsTokenCount?: number;
398
470
  cachedContentTokenCount?: number;
471
+ /** The prompt the built-in tools' results added, billed as input. */
472
+ toolUsePromptTokenCount?: number;
399
473
  };
400
474
  /** Present only on the in-band failure below. Never on a real candidate. */
401
475
  error?: GeminiStatus;
@@ -404,6 +478,11 @@ interface GeminiResponse {
404
478
  }
405
479
 
406
480
  export function createGeminiProvider(config: GeminiConfig): Provider {
481
+ return geminiProvider(config);
482
+ }
483
+
484
+ /** @internal Shared by the plain and hosted entry points. */
485
+ export function geminiProvider(config: GeminiConfig, codecs?: HostedCodecs): Provider {
407
486
  const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
408
487
  const id = config.id ?? "gemini";
409
488
  const isAntigravity = id === "antigravity";
@@ -421,11 +500,16 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
421
500
  ): AsyncIterable<ProviderChunk> {
422
501
  const { maxAttempts } = opts;
423
502
  validateMaxAttempts(maxAttempts);
503
+ if (!codecs) refuseHostedTools(id, opts, messages);
424
504
  const effort = opts.effort ?? config.effort;
425
505
  const maxTokens = opts.maxTokens ?? config.maxTokens;
426
506
  // Repaired first: a result with no call, or a call with no result, is a
427
507
  // 400 that loses the turn. See repairHistory.
428
- const { system, contents } = toGeminiContents(repairHistory(messages).messages);
508
+ const { system, contents } = geminiContents(
509
+ repairHistory(messages).messages,
510
+ id,
511
+ codecs?.gemini.replay,
512
+ );
429
513
 
430
514
  // A model id copied out of Gemini's docs is often already `models/…`, and
431
515
  // the doubled segment 404s as "model not found" — a confusing way to
@@ -436,6 +520,15 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
436
520
  // Not `!isGemini3`: `gemini-flash-latest` and models newer than 3 are not
437
521
  // Gemini 3 either, and they must get a level.
438
522
  const isGemini2 = /gemini-2/i.test(model);
523
+ const hosted = codecs?.gemini.plan(
524
+ config,
525
+ baseUrl,
526
+ id,
527
+ isAntigravity,
528
+ isGemini2,
529
+ tools,
530
+ opts,
531
+ );
439
532
 
440
533
  const generationConfig: Record<string, unknown> = {};
441
534
  if (maxTokens !== undefined) generationConfig.maxOutputTokens = maxTokens;
@@ -460,9 +553,10 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
460
553
  // outright with `400 "Function calling with a response mime type:
461
554
  // 'application/json' is unsupported"`, so the prompt is the only way a
462
555
  // schema travels beside tools there. Gemini 3 serves both, and there the
463
- // setting is the caller's measured preference.
556
+ // setting is the caller's measured preference. A built-in tool is a tool
557
+ // here too: the prompt is the one route that works beside it everywhere.
464
558
  const promptCarried =
465
- tools.length > 0 &&
559
+ (tools.length > 0 || (hosted?.tools.length ?? 0) > 0) &&
466
560
  ((opts.jsonWithTools ?? config.jsonWithTools) === "prompt" || !isGemini3);
467
561
  if (opts.json && !promptCarried) {
468
562
  generationConfig.responseMimeType = "application/json";
@@ -490,34 +584,51 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
490
584
  ...(opts.json && promptCarried ? [{ text: schemaPrompt(opts.json.schema) }] : []),
491
585
  ];
492
586
  if (systemParts.length > 0) request.systemInstruction = { parts: systemParts };
493
- if (tools.length > 0) {
494
- // Gemini has no switch for one call per turn. Sending the request
495
- // anyway would hand back the parallel calls the caller ruled out.
496
- if (opts.parallelToolCalls === false) {
497
- throw new ProviderError(
498
- id,
499
- "invalid",
500
- `${id}: Gemini can't limit a turn to one tool call. Leave parallelToolCalls unset ` +
501
- "for Gemini models, or run the calls it returns one at a time.",
502
- );
503
- }
587
+ // Gemini has no switch for one call per turn. Sending the request anyway
588
+ // would hand back the parallel calls the caller ruled out.
589
+ if (tools.length > 0 && opts.parallelToolCalls === false) {
590
+ throw new ProviderError(
591
+ id,
592
+ "invalid",
593
+ `${id}: Gemini can't limit a turn to one tool call. Leave parallelToolCalls unset ` +
594
+ "for Gemini models, or run the calls it returns one at a time.",
595
+ );
596
+ }
597
+ const builtIns = hosted?.tools ?? [];
598
+ if (tools.length > 0 || builtIns.length > 0) {
504
599
  request.tools = [
505
- {
506
- functionDeclarations: tools.map((tool) => ({
507
- name: tool.name,
508
- description: tool.description,
509
- // Antigravity (Cloud Code Assist) still expects the OpenAPI-subset
510
- // `parameters` field; the Gemini API and OpenCode Zen take standard
511
- // JSON Schema in `parametersJsonSchema` (cc-proxy 0.2.8).
512
- ...(isAntigravity
513
- ? { parameters: toAntigravityToolSchema(tool.inputSchema) }
514
- : { parametersJsonSchema: toGeminiToolSchema(tool.inputSchema) }),
515
- })),
516
- },
600
+ ...(tools.length > 0
601
+ ? [
602
+ {
603
+ functionDeclarations: tools.map((tool) => ({
604
+ name: tool.name,
605
+ description: tool.description,
606
+ // Antigravity (Cloud Code Assist) still expects the OpenAPI-subset
607
+ // `parameters` field; the Gemini API and OpenCode Zen take standard
608
+ // JSON Schema in `parametersJsonSchema` (cc-proxy 0.2.8).
609
+ ...(isAntigravity
610
+ ? { parameters: toAntigravityToolSchema(tool.inputSchema) }
611
+ : { parametersJsonSchema: toGeminiToolSchema(tool.inputSchema) }),
612
+ })),
613
+ },
614
+ ]
615
+ : []),
616
+ ...builtIns,
517
617
  ];
618
+ }
619
+ if (tools.length > 0) {
518
620
  // Pointless without declarations, and Gemini 400s on a tool config that
519
- // names a function it was never given.
520
- const toolConfig = toToolConfig(opts.toolChoice);
621
+ // names a function it was never given. Built-ins beside functions need
622
+ // the mode that lets both run and the server's own calls streamed back,
623
+ // which then go back with the turn (google-prepare-tools.ts:187-231).
624
+ // A forced choice never gets here with built-ins: it is refused.
625
+ const toolConfig =
626
+ builtIns.length > 0
627
+ ? {
628
+ functionCallingConfig: { mode: "VALIDATED" },
629
+ includeServerSideToolInvocations: true,
630
+ }
631
+ : toToolConfig(opts.toolChoice);
521
632
  if (toolConfig) request.toolConfig = toolConfig;
522
633
  }
523
634
 
@@ -571,6 +682,7 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
571
682
  let callIndex = 0;
572
683
  let finished = false;
573
684
  let emitted = false;
685
+ const hostedStream = codecs?.gemini.stream(id, hosted?.hosted ?? [], isGemini2);
574
686
 
575
687
  for (let baseIdx = 0; baseIdx < bases.length; baseIdx++) {
576
688
  const currentBase = bases[baseIdx]!;
@@ -615,33 +727,43 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
615
727
  if (chunk.usageMetadata) {
616
728
  const usage = chunk.usageMetadata;
617
729
  emitted = true;
618
- yield {
619
- type: "usage",
620
- usage: {
621
- inputTokens: usage.promptTokenCount ?? 0,
622
- // Thoughts bill as OUTPUT, and Gemini reports them OUTSIDE
623
- // candidatesTokenCount — leaving them out undercounts a thinking
624
- // turn by most of what it cost.
625
- outputTokens: (usage.candidatesTokenCount ?? 0) + (usage.thoughtsTokenCount ?? 0),
626
- cachedInputTokens: usage.cachedContentTokenCount ?? 0,
627
- ...(usage.thoughtsTokenCount !== undefined
628
- ? { reasoningTokens: usage.thoughtsTokenCount }
629
- : {}),
630
- },
730
+ const tokenUsage: TokenUsage = {
731
+ // What the built-in tools' results added to the prompt bills as
732
+ // input too, and Gemini reports it apart
733
+ // (Vercel convert-google-usage.ts:31-36).
734
+ inputTokens: (usage.promptTokenCount ?? 0) + (usage.toolUsePromptTokenCount ?? 0),
735
+ // Thoughts bill as OUTPUT, and Gemini reports them OUTSIDE
736
+ // candidatesTokenCount, so leaving them out undercounts a thinking
737
+ // turn by most of what it cost.
738
+ outputTokens: (usage.candidatesTokenCount ?? 0) + (usage.thoughtsTokenCount ?? 0),
739
+ cachedInputTokens: usage.cachedContentTokenCount ?? 0,
740
+ ...(usage.thoughtsTokenCount !== undefined
741
+ ? { reasoningTokens: usage.thoughtsTokenCount }
742
+ : {}),
631
743
  };
744
+ yield { type: "usage", usage: hostedStream?.usage(tokenUsage) ?? tokenUsage };
632
745
  }
633
746
 
634
747
  const candidate = chunk.candidates?.[0];
635
748
  if (!candidate) continue;
749
+ hostedStream?.candidate(candidate);
636
750
 
637
- // One chunk can carry several parts of each kind. They are coalesced
638
- // per kind so the seam sees one reasoning delta and one content delta
639
- // per chunk, in that order, rather than interleaved fragments.
751
+ // Coalesce text per frame, but emit hosted work at once: validating
752
+ // a later function call can throw, and must never hide a paid run
753
+ // from retry and fallback (invariant 19).
640
754
  let reasoning = "";
641
755
  let content = "";
642
756
  const toolCalls: ToolCallDelta[] = [];
643
- for (const part of candidate.content?.parts ?? []) {
644
- if (part.text) {
757
+ for (const [partIndex, part] of (candidate.content?.parts ?? []).entries()) {
758
+ const ran = hostedStream?.part(part, partIndex, content.length);
759
+ if (ran) {
760
+ if (ran.delta) {
761
+ emitted = true;
762
+ yield { type: "delta", hosted: [ran.delta] };
763
+ }
764
+ continue;
765
+ }
766
+ if (part.text !== undefined) {
645
767
  if (part.thought) reasoning += part.text;
646
768
  else content += part.text;
647
769
  continue;
@@ -670,6 +792,7 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
670
792
  }
671
793
  if (content) {
672
794
  emitted = true;
795
+ hostedStream?.text(content);
673
796
  yield { type: "delta", content };
674
797
  }
675
798
  if (toolCalls.length > 0) {
@@ -683,6 +806,7 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
683
806
  const finishReason = mapFinishReason(id, candidate.finishReason);
684
807
  finished = true;
685
808
  emitted = true;
809
+ if (hostedStream) yield* hostedStream.close();
686
810
  yield {
687
811
  type: "finish",
688
812
  // Gemini routinely reports STOP on a turn that called tools, and a
@@ -718,5 +842,5 @@ export function createGeminiProvider(config: GeminiConfig): Provider {
718
842
  },
719
843
  };
720
844
 
721
- return withConfiguredFallbacks(provider, config);
845
+ return configuredFallbacks(provider, config, codecs);
722
846
  }
@@ -0,0 +1,29 @@
1
+ import { ProviderError } from "../errors.ts";
2
+ import type { ChatMessage, StreamOptions } from "../types.ts";
3
+
4
+ /** Internal wiring only. Type imports never pull the codecs into plain chat. */
5
+ export interface HostedCodecs {
6
+ anthropic: typeof import("../hosted/anthropic.ts").anthropicHosted;
7
+ responses: typeof import("../hosted/responses.ts").responsesHosted;
8
+ gemini: typeof import("../hosted/gemini.ts").geminiHosted;
9
+ openai: typeof import("../hosted/openai.ts").openaiHosted;
10
+ }
11
+
12
+ export function refuseHostedTools(
13
+ provider: string,
14
+ opts: StreamOptions,
15
+ messages: readonly ChatMessage[],
16
+ ): void {
17
+ if (
18
+ opts.hostedTools?.length ||
19
+ messages.some(
20
+ (message) => message.role === "assistant" && message.hostedItems?.provider === provider,
21
+ )
22
+ ) {
23
+ throw new ProviderError(
24
+ provider,
25
+ "invalid",
26
+ `${provider}: hosted tools need the opt-in provider. Import its constructor from @providerkit/core/hosted.`,
27
+ );
28
+ }
29
+ }
@@ -0,0 +1,177 @@
1
+ // The OpenAI transcription wire: POST {base}/audio/transcriptions, multipart in, JSON out.
2
+ // Serves OpenAI and any server that copies the route (Groq, a local Whisper server, a proxy).
3
+ // One request per call; transcription-core.ts says why.
4
+ //
5
+ // Three things depend on the model, from OpenAI's speech-to-text guide and transcription
6
+ // reference (read 2026-11-11):
7
+ // - `gpt-transcribe` takes the language as `languages[]`; every other model takes `language`,
8
+ // and the guide says not to send both.
9
+ // - Only `gpt-4o-transcribe-diarize` labels speakers, in `diarized_json`, and it needs a
10
+ // `chunking_strategy` for anything over 30 seconds.
11
+ // - Only Whisper models return timed segments without speakers, in `verbose_json`. The rest
12
+ // answer `json`: the text alone.
13
+ import { attributionHeaders, type Attribution } from "../attribution.ts";
14
+ import {
15
+ asRecord,
16
+ inBandFailure,
17
+ MAX_REPLY_BYTES,
18
+ mediaType,
19
+ parseObject,
20
+ prepareTranscription,
21
+ readSegments,
22
+ resolveLimits,
23
+ seconds,
24
+ transcribeOnce,
25
+ unusable,
26
+ type TranscriptionLimits,
27
+ type TranscriptionProvider,
28
+ type TranscriptionUsage,
29
+ } from "../transcription-core.ts";
30
+ import { apiUrl, postBytes, versionedPath, type FetchLike } from "../transport.ts";
31
+
32
+ export interface OpenAITranscriptionConfig extends TranscriptionLimits, Attribution {
33
+ /** "" sends no Authorization header, for a local server that wants none. */
34
+ apiKey: string;
35
+ model: string;
36
+ /** Default `https://api.openai.com`. */
37
+ baseUrl?: string;
38
+ /** Default `/v1/audio/transcriptions`, or `/audio/transcriptions` when the base already ends
39
+ * in a version. */
40
+ path?: string;
41
+ /** The preset id. Names the provider in errors and on `TranscriptionProvider.id`. Default
42
+ * "openai". */
43
+ id?: string;
44
+ headers?: Record<string, string>;
45
+ /** Swapped in tests, or to route through a proxy. Defaults to global fetch. */
46
+ fetchImpl?: FetchLike;
47
+ }
48
+
49
+ // The server reads the format from the file name, so the name carries the extension the media
50
+ // type stands for. The formats are the ones the reference lists.
51
+ const EXTENSION: Readonly<Record<string, string>> = {
52
+ "audio/flac": "flac",
53
+ "audio/x-flac": "flac",
54
+ "audio/mpeg": "mp3",
55
+ "audio/mp3": "mp3",
56
+ "audio/mp4": "m4a",
57
+ "audio/m4a": "m4a",
58
+ "audio/x-m4a": "m4a",
59
+ "video/mp4": "mp4",
60
+ "video/mpeg": "mpeg",
61
+ "audio/ogg": "ogg",
62
+ "audio/wav": "wav",
63
+ "audio/wave": "wav",
64
+ "audio/x-wav": "wav",
65
+ "audio/vnd.wave": "wav",
66
+ "audio/webm": "webm",
67
+ "video/webm": "webm",
68
+ };
69
+
70
+ /** A type the table knows names the file; otherwise a `File` keeps its own name. */
71
+ function fileName(audio: Blob): string {
72
+ const extension = EXTENSION[mediaType(audio.type)];
73
+ if (extension) return `audio.${extension}`;
74
+ return audio instanceof File && audio.name ? audio.name : "audio";
75
+ }
76
+
77
+ const count = (value: unknown): number | undefined =>
78
+ typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
79
+
80
+ /**
81
+ * The provider's own usage, or nothing. Token counts that do not add up are dropped whole: a
82
+ * wrong number in a ledger is worse than a missing one. The audio and text parts are kept only
83
+ * when they fit inside the input.
84
+ */
85
+ function readUsage(raw: unknown): TranscriptionUsage | undefined {
86
+ const usage = asRecord(raw);
87
+ if (usage?.type === "duration") {
88
+ const billed = seconds(usage.seconds);
89
+ return billed === undefined ? undefined : { type: "duration", seconds: billed };
90
+ }
91
+ if (usage?.type !== "tokens") return undefined;
92
+ const input = count(usage.input_tokens);
93
+ const output = count(usage.output_tokens);
94
+ const total = count(usage.total_tokens);
95
+ if (input === undefined || output === undefined || total === undefined) return undefined;
96
+ if (!Number.isSafeInteger(input + output) || total < input + output) return undefined;
97
+ const details = asRecord(usage.input_token_details);
98
+ const audio = count(details?.audio_tokens);
99
+ const text = count(details?.text_tokens);
100
+ const parts = audio !== undefined && text !== undefined && audio + text <= input;
101
+ return {
102
+ type: "tokens",
103
+ inputTokens: input,
104
+ outputTokens: output,
105
+ totalTokens: total,
106
+ ...(parts ? { audioTokens: audio, textTokens: text } : {}),
107
+ };
108
+ }
109
+
110
+ export function createOpenAITranscriptionProvider(
111
+ config: OpenAITranscriptionConfig,
112
+ ): TranscriptionProvider {
113
+ const id = config.id ?? "openai";
114
+ const provider = `transcription:${id}`;
115
+ const baseUrl = config.baseUrl ?? "https://api.openai.com";
116
+ const limits = resolveLimits(config);
117
+ return {
118
+ id,
119
+ model: config.model,
120
+ async transcribe(request) {
121
+ const { audio, model, language, diarize } = prepareTranscription(provider, request, {
122
+ apiKey: config.apiKey,
123
+ keyRequired: false,
124
+ model: config.model,
125
+ maxAudioBytes: limits.maxAudioBytes,
126
+ });
127
+ return transcribeOnce(provider, limits.timeoutMs, request, async (signal) => {
128
+ const form = new FormData();
129
+ form.append("file", audio, fileName(audio));
130
+ form.append("model", model);
131
+ // The models take ISO 639-1 ("pt"), so a BCP-47 tag goes as its language subtag.
132
+ if (language) {
133
+ const code = language.split("-")[0]!.toLowerCase();
134
+ form.append(model.startsWith("gpt-transcribe") ? "languages[]" : "language", code);
135
+ }
136
+ if (diarize) {
137
+ // A model that cannot label speakers answers this with a 400, which is the honest
138
+ // refusal: speakers asked for and silently dropped would read as one voice.
139
+ form.append("response_format", "diarized_json");
140
+ form.append("chunking_strategy", "auto");
141
+ } else if (/whisper/i.test(model)) {
142
+ form.append("response_format", "verbose_json");
143
+ }
144
+ const { bytes } = await postBytes({
145
+ url: apiUrl(baseUrl, config.path ?? versionedPath(baseUrl, "/audio/transcriptions")),
146
+ headers: {
147
+ ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}),
148
+ ...attributionHeaders(config, config.headers),
149
+ ...config.headers,
150
+ },
151
+ body: form,
152
+ provider,
153
+ signal,
154
+ redirect: "error",
155
+ maxBytes: MAX_REPLY_BYTES,
156
+ ...(config.fetchImpl ? { fetchImpl: config.fetchImpl } : {}),
157
+ });
158
+ const body = parseObject(bytes);
159
+ const usage = readUsage(body?.usage);
160
+ // A failure inside a 200 is still a failure (invariant 11).
161
+ if (body?.error !== undefined && body.error !== null) throw inBandFailure(provider, body);
162
+ if (typeof body?.text !== "string") {
163
+ throw unusable(provider, "the reply holds no transcript", usage);
164
+ }
165
+ const durationSeconds = seconds(body.duration);
166
+ signal.throwIfAborted();
167
+ return {
168
+ text: body.text.trim(),
169
+ segments: readSegments(body.segments, "text", diarize),
170
+ ...(durationSeconds !== undefined ? { durationSeconds } : {}),
171
+ ...(usage ? { usage } : {}),
172
+ model,
173
+ };
174
+ });
175
+ },
176
+ };
177
+ }