@ai-sdk/google 3.0.89 → 3.0.91

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @ai-sdk/google
 
+## 3.0.91
+
+### Patch Changes
+
+- bbac4da: Fix Google tool result conversion to send file data as inline data instead of JSON text on the legacy tool-result path.
+- 3fda3bc: Expand standalone Google `threshold` provider options into safety settings.
+- e1af05f: feat (video): support video (not just image) reference inputs in `inputReferences` for reference-to-video generation
+- Updated dependencies [e1af05f]
+  - @ai-sdk/provider@3.0.14
+  - @ai-sdk/provider-utils@4.0.38
+
+## 3.0.90
+
+### Patch Changes
+
+- b585445: Backport Gemini Interactions video output parsing and per-modality output token breakdown for AI SDK v6.
+- 1825ecc: Backport `vertex.interactions()` for the Gemini Interactions API on Vertex AI for AI SDK v6.
+- afee362: Pass documented Gemini external HTTPS file URLs through without downloading them.
+- 327642b: fix: more precise default message for tool execution denial
+- Updated dependencies [d559de9]
+  - @ai-sdk/provider-utils@4.0.37
+
 ## 3.0.89
 
 ### Patch Changes

package/dist/index.d.mts

@@ -373,6 +373,13 @@ type GoogleInteractionsProviderMetadata = {
      * Service tier used for this interaction (passthrough for observability).
      */
     serviceTier?: string;
+    /**
+     * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+     * from the Interactions API `output_tokens_by_modality`. Present only when
+     * the response reports a breakdown. Preview surface for per-modality billing;
+     * may be promoted to a first-class usage field later.
+     */
+    outputTokensByModality?: Record<string, number>;
     /**
      * Per-block signature hash for backend validation. Set by the SDK on output
      * reasoning / tool-call parts and round-tripped on input parts.

package/dist/index.d.ts

@@ -373,6 +373,13 @@ type GoogleInteractionsProviderMetadata = {
      * Service tier used for this interaction (passthrough for observability).
      */
     serviceTier?: string;
+    /**
+     * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+     * from the Interactions API `output_tokens_by_modality`. Present only when
+     * the response reports a breakdown. Preview surface for per-modality billing;
+     * may be promoted to a first-class usage field later.
+     */
+    outputTokensByModality?: Record<string, number>;
     /**
      * Per-block signature hash for backend validation. Set by the SDK on output
      * reasoning / tool-call parts and round-tripped on input parts.

package/dist/index.js

@@ -30,7 +30,7 @@ module.exports = __toCommonJS(index_exports);
 var import_provider_utils23 = require("@ai-sdk/provider-utils");
 
 // src/version.ts
-var VERSION = true ? "3.0.89" : "0.0.0-test";
+var VERSION = true ? "3.0.91" : "0.0.0-test";
 
 // src/google-generative-ai-embedding-model.ts
 var import_provider = require("@ai-sdk/provider");
@@ -511,6 +511,8 @@ function appendLegacyToolResultParts(parts, toolName, outputValue, toolCallId) {
         });
         break;
       case "image-data":
+      case "file-data": {
+        const topLevelMediaType = String(contentPart.mediaType).split("/")[0];
         parts.push(
           {
             inlineData: {
@@ -519,10 +521,11 @@ function appendLegacyToolResultParts(parts, toolName, outputValue, toolCallId) {
             }
           },
           {
-            text: "Tool executed successfully and returned this image as a response"
+            text: `Tool executed successfully and returned this ${topLevelMediaType === "image" ? "image" : "file"} as a response`
           }
         );
         break;
+      }
       default:
         parts.push({ text: JSON.stringify(contentPart) });
         break;
@@ -724,7 +727,7 @@ function convertToGoogleGenerativeAIMessages(prompt, options) {
                 name: part.toolName,
                 response: {
                   name: part.toolName,
-                  content: output.type === "execution-denied" ? (_e = output.reason) != null ? _e : "Tool execution denied." : output.value
+                  content: output.type === "execution-denied" ? (_e = output.reason) != null ? _e : "Tool call execution denied." : output.value
                 }
               }
             });
@@ -1434,6 +1437,12 @@ function mapGoogleGenerativeAIFinishReason({
 }
 
 // src/google-generative-ai-language-model.ts
+var configurableSafetySettingCategories = [
+  "HARM_CATEGORY_HATE_SPEECH",
+  "HARM_CATEGORY_DANGEROUS_CONTENT",
+  "HARM_CATEGORY_HARASSMENT",
+  "HARM_CATEGORY_SEXUALLY_EXPLICIT"
+];
 var GoogleGenerativeAILanguageModel = class {
   constructor(modelId, config) {
     this.specificationVersion = "v3";
@@ -1464,7 +1473,7 @@ var GoogleGenerativeAILanguageModel = class {
     toolChoice,
     providerOptions
   }, { isStreaming = false } = {}) {
-    var _a, _b;
+    var _a, _b, _c;
     const warnings = [];
     const providerOptionsName = this.config.provider.includes("vertex") ? "vertex" : "google";
     let googleOptions = await (0, import_provider_utils6.parseProviderOptions)({
@@ -1539,6 +1548,11 @@ var GoogleGenerativeAILanguageModel = class {
       isVertexProvider
     });
     const streamFunctionCallArguments = isStreaming && isVertexProvider ? (_a = googleOptions == null ? void 0 : googleOptions.streamFunctionCallArguments) != null ? _a : false : void 0;
+    const safetyThreshold = googleOptions == null ? void 0 : googleOptions.threshold;
+    const safetySettings = (_b = googleOptions == null ? void 0 : googleOptions.safetySettings) != null ? _b : safetyThreshold != null ? configurableSafetySettingCategories.map((category) => ({
+      category,
+      threshold: safetyThreshold
+    })) : void 0;
     const toolConfig = googleToolConfig || streamFunctionCallArguments || (googleOptions == null ? void 0 : googleOptions.retrievalConfig) ? {
       ...googleToolConfig,
       ...streamFunctionCallArguments && {
@@ -1568,7 +1582,7 @@ var GoogleGenerativeAILanguageModel = class {
           responseSchema: (responseFormat == null ? void 0 : responseFormat.type) === "json" && responseFormat.schema != null && // Google GenAI does not support all OpenAPI Schema features,
           // so this is needed as an escape hatch:
           // TODO convert into provider option
-          ((_b = googleOptions == null ? void 0 : googleOptions.structuredOutputs) != null ? _b : true) ? convertJSONSchemaToOpenAPISchema(responseFormat.schema) : void 0,
+          ((_c = googleOptions == null ? void 0 : googleOptions.structuredOutputs) != null ? _c : true) ? convertJSONSchemaToOpenAPISchema(responseFormat.schema) : void 0,
           ...(googleOptions == null ? void 0 : googleOptions.audioTimestamp) && {
             audioTimestamp: googleOptions.audioTimestamp
           },
@@ -1584,7 +1598,7 @@ var GoogleGenerativeAILanguageModel = class {
         },
         contents,
         systemInstruction: isGemmaModel ? void 0 : systemInstruction,
-        safetySettings: googleOptions == null ? void 0 : googleOptions.safetySettings,
+        safetySettings,
         tools: googleTools2,
         toolConfig,
         cachedContent: googleOptions == null ? void 0 : googleOptions.cachedContent,
@@ -2957,31 +2971,32 @@ function convertFileToGoogleImage(file, warnings) {
   }
   const base64Data = typeof file.data === "string" ? file.data : (0, import_provider_utils15.convertUint8ArrayToBase64)(file.data);
   return {
-    inlineData: {
-      mimeType: file.mediaType || "image/png",
-      data: base64Data
-    }
+    bytesBase64Encoded: base64Data,
+    mimeType: file.mediaType || "image/png"
   };
 }
 function convertProviderReferenceImage(refImg) {
   if (refImg.bytesBase64Encoded) {
     return {
-      inlineData: {
-        mimeType: "image/png",
-        data: refImg.bytesBase64Encoded
+      image: {
+        bytesBase64Encoded: refImg.bytesBase64Encoded,
+        mimeType: "image/png"
       }
     };
   }
   if (refImg.gcsUri) {
     return {
-      gcsUri: refImg.gcsUri
+      image: {
+        gcsUri: refImg.gcsUri,
+        mimeType: "image/png"
+      }
     };
   }
   return refImg;
 }
 function convertInputReferenceImage(file, warnings) {
   const image = convertFileToGoogleImage(file, warnings);
-  return image != null ? { image, referenceType: "asset" } : void 0;
+  return image != null ? { image } : void 0;
 }
 var GoogleGenerativeAIVideoModel = class {
   constructor(modelId, config) {
@@ -3263,6 +3278,19 @@ function convertGoogleInteractionsUsage(usage) {
     raw: usage
   };
 }
+function getGoogleInteractionsOutputTokensByModality(usage) {
+  const byModality = usage == null ? void 0 : usage.output_tokens_by_modality;
+  if (byModality == null) {
+    return void 0;
+  }
+  const result = {};
+  for (const entry of byModality) {
+    if ((entry == null ? void 0 : entry.modality) != null && entry.tokens != null) {
+      result[entry.modality] = entry.tokens;
+    }
+  }
+  return Object.keys(result).length > 0 ? result : void 0;
+}
 
 // src/interactions/extract-google-interactions-sources.ts
 var KNOWN_DOC_EXTENSIONS = {
@@ -3514,7 +3542,7 @@ function buildGoogleInteractionsStreamTransform({
       controller.enqueue({ type: "stream-start", warnings });
     },
     transform(chunk, controller) {
-      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t;
       if (includeRawChunks) {
         controller.enqueue({ type: "raw", rawValue: chunk.rawValue });
       }
@@ -3703,9 +3731,37 @@ function buildGoogleInteractionsStreamTransform({
             }
             break;
           }
+          if (dtype === "video" && (open.kind === "pending_model_output" || open.kind === "text")) {
+            const videoDelta = event.delta;
+            const google2 = {};
+            if (interactionId != null) google2.interactionId = interactionId;
+            const providerMetadata = Object.keys(google2).length > 0 ? { google: google2 } : void 0;
+            if ((videoDelta == null ? void 0 : videoDelta.data) != null && videoDelta.data.length > 0) {
+              controller.enqueue({
+                type: "file",
+                mediaType: (_m = videoDelta.mime_type) != null ? _m : "video/mp4",
+                data: videoDelta.data,
+                ...providerMetadata ? { providerMetadata } : {}
+              });
+            } else if ((videoDelta == null ? void 0 : videoDelta.uri) != null && videoDelta.uri.length > 0) {
+              const uriProviderMetadata = {
+                google: {
+                  ...interactionId != null ? { interactionId } : {},
+                  videoUri: videoDelta.uri
+                }
+              };
+              controller.enqueue({
+                type: "file",
+                mediaType: (_n = videoDelta.mime_type) != null ? _n : "video/mp4",
+                data: "",
+                providerMetadata: uriProviderMetadata
+              });
+            }
+            break;
+          }
           const delta = event.delta;
           if (open.kind === "text" && (delta == null ? void 0 : delta.type) === "text") {
-            const text = (_m = delta.text) != null ? _m : "";
+            const text = (_o = delta.text) != null ? _o : "";
             if (text.length > 0) {
               controller.enqueue({
                 type: "text-delta",
@@ -3808,7 +3864,7 @@ function buildGoogleInteractionsStreamTransform({
             if (open.data != null && open.data.length > 0) {
               controller.enqueue({
                 type: "file",
-                mediaType: (_n = open.mimeType) != null ? _n : "image/png",
+                mediaType: (_p = open.mimeType) != null ? _p : "image/png",
                 data: open.data,
                 ...providerMetadata ? { providerMetadata } : {}
               });
@@ -3821,7 +3877,7 @@ function buildGoogleInteractionsStreamTransform({
               };
               controller.enqueue({
                 type: "file",
-                mediaType: (_o = open.mimeType) != null ? _o : "image/png",
+                mediaType: (_q = open.mimeType) != null ? _q : "image/png",
                 data: "",
                 providerMetadata: uriProviderMetadata
               });
@@ -3848,7 +3904,7 @@ function buildGoogleInteractionsStreamTransform({
               type: "tool-call",
               toolCallId: open.toolCallId,
               toolName: open.toolName,
-              input: JSON.stringify((_p = open.arguments) != null ? _p : {}),
+              input: JSON.stringify((_r = open.arguments) != null ? _r : {}),
               providerExecuted: true
             });
             open.callEmitted = true;
@@ -3857,7 +3913,7 @@ function buildGoogleInteractionsStreamTransform({
               type: "tool-result",
               toolCallId: open.callId,
               toolName: open.toolName,
-              result: (_q = open.result) != null ? _q : null
+              result: (_s = open.result) != null ? _s : null
             });
             open.resultEmitted = true;
             const sources = builtinToolResultToSources({
@@ -3911,7 +3967,7 @@ function buildGoogleInteractionsStreamTransform({
         case "error": {
           const event = value;
           finishStatus = "failed";
-          const errorPayload = (_r = event.error) != null ? _r : {
+          const errorPayload = (_t = event.error) != null ? _t : {
             message: "Unknown interaction error"
           };
           controller.enqueue({ type: "error", error: errorPayload });
@@ -3929,10 +3985,12 @@ function buildGoogleInteractionsStreamTransform({
         }),
         raw: finishStatus
       };
+      const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(usage);
       const providerMetadata = {
         google: {
           ...interactionId != null ? { interactionId } : {},
-          ...serviceTier != null ? { serviceTier } : {}
+          ...serviceTier != null ? { serviceTier } : {},
+          ...outputTokensByModality != null ? { outputTokensByModality } : {}
         }
       };
       controller.enqueue({
@@ -4421,9 +4479,16 @@ var contentBlockSchema = () => {
     resolution: import_v415.z.enum(["low", "medium", "high", "ultra_high"]).nullish(),
     uri: import_v415.z.string().nullish()
   }).loose();
+  const videoContent = import_v415.z.object({
+    type: import_v415.z.literal("video"),
+    data: import_v415.z.string().nullish(),
+    mime_type: import_v415.z.string().nullish(),
+    uri: import_v415.z.string().nullish()
+  }).loose();
   return import_v415.z.union([
     textContent,
     imageContent,
+    videoContent,
     import_v415.z.object({ type: import_v415.z.string() }).loose()
   ]);
 };
@@ -4570,6 +4635,12 @@ var googleInteractionsEventSchema = (0, import_provider_utils17.lazySchema)(
         resolution: import_v415.z.enum(["low", "medium", "high", "ultra_high"]).nullish(),
         uri: import_v415.z.string().nullish()
       }).loose();
+      const stepDeltaVideo = import_v415.z.object({
+        type: import_v415.z.literal("video"),
+        data: import_v415.z.string().nullish(),
+        mime_type: import_v415.z.string().nullish(),
+        uri: import_v415.z.string().nullish()
+      }).loose();
       const stepDeltaBuiltinToolCall = import_v415.z.object({
         type: import_v415.z.enum(BUILTIN_TOOL_CALL_STEP_TYPES),
         id: import_v415.z.string().nullish(),
@@ -4592,6 +4663,7 @@ var googleInteractionsEventSchema = (0, import_provider_utils17.lazySchema)(
       const stepDeltaUnion = import_v415.z.union([
         stepDeltaText,
         stepDeltaImage,
+        stepDeltaVideo,
         stepDeltaThoughtSummary,
         stepDeltaThoughtSignature,
         stepDeltaArgumentsDelta,
@@ -4880,7 +4952,7 @@ function parseGoogleInteractionsOutputs({
   generateId: generateId3,
   interactionId
 }) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
   const content = [];
   let hasFunctionCall = false;
   if (steps == null) {
@@ -4933,6 +5005,28 @@ function parseGoogleInteractionsOutputs({
                 }
               });
             }
+          } else if (blockType === "video") {
+            const video = block;
+            if (video.data != null && video.data.length > 0) {
+              content.push({
+                type: "file",
+                mediaType: (_e = video.mime_type) != null ? _e : "video/mp4",
+                data: video.data,
+                ...googleProviderMetadata({ interactionId })
+              });
+            } else if (video.uri != null && video.uri.length > 0) {
+              content.push({
+                type: "file",
+                mediaType: (_f = video.mime_type) != null ? _f : "video/mp4",
+                data: "",
+                providerMetadata: {
+                  google: {
+                    ...interactionId != null ? { interactionId } : {},
+                    videoUri: video.uri
+                  }
+                }
+              });
+            }
           }
         }
         break;
@@ -4960,7 +5054,7 @@ function parseGoogleInteractionsOutputs({
           type: "tool-call",
           toolCallId: call.id,
           toolName: call.name,
-          input: JSON.stringify((_e = call.arguments) != null ? _e : {}),
+          input: JSON.stringify((_g = call.arguments) != null ? _g : {}),
           ...googleProviderMetadata({
             signature: call.signature,
             interactionId
@@ -4971,23 +5065,23 @@ function parseGoogleInteractionsOutputs({
       default: {
         if (BUILTIN_TOOL_CALL_TYPES2.has(type)) {
           const call = step;
-          const toolName = type === "mcp_server_tool_call" ? (_f = call.name) != null ? _f : "mcp_server_tool" : builtinToolNameFromCallType2(type);
-          const input = JSON.stringify((_g = call.arguments) != null ? _g : {});
+          const toolName = type === "mcp_server_tool_call" ? (_h = call.name) != null ? _h : "mcp_server_tool" : builtinToolNameFromCallType2(type);
+          const input = JSON.stringify((_i = call.arguments) != null ? _i : {});
           content.push({
             type: "tool-call",
-            toolCallId: (_h = call.id) != null ? _h : generateId3(),
+            toolCallId: (_j = call.id) != null ? _j : generateId3(),
             toolName,
             input,
             providerExecuted: true
           });
         } else if (BUILTIN_TOOL_RESULT_TYPES2.has(type)) {
           const result = step;
-          const toolName = type === "mcp_server_tool_result" ? (_i = result.name) != null ? _i : "mcp_server_tool" : builtinToolNameFromResultType2(type);
+          const toolName = type === "mcp_server_tool_result" ? (_k = result.name) != null ? _k : "mcp_server_tool" : builtinToolNameFromResultType2(type);
           content.push({
             type: "tool-result",
-            toolCallId: (_j = result.call_id) != null ? _j : generateId3(),
+            toolCallId: (_l = result.call_id) != null ? _l : generateId3(),
             toolName,
-            result: (_k = result.result) != null ? _k : null
+            result: (_m = result.result) != null ? _m : null
           });
           const sources = builtinToolResultToSources({
             block: step,
@@ -5871,10 +5965,14 @@ var GoogleInteractionsLanguageModel = class {
       raw: response.status
     };
     const serviceTier = (_e = (_d = response.service_tier) != null ? _d : responseHeaders == null ? void 0 : responseHeaders["x-gemini-service-tier"]) != null ? _e : void 0;
+    const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(
+      response.usage
+    );
     const providerMetadata = {
       google: {
         ...interactionId != null ? { interactionId } : {},
-        ...serviceTier != null ? { serviceTier } : {}
+        ...serviceTier != null ? { serviceTier } : {},
+        ...outputTokensByModality != null ? { outputTokensByModality } : {}
       }
     };
     let timestamp;
@@ -6038,6 +6136,34 @@ function pruneUndefined(obj) {
 }
 
 // src/google-provider.ts
+var supportedExternalUrlMediaTypes = [
+  "text/html",
+  "text/css",
+  "text/plain",
+  "text/xml",
+  "text/csv",
+  "text/rtf",
+  "text/javascript",
+  "application/json",
+  "application/pdf",
+  "image/bmp",
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "video/mp4",
+  "video/mpeg",
+  "video/quicktime",
+  "video/avi",
+  "video/x-flv",
+  "video/mpg",
+  "video/webm",
+  "video/wmv",
+  "video/3gpp"
+];
+var externalHttpsUrlPattern = /^https:\/\/.*$/;
+function supportsExternalFileUrls(modelId) {
+  return /(^|\/)gemini-/.test(modelId) && !/(^|\/)gemini-2\.0/.test(modelId);
+}
 function createGoogleGenerativeAI(options = {}) {
   var _a, _b;
   const baseURL = (_a = (0, import_provider_utils23.withoutTrailingSlash)(options.baseURL)) != null ? _a : "https://generativelanguage.googleapis.com/v1beta";
@@ -6070,7 +6196,13 @@ function createGoogleGenerativeAI(options = {}) {
             `^https://(?:www\\.)?youtube\\.com/watch\\?v=[\\w-]+(?:&[\\w=&.-]*)?$`
           ),
           new RegExp(`^https://youtu\\.be/[\\w-]+(?:\\?[\\w=&.-]*)?$`)
-        ]
+        ],
+        ...supportsExternalFileUrls(modelId) ? Object.fromEntries(
+          supportedExternalUrlMediaTypes.map((mediaType) => [
+            mediaType,
+            [externalHttpsUrlPattern]
+          ])
+        ) : {}
       }),
       fetch: options.fetch
     });