@ai-sdk/google 4.0.2 → 4.0.4

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @ai-sdk/google
 
+## 4.0.4
+
+### Patch Changes
+
+- dc1eb8d: Support Gemini Interactions video output. Parse video output blocks from the Google Interactions API into file parts (buffered and streaming), and surface the per-modality output token breakdown via `providerMetadata.google.outputTokensByModality`.
+
+## 4.0.3
+
+### Patch Changes
+
+- 0274f34: feat (video): add first-class `frameImages` and `inputReferences` call options for video generation
+- Updated dependencies [0274f34]
+  - @ai-sdk/provider@4.0.1
+  - @ai-sdk/provider-utils@5.0.2
+
 ## 4.0.2
 
 ### Patch Changes

package/dist/index.d.ts

@@ -387,6 +387,13 @@ type GoogleInteractionsProviderMetadata = {
      * Service tier used for this interaction (passthrough for observability).
      */
     serviceTier?: string;
+    /**
+     * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+     * from the Interactions API `output_tokens_by_modality`. Present only when
+     * the response reports a breakdown. Preview surface for per-modality billing;
+     * may be promoted to a first-class usage field later.
+     */
+    outputTokensByModality?: Record<string, number>;
     /**
      * Per-block signature hash for backend validation. Set by the SDK on output
      * reasoning / tool-call parts and round-tripped on input parts.

package/dist/index.js

@@ -7,7 +7,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "4.0.2" : "0.0.0-test";
+var VERSION = true ? "4.0.4" : "0.0.0-test";
 
 // src/google-embedding-model.ts
 import {
@@ -3360,6 +3360,67 @@ var googleVideoModelOptionsSchema = lazySchema15(
 );
 
 // src/google-video-model.ts
+function getFirstFrameImage(options) {
+  var _a, _b;
+  return (_b = (_a = options.frameImages) == null ? void 0 : _a.find((frame) => frame.frameType === "first_frame")) == null ? void 0 : _b.image;
+}
+function resolveStartImage(options) {
+  var _a;
+  return (_a = getFirstFrameImage(options)) != null ? _a : options.image;
+}
+function getLastFrameImage(options) {
+  var _a, _b;
+  return (_b = (_a = options.frameImages) == null ? void 0 : _a.find((frame) => frame.frameType === "last_frame")) == null ? void 0 : _b.image;
+}
+function getInputReferences(options) {
+  if (options.frameImages != null && options.frameImages.length > 0) {
+    return void 0;
+  }
+  return options.inputReferences != null && options.inputReferences.length > 0 ? options.inputReferences : void 0;
+}
+function convertFileToGoogleImage(file, warnings) {
+  if (file.type === "url") {
+    if (file.url.startsWith("gs://")) {
+      return {
+        gcsUri: file.url,
+        mimeType: "image/png"
+      };
+    }
+    warnings.push({
+      type: "unsupported",
+      feature: "URL-based image input",
+      details: "Google Generative AI video models require base64-encoded images or GCS URIs. URL will be ignored."
+    });
+    return void 0;
+  }
+  const base64Data = typeof file.data === "string" ? file.data : convertUint8ArrayToBase64(file.data);
+  return {
+    inlineData: {
+      mimeType: file.mediaType || "image/png",
+      data: base64Data
+    }
+  };
+}
+function convertProviderReferenceImage(refImg) {
+  if (refImg.bytesBase64Encoded) {
+    return {
+      inlineData: {
+        mimeType: "image/png",
+        data: refImg.bytesBase64Encoded
+      }
+    };
+  }
+  if (refImg.gcsUri) {
+    return {
+      gcsUri: refImg.gcsUri
+    };
+  }
+  return refImg;
+}
+function convertInputReferenceImage(file, warnings) {
+  const image = convertFileToGoogleImage(file, warnings);
+  return image != null ? { image, referenceType: "asset" } : void 0;
+}
 var GoogleVideoModel = class {
   constructor(modelId, config) {
     this.modelId = modelId;
@@ -3386,39 +3447,30 @@ var GoogleVideoModel = class {
     if (options.prompt != null) {
       instance.prompt = options.prompt;
     }
-    if (options.image != null) {
-      if (options.image.type === "url") {
-        warnings.push({
-          type: "unsupported",
-          feature: "URL-based image input",
-          details: "Google Generative AI video models require base64-encoded images. URL will be ignored."
-        });
-      } else {
-        const base64Data = typeof options.image.data === "string" ? options.image.data : convertUint8ArrayToBase64(options.image.data);
-        instance.image = {
-          inlineData: {
-            mimeType: options.image.mediaType || "image/png",
-            data: base64Data
-          }
-        };
+    const startImage = resolveStartImage(options);
+    if (startImage != null) {
+      const image = convertFileToGoogleImage(startImage, warnings);
+      if (image != null) {
+        instance.image = image;
       }
     }
-    if ((googleOptions == null ? void 0 : googleOptions.referenceImages) != null) {
-      instance.referenceImages = googleOptions.referenceImages.map((refImg) => {
-        if (refImg.bytesBase64Encoded) {
-          return {
-            inlineData: {
-              mimeType: "image/png",
-              data: refImg.bytesBase64Encoded
-            }
-          };
-        } else if (refImg.gcsUri) {
-          return {
-            gcsUri: refImg.gcsUri
-          };
-        }
-        return refImg;
+    const lastFrameImage = getLastFrameImage(options);
+    if (lastFrameImage != null) {
+      const lastFrame = convertFileToGoogleImage(lastFrameImage, warnings);
+      if (lastFrame != null) {
+        instance.lastFrame = lastFrame;
+      }
+    }
+    const inputReferences = getInputReferences(options);
+    if (inputReferences != null) {
+      instance.referenceImages = inputReferences.flatMap((reference) => {
+        const converted = convertInputReferenceImage(reference, warnings);
+        return converted != null ? [converted] : [];
       });
+    } else if ((googleOptions == null ? void 0 : googleOptions.referenceImages) != null) {
+      instance.referenceImages = googleOptions.referenceImages.map(
+        (refImg) => convertProviderReferenceImage(refImg)
+      );
     }
     const parameters = {
       sampleCount: options.n
@@ -3917,6 +3969,19 @@ function convertGoogleInteractionsUsage(usage) {
     raw: usage
   };
 }
+function getGoogleInteractionsOutputTokensByModality(usage) {
+  const byModality = usage == null ? void 0 : usage.output_tokens_by_modality;
+  if (byModality == null) {
+    return void 0;
+  }
+  const result = {};
+  for (const entry of byModality) {
+    if ((entry == null ? void 0 : entry.modality) != null && entry.tokens != null) {
+      result[entry.modality] = entry.tokens;
+    }
+  }
+  return Object.keys(result).length > 0 ? result : void 0;
+}
 
 // src/interactions/extract-google-interactions-sources.ts
 var KNOWN_DOC_EXTENSIONS = {
@@ -4172,7 +4237,7 @@ function buildGoogleInteractionsStreamTransform({
       controller.enqueue({ type: "stream-start", warnings });
     },
     transform(chunk, controller) {
-      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t;
       if (includeRawChunks) {
         controller.enqueue({ type: "raw", rawValue: chunk.rawValue });
       }
@@ -4355,9 +4420,31 @@ function buildGoogleInteractionsStreamTransform({
             }
             break;
           }
+          if (dtype === "video" && (open.kind === "pending_model_output" || open.kind === "text")) {
+            const videoDelta = event.delta;
+            const google2 = {};
+            if (interactionId != null) google2.interactionId = interactionId;
+            const providerMetadata = Object.keys(google2).length > 0 ? { google: google2 } : void 0;
+            if ((videoDelta == null ? void 0 : videoDelta.data) != null && videoDelta.data.length > 0) {
+              controller.enqueue({
+                type: "file",
+                mediaType: (_m = videoDelta.mime_type) != null ? _m : "video/mp4",
+                data: { type: "data", data: videoDelta.data },
+                ...providerMetadata ? { providerMetadata } : {}
+              });
+            } else if ((videoDelta == null ? void 0 : videoDelta.uri) != null && videoDelta.uri.length > 0) {
+              controller.enqueue({
+                type: "file",
+                mediaType: (_n = videoDelta.mime_type) != null ? _n : "video/mp4",
+                data: { type: "url", url: new URL(videoDelta.uri) },
+                ...providerMetadata ? { providerMetadata } : {}
+              });
+            }
+            break;
+          }
           const delta = event.delta;
           if (open.kind === "text" && (delta == null ? void 0 : delta.type) === "text") {
-            const text = (_m = delta.text) != null ? _m : "";
+            const text = (_o = delta.text) != null ? _o : "";
             if (text.length > 0) {
               controller.enqueue({
                 type: "text-delta",
@@ -4460,14 +4547,14 @@ function buildGoogleInteractionsStreamTransform({
             if (open.data != null && open.data.length > 0) {
               controller.enqueue({
                 type: "file",
-                mediaType: (_n = open.mimeType) != null ? _n : "image/png",
+                mediaType: (_p = open.mimeType) != null ? _p : "image/png",
                 data: { type: "data", data: open.data },
                 ...providerMetadata ? { providerMetadata } : {}
               });
             } else if (open.uri != null && open.uri.length > 0) {
               controller.enqueue({
                 type: "file",
-                mediaType: (_o = open.mimeType) != null ? _o : "image/png",
+                mediaType: (_q = open.mimeType) != null ? _q : "image/png",
                 data: { type: "url", url: new URL(open.uri) },
                 ...providerMetadata ? { providerMetadata } : {}
               });
@@ -4494,7 +4581,7 @@ function buildGoogleInteractionsStreamTransform({
               type: "tool-call",
               toolCallId: open.toolCallId,
               toolName: open.toolName,
-              input: JSON.stringify((_p = open.arguments) != null ? _p : {}),
+              input: JSON.stringify((_r = open.arguments) != null ? _r : {}),
               providerExecuted: true
             });
             open.callEmitted = true;
@@ -4503,7 +4590,7 @@ function buildGoogleInteractionsStreamTransform({
               type: "tool-result",
               toolCallId: open.callId,
               toolName: open.toolName,
-              result: (_q = open.result) != null ? _q : null
+              result: (_s = open.result) != null ? _s : null
             });
             open.resultEmitted = true;
             const sources = builtinToolResultToSources({
@@ -4557,7 +4644,7 @@ function buildGoogleInteractionsStreamTransform({
         case "error": {
           const event = value;
           finishStatus = "failed";
-          const errorPayload = (_r = event.error) != null ? _r : {
+          const errorPayload = (_t = event.error) != null ? _t : {
             message: "Unknown interaction error"
           };
           controller.enqueue({ type: "error", error: errorPayload });
@@ -4575,10 +4662,12 @@ function buildGoogleInteractionsStreamTransform({
         }),
         raw: finishStatus
       };
+      const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(usage);
       const providerMetadata = {
         google: {
           ...interactionId != null ? { interactionId } : {},
-          ...serviceTier != null ? { serviceTier } : {}
+          ...serviceTier != null ? { serviceTier } : {},
+          ...outputTokensByModality != null ? { outputTokensByModality } : {}
         }
       };
       controller.enqueue({
@@ -5065,9 +5154,16 @@ var contentBlockSchema = () => {
     resolution: z20.enum(["low", "medium", "high", "ultra_high"]).nullish(),
     uri: z20.string().nullish()
   }).loose();
+  const videoContent = z20.object({
+    type: z20.literal("video"),
+    data: z20.string().nullish(),
+    mime_type: z20.string().nullish(),
+    uri: z20.string().nullish()
+  }).loose();
   return z20.union([
     textContent,
     imageContent,
+    videoContent,
     z20.object({ type: z20.string() }).loose()
   ]);
 };
@@ -5214,6 +5310,12 @@ var googleInteractionsEventSchema = lazySchema18(
         resolution: z20.enum(["low", "medium", "high", "ultra_high"]).nullish(),
         uri: z20.string().nullish()
       }).loose();
+      const stepDeltaVideo = z20.object({
+        type: z20.literal("video"),
+        data: z20.string().nullish(),
+        mime_type: z20.string().nullish(),
+        uri: z20.string().nullish()
+      }).loose();
       const stepDeltaBuiltinToolCall = z20.object({
         type: z20.enum(BUILTIN_TOOL_CALL_STEP_TYPES),
         id: z20.string().nullish(),
@@ -5236,6 +5338,7 @@ var googleInteractionsEventSchema = lazySchema18(
       const stepDeltaUnion = z20.union([
         stepDeltaText,
         stepDeltaImage,
+        stepDeltaVideo,
         stepDeltaThoughtSummary,
         stepDeltaThoughtSignature,
         stepDeltaArgumentsDelta,
@@ -5527,7 +5630,7 @@ function parseGoogleInteractionsOutputs({
   generateId: generateId3,
   interactionId
 }) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
   const content = [];
   let hasFunctionCall = false;
   if (steps == null) {
@@ -5575,6 +5678,23 @@ function parseGoogleInteractionsOutputs({
                 ...googleProviderMetadata({ interactionId })
               });
             }
+          } else if (blockType === "video") {
+            const video = block;
+            if (video.data != null && video.data.length > 0) {
+              content.push({
+                type: "file",
+                mediaType: (_e = video.mime_type) != null ? _e : "video/mp4",
+                data: { type: "data", data: video.data },
+                ...googleProviderMetadata({ interactionId })
+              });
+            } else if (video.uri != null && video.uri.length > 0) {
+              content.push({
+                type: "file",
+                mediaType: (_f = video.mime_type) != null ? _f : "video/mp4",
+                data: { type: "url", url: new URL(video.uri) },
+                ...googleProviderMetadata({ interactionId })
+              });
+            }
           }
         }
         break;
@@ -5602,7 +5722,7 @@ function parseGoogleInteractionsOutputs({
           type: "tool-call",
           toolCallId: call.id,
           toolName: call.name,
-          input: JSON.stringify((_e = call.arguments) != null ? _e : {}),
+          input: JSON.stringify((_g = call.arguments) != null ? _g : {}),
           ...googleProviderMetadata({
             signature: call.signature,
             interactionId
@@ -5613,23 +5733,23 @@ function parseGoogleInteractionsOutputs({
       default: {
         if (BUILTIN_TOOL_CALL_TYPES2.has(type)) {
           const call = step;
-          const toolName = type === "mcp_server_tool_call" ? (_f = call.name) != null ? _f : "mcp_server_tool" : builtinToolNameFromCallType2(type);
-          const input = JSON.stringify((_g = call.arguments) != null ? _g : {});
+          const toolName = type === "mcp_server_tool_call" ? (_h = call.name) != null ? _h : "mcp_server_tool" : builtinToolNameFromCallType2(type);
+          const input = JSON.stringify((_i = call.arguments) != null ? _i : {});
           content.push({
             type: "tool-call",
-            toolCallId: (_h = call.id) != null ? _h : generateId3(),
+            toolCallId: (_j = call.id) != null ? _j : generateId3(),
             toolName,
             input,
             providerExecuted: true
           });
         } else if (BUILTIN_TOOL_RESULT_TYPES2.has(type)) {
           const result = step;
-          const toolName = type === "mcp_server_tool_result" ? (_i = result.name) != null ? _i : "mcp_server_tool" : builtinToolNameFromResultType2(type);
+          const toolName = type === "mcp_server_tool_result" ? (_k = result.name) != null ? _k : "mcp_server_tool" : builtinToolNameFromResultType2(type);
           content.push({
             type: "tool-result",
-            toolCallId: (_j = result.call_id) != null ? _j : generateId3(),
+            toolCallId: (_l = result.call_id) != null ? _l : generateId3(),
             toolName,
-            result: (_k = result.result) != null ? _k : null
+            result: (_m = result.result) != null ? _m : null
           });
           const sources = builtinToolResultToSources({
             block: step,
@@ -6542,10 +6662,14 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
       raw: response.status
     };
     const serviceTier = (_e = (_d = response.service_tier) != null ? _d : responseHeaders == null ? void 0 : responseHeaders["x-gemini-service-tier"]) != null ? _e : void 0;
+    const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(
+      response.usage
+    );
     const providerMetadata = {
       google: {
         ...interactionId != null ? { interactionId } : {},
-        ...serviceTier != null ? { serviceTier } : {}
+        ...serviceTier != null ? { serviceTier } : {},
+        ...outputTokensByModality != null ? { outputTokensByModality } : {}
       }
     };
     let timestamp;