@ai-sdk/google 4.0.2 → 4.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.2",
+  "version": "4.0.4",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -35,16 +35,16 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.0",
-    "@ai-sdk/provider-utils": "5.0.1"
+    "@ai-sdk/provider": "4.0.1",
+    "@ai-sdk/provider-utils": "5.0.2"
   },
   "devDependencies": {
     "@types/node": "22.19.19",
     "tsup": "^8.5.1",
     "typescript": "5.8.3",
     "zod": "3.25.76",
-    "@vercel/ai-tsconfig": "0.0.0",
-    "@ai-sdk/test-server": "2.0.0"
+    "@ai-sdk/test-server": "2.0.0",
+    "@vercel/ai-tsconfig": "0.0.0"
   },
   "peerDependencies": {
     "zod": "^3.25.76 || ^4.1.8"

package/src/google-video-model.ts

@@ -1,6 +1,7 @@
 import {
   AISDKError,
   type Experimental_VideoModelV4,
+  type Experimental_VideoModelV4File,
   type SharedV4Warning,
 } from '@ai-sdk/provider';
 import {
@@ -35,6 +36,103 @@ interface GoogleVideoModelConfig {
   };
 }
 
+function getFirstFrameImage(
+  options: Parameters<Experimental_VideoModelV4['doGenerate']>[0],
+): Experimental_VideoModelV4File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'first_frame')
+    ?.image;
+}
+
+function resolveStartImage(
+  options: Parameters<Experimental_VideoModelV4['doGenerate']>[0],
+): Experimental_VideoModelV4File | undefined {
+  return getFirstFrameImage(options) ?? options.image;
+}
+
+function getLastFrameImage(
+  options: Parameters<Experimental_VideoModelV4['doGenerate']>[0],
+): Experimental_VideoModelV4File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'last_frame')
+    ?.image;
+}
+
+function getInputReferences(
+  options: Parameters<Experimental_VideoModelV4['doGenerate']>[0],
+): Array<Experimental_VideoModelV4File> | undefined {
+  if (options.frameImages != null && options.frameImages.length > 0) {
+    return undefined;
+  }
+
+  return options.inputReferences != null && options.inputReferences.length > 0
+    ? options.inputReferences
+    : undefined;
+}
+
+function convertFileToGoogleImage(
+  file: Experimental_VideoModelV4File,
+  warnings: SharedV4Warning[],
+): Record<string, unknown> | undefined {
+  if (file.type === 'url') {
+    if (file.url.startsWith('gs://')) {
+      return {
+        gcsUri: file.url,
+        mimeType: 'image/png',
+      };
+    }
+
+    warnings.push({
+      type: 'unsupported',
+      feature: 'URL-based image input',
+      details:
+        'Google Generative AI video models require base64-encoded images or GCS URIs. URL will be ignored.',
+    });
+    return undefined;
+  }
+
+  const base64Data =
+    typeof file.data === 'string'
+      ? file.data
+      : convertUint8ArrayToBase64(file.data);
+
+  // The Gemini Developer API (generativelanguage.googleapis.com) requires
+  // inline image bytes wrapped as `inlineData`.
+  return {
+    inlineData: {
+      mimeType: file.mediaType || 'image/png',
+      data: base64Data,
+    },
+  };
+}
+
+function convertProviderReferenceImage(
+  refImg: NonNullable<GoogleVideoModelOptions['referenceImages']>[number],
+): Record<string, unknown> {
+  if (refImg.bytesBase64Encoded) {
+    return {
+      inlineData: {
+        mimeType: 'image/png',
+        data: refImg.bytesBase64Encoded,
+      },
+    };
+  }
+
+  if (refImg.gcsUri) {
+    return {
+      gcsUri: refImg.gcsUri,
+    };
+  }
+
+  return refImg;
+}
+
+function convertInputReferenceImage(
+  file: Experimental_VideoModelV4File,
+  warnings: SharedV4Warning[],
+): Record<string, unknown> | undefined {
+  const image = convertFileToGoogleImage(file, warnings);
+  return image != null ? { image, referenceType: 'asset' } : undefined;
+}
+
 export class GoogleVideoModel implements Experimental_VideoModelV4 {
   readonly specificationVersion = 'v4';
 
@@ -71,46 +169,32 @@ export class GoogleVideoModel implements Experimental_VideoModelV4 {
       instance.prompt = options.prompt;
     }
 
-    // Handle image-to-video: convert image to base64
-    if (options.image != null) {
-      if (options.image.type === 'url') {
-        warnings.push({
-          type: 'unsupported',
-          feature: 'URL-based image input',
-          details:
-            'Google Generative AI video models require base64-encoded images. URL will be ignored.',
-        });
-      } else {
-        const base64Data =
-          typeof options.image.data === 'string'
-            ? options.image.data
-            : convertUint8ArrayToBase64(options.image.data);
-
-        instance.image = {
-          inlineData: {
-            mimeType: options.image.mediaType || 'image/png',
-            data: base64Data,
-          },
-        };
+    const startImage = resolveStartImage(options);
+    if (startImage != null) {
+      const image = convertFileToGoogleImage(startImage, warnings);
+      if (image != null) {
+        instance.image = image;
       }
     }
 
-    if (googleOptions?.referenceImages != null) {
-      instance.referenceImages = googleOptions.referenceImages.map(refImg => {
-        if (refImg.bytesBase64Encoded) {
-          return {
-            inlineData: {
-              mimeType: 'image/png',
-              data: refImg.bytesBase64Encoded,
-            },
-          };
-        } else if (refImg.gcsUri) {
-          return {
-            gcsUri: refImg.gcsUri,
-          };
-        }
-        return refImg;
+    const lastFrameImage = getLastFrameImage(options);
+    if (lastFrameImage != null) {
+      const lastFrame = convertFileToGoogleImage(lastFrameImage, warnings);
+      if (lastFrame != null) {
+        instance.lastFrame = lastFrame;
+      }
+    }
+
+    const inputReferences = getInputReferences(options);
+    if (inputReferences != null) {
+      instance.referenceImages = inputReferences.flatMap(reference => {
+        const converted = convertInputReferenceImage(reference, warnings);
+        return converted != null ? [converted] : [];
       });
+    } else if (googleOptions?.referenceImages != null) {
+      instance.referenceImages = googleOptions.referenceImages.map(refImg =>
+        convertProviderReferenceImage(refImg),
+      );
     }
 
     const parameters: Record<string, unknown> = {

package/src/interactions/build-google-interactions-stream-transform.ts

@@ -11,7 +11,10 @@ import type {
   GoogleInteractionsEvent,
   GoogleInteractionsUsage,
 } from './google-interactions-api';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import {
   annotationsToSources,
   builtinToolResultToSources,
@@ -478,6 +481,41 @@ export function buildGoogleInteractionsStreamTransform({
             break;
           }
 
+          /*
+           * `video` deltas inside `model_output` carry the full payload in a
+           * single chunk (no per-byte streaming), mirroring `image`. Emit the
+           * `file` part as soon as the delta arrives so it surfaces regardless
+           * of whether a text block is currently open at the same index.
+           */
+          if (
+            dtype === 'video' &&
+            (open.kind === 'pending_model_output' || open.kind === 'text')
+          ) {
+            const videoDelta = event.delta as
+              | { data?: string; mime_type?: string; uri?: string }
+              | undefined;
+            const google: Record<string, string> = {};
+            if (interactionId != null) google.interactionId = interactionId;
+            const providerMetadata =
+              Object.keys(google).length > 0 ? { google } : undefined;
+            if (videoDelta?.data != null && videoDelta.data.length > 0) {
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: { type: 'data', data: videoDelta.data },
+                ...(providerMetadata ? { providerMetadata } : {}),
+              });
+            } else if (videoDelta?.uri != null && videoDelta.uri.length > 0) {
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: { type: 'url', url: new URL(videoDelta.uri) },
+                ...(providerMetadata ? { providerMetadata } : {}),
+              });
+            }
+            break;
+          }
+
           const delta = event.delta as
             | {
                 type?: string;
@@ -800,10 +838,14 @@ export function buildGoogleInteractionsStreamTransform({
         raw: finishStatus,
       };
 
+      const outputTokensByModality =
+        getGoogleInteractionsOutputTokensByModality(usage);
+
       const providerMetadata: SharedV4ProviderMetadata = {
         google: {
           ...(interactionId != null ? { interactionId } : {}),
           ...(serviceTier != null ? { serviceTier } : {}),
+          ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
         },
       };
 

package/src/interactions/convert-google-interactions-usage.ts

@@ -45,3 +45,25 @@ export function convertGoogleInteractionsUsage(
     raw: usage as unknown as JSONObject,
   };
 }
+
+/**
+ * Extracts the per-modality output token breakdown from an Interactions usage
+ * record (e.g. `{ video: 57920, text: 12 }`).
+ */
+export function getGoogleInteractionsOutputTokensByModality(
+  usage: GoogleInteractionsUsage | undefined | null,
+): Record<string, number> | undefined {
+  const byModality = usage?.output_tokens_by_modality;
+  if (byModality == null) {
+    return undefined;
+  }
+
+  const result: Record<string, number> = {};
+  for (const entry of byModality) {
+    if (entry?.modality != null && entry.tokens != null) {
+      result[entry.modality] = entry.tokens;
+    }
+  }
+
+  return Object.keys(result).length > 0 ? result : undefined;
+}

package/src/interactions/google-interactions-api.ts

@@ -140,9 +140,19 @@ const contentBlockSchema = () => {
     })
     .loose();
 
+  const videoContent = z
+    .object({
+      type: z.literal('video'),
+      data: z.string().nullish(),
+      mime_type: z.string().nullish(),
+      uri: z.string().nullish(),
+    })
+    .loose();
+
   return z.union([
     textContent,
     imageContent,
+    videoContent,
     z.object({ type: z.string() }).loose(),
   ]);
 };
@@ -384,6 +394,19 @@ export const googleInteractionsEventSchema = lazySchema(() =>
         })
         .loose();
 
+      /*
+       * `video` deltas carry the entire payload per delta (`data` base64 +
+       * `mime_type`, or `uri`) — there is no per-byte streaming.
+       */
+      const stepDeltaVideo = z
+        .object({
+          type: z.literal('video'),
+          data: z.string().nullish(),
+          mime_type: z.string().nullish(),
+          uri: z.string().nullish(),
+        })
+        .loose();
+
       /*
        * Built-in tool call/result step deltas mirror the shape of their step
        * counterparts (full payload per delta — there is no per-token
@@ -419,6 +442,7 @@ export const googleInteractionsEventSchema = lazySchema(() =>
       const stepDeltaUnion = z.union([
         stepDeltaText,
         stepDeltaImage,
+        stepDeltaVideo,
         stepDeltaThoughtSummary,
         stepDeltaThoughtSignature,
         stepDeltaArgumentsDelta,

package/src/interactions/google-interactions-language-model.ts

@@ -23,7 +23,10 @@ import {
 } from '@ai-sdk/provider-utils';
 import { googleFailedResponseHandler } from '../google-error';
 import { buildGoogleInteractionsStreamTransform } from './build-google-interactions-stream-transform';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import { convertToGoogleInteractionsInput } from './convert-to-google-interactions-input';
 import {
   googleInteractionsEventSchema,
@@ -546,10 +549,15 @@ export class GoogleInteractionsLanguageModel implements LanguageModelV4 {
      * `response.id` is omitted when `store: false` (fully stateless mode), so
      * `interactionId` is only surfaced when the API actually returned one.
      */
+    const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(
+      response.usage,
+    );
+
     const providerMetadata: SharedV4ProviderMetadata = {
       google: {
         ...(interactionId != null ? { interactionId } : {}),
         ...(serviceTier != null ? { serviceTier } : {}),
+        ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
       },
     };
 

package/src/interactions/google-interactions-provider-metadata.ts

@@ -15,6 +15,14 @@ export type GoogleInteractionsProviderMetadata = {
    */
   serviceTier?: string;
 
+  /**
+   * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+   * from the Interactions API `output_tokens_by_modality`. Present only when
+   * the response reports a breakdown. Preview surface for per-modality billing;
+   * may be promoted to a first-class usage field later.
+   */
+  outputTokensByModality?: Record<string, number>;
+
   /**
    * Per-block signature hash for backend validation. Set by the SDK on output
    * reasoning / tool-call parts and round-tripped on input parts.

package/src/interactions/parse-google-interactions-outputs.ts

@@ -151,6 +151,27 @@ export function parseGoogleInteractionsOutputs({
                 ...googleProviderMetadata({ interactionId }),
               });
             }
+          } else if (blockType === 'video') {
+            const video = block as {
+              data?: string;
+              mime_type?: string;
+              uri?: string;
+            };
+            if (video.data != null && video.data.length > 0) {
+              content.push({
+                type: 'file',
+                mediaType: video.mime_type ?? 'video/mp4',
+                data: { type: 'data', data: video.data },
+                ...googleProviderMetadata({ interactionId }),
+              });
+            } else if (video.uri != null && video.uri.length > 0) {
+              content.push({
+                type: 'file',
+                mediaType: video.mime_type ?? 'video/mp4',
+                data: { type: 'url', url: new URL(video.uri) },
+                ...googleProviderMetadata({ interactionId }),
+              });
+            }
           }
         }
         break;