@ai-sdk/google 3.0.89 → 3.0.91

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "3.0.89",
+  "version": "3.0.91",
   "license": "Apache-2.0",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -36,8 +36,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "3.0.13",
-    "@ai-sdk/provider-utils": "4.0.36"
+    "@ai-sdk/provider": "3.0.14",
+    "@ai-sdk/provider-utils": "4.0.38"
   },
   "devDependencies": {
     "@types/node": "20.17.24",

package/src/convert-to-google-generative-ai-messages.ts

@@ -188,6 +188,9 @@ function appendLegacyToolResultParts(
         });
         break;
       case 'image-data':
+      case 'file-data': {
+        const topLevelMediaType = String(contentPart.mediaType).split('/')[0];
+
         parts.push(
           {
             inlineData: {
@@ -196,10 +199,14 @@ function appendLegacyToolResultParts(
             },
           },
           {
-            text: 'Tool executed successfully and returned this image as a response',
+            text:
+              `Tool executed successfully and returned this ` +
+              `${topLevelMediaType === 'image' ? 'image' : 'file'} ` +
+              `as a response`,
           },
         );
         break;
+      }
       default:
         parts.push({ text: JSON.stringify(contentPart) });
         break;
@@ -510,7 +517,7 @@ export function convertToGoogleGenerativeAIMessages(
                   name: part.toolName,
                   content:
                     output.type === 'execution-denied'
-                      ? (output.reason ?? 'Tool execution denied.')
+                      ? (output.reason ?? 'Tool call execution denied.')
                       : output.value,
                 },
               },

package/src/google-generative-ai-language-model.ts

@@ -50,6 +50,13 @@ import {
 } from './google-json-accumulator';
 import { mapGoogleGenerativeAIFinishReason } from './map-google-generative-ai-finish-reason';
 
+const configurableSafetySettingCategories = [
+  'HARM_CATEGORY_HATE_SPEECH',
+  'HARM_CATEGORY_DANGEROUS_CONTENT',
+  'HARM_CATEGORY_HARASSMENT',
+  'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+] as const;
+
 type GoogleGenerativeAIConfig = {
   provider: string;
   baseURL: string;
@@ -223,6 +230,16 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV3 {
         ? (googleOptions?.streamFunctionCallArguments ?? false)
         : undefined;
 
+    const safetyThreshold = googleOptions?.threshold;
+    const safetySettings =
+      googleOptions?.safetySettings ??
+      (safetyThreshold != null
+        ? configurableSafetySettingCategories.map(category => ({
+            category,
+            threshold: safetyThreshold,
+          }))
+        : undefined);
+
     const toolConfig =
       googleToolConfig ||
       streamFunctionCallArguments ||
@@ -282,7 +299,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV3 {
         },
         contents,
         systemInstruction: isGemmaModel ? undefined : systemInstruction,
-        safetySettings: googleOptions?.safetySettings,
+        safetySettings,
         tools: googleTools,
         toolConfig,
         cachedContent: googleOptions?.cachedContent,

package/src/google-generative-ai-video-model.ts

@@ -110,13 +110,11 @@ function convertFileToGoogleImage(
       ? file.data
       : convertUint8ArrayToBase64(file.data);
 
-  // The Gemini Developer API (generativelanguage.googleapis.com) requires
-  // inline image bytes wrapped as `inlineData`.
+  // Veo's predictLongRunning endpoint uses Vertex-style image payloads, not
+  // Gemini generateContent inlineData.
   return {
-    inlineData: {
-      mimeType: file.mediaType || 'image/png',
-      data: base64Data,
-    },
+    bytesBase64Encoded: base64Data,
+    mimeType: file.mediaType || 'image/png',
   };
 }
 
@@ -125,16 +123,19 @@ function convertProviderReferenceImage(
 ): Record<string, unknown> {
   if (refImg.bytesBase64Encoded) {
     return {
-      inlineData: {
+      image: {
+        bytesBase64Encoded: refImg.bytesBase64Encoded,
         mimeType: 'image/png',
-        data: refImg.bytesBase64Encoded,
       },
     };
   }
 
   if (refImg.gcsUri) {
     return {
-      gcsUri: refImg.gcsUri,
+      image: {
+        gcsUri: refImg.gcsUri,
+        mimeType: 'image/png',
+      },
     };
   }
 
@@ -146,7 +147,7 @@ function convertInputReferenceImage(
   warnings: SharedV3Warning[],
 ): Record<string, unknown> | undefined {
   const image = convertFileToGoogleImage(file, warnings);
-  return image != null ? { image, referenceType: 'asset' } : undefined;
+  return image != null ? { image } : undefined;
 }
 
 export class GoogleGenerativeAIVideoModel implements Experimental_VideoModelV3 {

package/src/google-provider.ts

@@ -141,6 +141,37 @@ export interface GoogleGenerativeAIProviderSettings {
   name?: string;
 }
 
+const supportedExternalUrlMediaTypes = [
+  'text/html',
+  'text/css',
+  'text/plain',
+  'text/xml',
+  'text/csv',
+  'text/rtf',
+  'text/javascript',
+  'application/json',
+  'application/pdf',
+  'image/bmp',
+  'image/jpeg',
+  'image/png',
+  'image/webp',
+  'video/mp4',
+  'video/mpeg',
+  'video/quicktime',
+  'video/avi',
+  'video/x-flv',
+  'video/mpg',
+  'video/webm',
+  'video/wmv',
+  'video/3gpp',
+];
+
+const externalHttpsUrlPattern = /^https:\/\/.*$/;
+
+function supportsExternalFileUrls(modelId: string) {
+  return /(^|\/)gemini-/.test(modelId) && !/(^|\/)gemini-2\.0/.test(modelId);
+}
+
 /**
  * Create a Google Generative AI provider instance.
  */
@@ -183,6 +214,14 @@ export function createGoogleGenerativeAI(
           ),
           new RegExp(`^https://youtu\\.be/[\\w-]+(?:\\?[\\w=&.-]*)?$`),
         ],
+        ...(supportsExternalFileUrls(modelId)
+          ? Object.fromEntries(
+              supportedExternalUrlMediaTypes.map(mediaType => [
+                mediaType,
+                [externalHttpsUrlPattern],
+              ]),
+            )
+          : {}),
       }),
       fetch: options.fetch,
     });

package/src/interactions/build-google-interactions-stream-transform.ts

@@ -11,7 +11,10 @@ import type {
   GoogleInteractionsEvent,
   GoogleInteractionsUsage,
 } from './google-interactions-api';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import {
   annotationsToSources,
   builtinToolResultToSources,
@@ -490,6 +493,47 @@ export function buildGoogleInteractionsStreamTransform({
             break;
           }
 
+          /*
+           * `video` deltas inside `model_output` carry the full payload in a
+           * single chunk (no per-byte streaming), mirroring `image`. Emit the
+           * `file` part as soon as the delta arrives so it surfaces regardless
+           * of whether a text block is currently open at the same index.
+           */
+          if (
+            dtype === 'video' &&
+            (open.kind === 'pending_model_output' || open.kind === 'text')
+          ) {
+            const videoDelta = event.delta as
+              | { data?: string; mime_type?: string; uri?: string }
+              | undefined;
+            const google: Record<string, string> = {};
+            if (interactionId != null) google.interactionId = interactionId;
+            const providerMetadata =
+              Object.keys(google).length > 0 ? { google } : undefined;
+            if (videoDelta?.data != null && videoDelta.data.length > 0) {
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: videoDelta.data,
+                ...(providerMetadata ? { providerMetadata } : {}),
+              });
+            } else if (videoDelta?.uri != null && videoDelta.uri.length > 0) {
+              const uriProviderMetadata = {
+                google: {
+                  ...(interactionId != null ? { interactionId } : {}),
+                  videoUri: videoDelta.uri,
+                },
+              };
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: '',
+                providerMetadata: uriProviderMetadata,
+              });
+            }
+            break;
+          }
+
           const delta = event.delta as
             | {
                 type?: string;
@@ -824,10 +868,14 @@ export function buildGoogleInteractionsStreamTransform({
         raw: finishStatus,
       };
 
+      const outputTokensByModality =
+        getGoogleInteractionsOutputTokensByModality(usage);
+
       const providerMetadata: SharedV3ProviderMetadata = {
         google: {
           ...(interactionId != null ? { interactionId } : {}),
           ...(serviceTier != null ? { serviceTier } : {}),
+          ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
         },
       };
 

package/src/interactions/convert-google-interactions-usage.ts

@@ -45,3 +45,25 @@ export function convertGoogleInteractionsUsage(
     raw: usage as unknown as JSONObject,
   };
 }
+
+/**
+ * Extracts the per-modality output token breakdown from an Interactions usage
+ * record (e.g. `{ video: 57920, text: 12 }`).
+ */
+export function getGoogleInteractionsOutputTokensByModality(
+  usage: GoogleInteractionsUsage | undefined | null,
+): Record<string, number> | undefined {
+  const byModality = usage?.output_tokens_by_modality;
+  if (byModality == null) {
+    return undefined;
+  }
+
+  const result: Record<string, number> = {};
+  for (const entry of byModality) {
+    if (entry?.modality != null && entry.tokens != null) {
+      result[entry.modality] = entry.tokens;
+    }
+  }
+
+  return Object.keys(result).length > 0 ? result : undefined;
+}

package/src/interactions/google-interactions-api.ts

@@ -140,9 +140,19 @@ const contentBlockSchema = () => {
     })
     .loose();
 
+  const videoContent = z
+    .object({
+      type: z.literal('video'),
+      data: z.string().nullish(),
+      mime_type: z.string().nullish(),
+      uri: z.string().nullish(),
+    })
+    .loose();
+
   return z.union([
     textContent,
     imageContent,
+    videoContent,
     z.object({ type: z.string() }).loose(),
   ]);
 };
@@ -384,6 +394,19 @@ export const googleInteractionsEventSchema = lazySchema(() =>
         })
         .loose();
 
+      /*
+       * `video` deltas carry the entire payload per delta (`data` base64 +
+       * `mime_type`, or `uri`) — there is no per-byte streaming.
+       */
+      const stepDeltaVideo = z
+        .object({
+          type: z.literal('video'),
+          data: z.string().nullish(),
+          mime_type: z.string().nullish(),
+          uri: z.string().nullish(),
+        })
+        .loose();
+
       /*
        * Built-in tool call/result step deltas mirror the shape of their step
        * counterparts (full payload per delta — there is no per-token
@@ -419,6 +442,7 @@ export const googleInteractionsEventSchema = lazySchema(() =>
       const stepDeltaUnion = z.union([
         stepDeltaText,
         stepDeltaImage,
+        stepDeltaVideo,
         stepDeltaThoughtSummary,
         stepDeltaThoughtSignature,
         stepDeltaArgumentsDelta,

package/src/interactions/google-interactions-language-model.ts

@@ -20,7 +20,10 @@ import {
 } from '@ai-sdk/provider-utils';
 import { googleFailedResponseHandler } from '../google-error';
 import { buildGoogleInteractionsStreamTransform } from './build-google-interactions-stream-transform';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import { convertToGoogleInteractionsInput } from './convert-to-google-interactions-input';
 import {
   googleInteractionsEventSchema,
@@ -519,10 +522,15 @@ export class GoogleInteractionsLanguageModel implements LanguageModelV3 {
      * `response.id` is omitted when `store: false` (fully stateless mode), so
      * `interactionId` is only surfaced when the API actually returned one.
      */
+    const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(
+      response.usage,
+    );
+
     const providerMetadata: SharedV3ProviderMetadata = {
       google: {
         ...(interactionId != null ? { interactionId } : {}),
         ...(serviceTier != null ? { serviceTier } : {}),
+        ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
       },
     };
 

package/src/interactions/google-interactions-provider-metadata.ts

@@ -15,6 +15,14 @@ export type GoogleInteractionsProviderMetadata = {
    */
   serviceTier?: string;
 
+  /**
+   * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+   * from the Interactions API `output_tokens_by_modality`. Present only when
+   * the response reports a breakdown. Preview surface for per-modality billing;
+   * may be promoted to a first-class usage field later.
+   */
+  outputTokensByModality?: Record<string, number>;
+
   /**
    * Per-block signature hash for backend validation. Set by the SDK on output
    * reasoning / tool-call parts and round-tripped on input parts.

package/src/interactions/parse-google-interactions-outputs.ts

@@ -68,7 +68,7 @@ function builtinToolNameFromResultType(type: string): string {
  * `LanguageModelV3Content[]`. Surfaces:
  *
  * - `model_output` steps: iterates `step.content[]` for `text` (with
- *   annotations → source parts) and `image` content blocks.
+ *   annotations → source parts), `image`, and `video` content blocks.
  * - `thought` steps: emits a single `reasoning` part from `summary[*]`.
  * - `function_call` steps: emits a `tool-call` part directly.
  * - Built-in tool `*_call` / `*_result` steps (Google Search, Code Execution,
@@ -162,6 +162,38 @@ export function parseGoogleInteractionsOutputs({
                 },
               });
             }
+          } else if (blockType === 'video') {
+            const video = block as {
+              data?: string;
+              mime_type?: string;
+              uri?: string;
+            };
+            if (video.data != null && video.data.length > 0) {
+              content.push({
+                type: 'file',
+                mediaType: video.mime_type ?? 'video/mp4',
+                data: video.data,
+                ...googleProviderMetadata({ interactionId }),
+              });
+            } else if (video.uri != null && video.uri.length > 0) {
+              /*
+               * V3 `LanguageModelV3File` only supports inline data (`string` /
+               * `Uint8Array`). URL-only video outputs cannot be represented as
+               * a file content part on the v3 spec; surface the URI through
+               * provider metadata so callers can still recover it.
+               */
+              content.push({
+                type: 'file',
+                mediaType: video.mime_type ?? 'video/mp4',
+                data: '',
+                providerMetadata: {
+                  google: {
+                    ...(interactionId != null ? { interactionId } : {}),
+                    videoUri: video.uri,
+                  },
+                },
+              });
+            }
           }
         }
         break;

package/src/internal/index.ts

@@ -1,3 +1,9 @@
 export * from '../google-generative-ai-language-model';
 export { googleTools } from '../google-tools';
 export type { GoogleGenerativeAIModelId } from '../google-generative-ai-options';
+export {
+  GoogleInteractionsLanguageModel,
+  type GoogleInteractionsModelInput,
+} from '../interactions/google-interactions-language-model';
+export type { GoogleInteractionsModelId } from '../interactions/google-interactions-language-model-options';
+export type { GoogleInteractionsAgentName } from '../interactions/google-interactions-agent';