@ai-sdk/google 3.0.127 → 3.0.129

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "3.0.127",
+  "version": "3.0.129",
   "license": "Apache-2.0",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -36,8 +36,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "3.0.17",
-    "@ai-sdk/provider-utils": "4.0.54"
+    "@ai-sdk/provider": "3.0.18",
+    "@ai-sdk/provider-utils": "4.0.56"
   },
   "devDependencies": {
     "@types/node": "20.17.24",

package/src/convert-to-google-generative-ai-messages.ts

@@ -3,7 +3,11 @@ import {
   type LanguageModelV3Prompt,
   type SharedV3Warning,
 } from '@ai-sdk/provider';
-import { convertToBase64, secureJsonParse } from '@ai-sdk/provider-utils';
+import {
+  convertToBase64,
+  isUrlSupported,
+  secureJsonParse,
+} from '@ai-sdk/provider-utils';
 import type {
   GoogleGenerativeAIContent,
   GoogleGenerativeAIContentPart,
@@ -71,21 +75,32 @@ function parseBase64DataUrl(
 
 function convertUrlToolResultPart(
   url: string,
+  mediaType: string | undefined,
+  supportedFunctionResponseUrls: Record<string, RegExp[]>,
 ): GoogleGenerativeAIFunctionResponsePart | undefined {
-  // Per https://ai.google.dev/api/caching#FunctionResponsePart, only inline data is supported.
-  // https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/function-calling#functionresponsepart suggests that this
-  // may be different for Vertex, but this needs to be confirmed and further tested for both APIs.
   const parsedDataUrl = parseBase64DataUrl(url);
-  if (parsedDataUrl == null) {
-    return undefined;
+  if (parsedDataUrl != null) {
+    return {
+      inlineData: {
+        mimeType: parsedDataUrl.mediaType,
+        data: parsedDataUrl.data,
+      },
+    };
   }
 
-  return {
-    inlineData: {
-      mimeType: parsedDataUrl.mediaType,
-      data: parsedDataUrl.data,
-    },
-  };
+  return mediaType != null &&
+    isUrlSupported({
+      url,
+      mediaType,
+      supportedUrls: supportedFunctionResponseUrls,
+    })
+    ? {
+        fileData: {
+          mimeType: mediaType,
+          fileUri: url,
+        },
+      }
+    : undefined;
 }
 
 /*
@@ -102,6 +117,7 @@ function appendToolResultParts(
   }>,
   toolCallId?: string,
   includeFunctionCallIds = true,
+  supportedFunctionResponseUrls: Record<string, RegExp[]> = {},
 ): void {
   const functionResponseParts: GoogleGenerativeAIFunctionResponsePart[] = [];
   const responseTextParts: string[] = [];
@@ -126,6 +142,8 @@ function appendToolResultParts(
       case 'file-url': {
         const functionResponsePart = convertUrlToolResultPart(
           contentPart.url as string,
+          contentPart.mediaType as string | undefined,
+          supportedFunctionResponseUrls,
         );
 
         if (functionResponsePart != null) {
@@ -241,6 +259,7 @@ export function convertToGoogleGenerativeAIMessages(
      */
     onWarning?: (warning: SharedV3Warning) => void;
     includeFunctionCallIds?: boolean;
+    supportedFunctionResponseUrls?: Record<string, RegExp[]>;
   },
 ): GoogleGenerativeAIPrompt {
   const systemInstructionParts: Array<{ text: string }> = [];
@@ -253,6 +272,8 @@ export function convertToGoogleGenerativeAIMessages(
     options?.supportsFunctionResponseParts ?? true;
   const onWarning = options?.onWarning;
   const includeFunctionCallIds = options?.includeFunctionCallIds ?? true;
+  const supportedFunctionResponseUrls =
+    options?.supportedFunctionResponseUrls ?? {};
 
   let sentinelInjected = false;
   const missingSignatureToolNames: string[] = [];
@@ -523,6 +544,7 @@ export function convertToGoogleGenerativeAIMessages(
                 output.value,
                 part.toolCallId,
                 includeFunctionCallIds,
+                supportedFunctionResponseUrls,
               );
             } else {
               appendLegacyToolResultParts(

package/src/download-tool-result-files.ts

@@ -0,0 +1,103 @@
+import type {
+  LanguageModelV3Prompt,
+  LanguageModelV3ToolResultOutput,
+} from '@ai-sdk/provider';
+import {
+  convertUint8ArrayToBase64,
+  downloadBlob,
+  isUrlSupported,
+} from '@ai-sdk/provider-utils';
+
+/**
+ * Vertex accepts more URLs in user messages than in function responses.
+ * Download tool result files that cannot be referenced in function responses.
+ */
+export async function downloadToolResultFiles(
+  prompt: LanguageModelV3Prompt,
+  {
+    abortSignal,
+    supportedUrls = {},
+  }: {
+    abortSignal: AbortSignal | undefined;
+    supportedUrls?: Record<string, RegExp[]>;
+  },
+): Promise<LanguageModelV3Prompt> {
+  async function downloadOutput(
+    output: LanguageModelV3ToolResultOutput,
+  ): Promise<LanguageModelV3ToolResultOutput> {
+    if (output.type !== 'content') {
+      return output;
+    }
+
+    return {
+      ...output,
+      value: await Promise.all(
+        output.value.map(async part => {
+          if (part.type !== 'file-url' && part.type !== 'image-url') {
+            return part;
+          }
+
+          if (
+            part.type === 'file-url' &&
+            part.mediaType != null &&
+            isUrlSupported({
+              url: part.url,
+              mediaType: part.mediaType,
+              supportedUrls,
+            })
+          ) {
+            return part;
+          }
+
+          const blob = await downloadBlob(part.url, { abortSignal });
+          return {
+            type:
+              part.type === 'file-url'
+                ? ('file-data' as const)
+                : ('image-data' as const),
+            data: convertUint8ArrayToBase64(
+              new Uint8Array(await blob.arrayBuffer()),
+            ),
+            mediaType:
+              blob.type ||
+              (part.type === 'file-url'
+                ? (part.mediaType ?? 'application/octet-stream')
+                : 'image/*'),
+            providerOptions: part.providerOptions,
+          };
+        }),
+      ),
+    };
+  }
+
+  return Promise.all(
+    prompt.map(async message => {
+      switch (message.role) {
+        case 'assistant':
+          return {
+            ...message,
+            content: await Promise.all(
+              message.content.map(async part =>
+                part.type === 'tool-result'
+                  ? { ...part, output: await downloadOutput(part.output) }
+                  : part,
+              ),
+            ),
+          };
+        case 'tool':
+          return {
+            ...message,
+            content: await Promise.all(
+              message.content.map(async part =>
+                part.type === 'tool-result'
+                  ? { ...part, output: await downloadOutput(part.output) }
+                  : part,
+              ),
+            ),
+          };
+        default:
+          return message;
+      }
+    }),
+  );
+}

package/src/google-generative-ai-language-model.ts

@@ -33,6 +33,7 @@ import {
 } from './convert-google-generative-ai-usage';
 import { convertJSONSchemaToOpenAPISchema } from './convert-json-schema-to-openapi-schema';
 import { convertToGoogleGenerativeAIMessages } from './convert-to-google-generative-ai-messages';
+import { downloadToolResultFiles } from './download-tool-result-files';
 import { getModelPath } from './get-model-path';
 import { googleFailedResponseHandler } from './google-error';
 import {
@@ -60,6 +61,14 @@ const configurableSafetySettingCategories = [
 
 const gemini25ModelPattern = /(^|\/)gemini-2\.5(?:[.-]|$)/i;
 
+const googleCloudStorageFunctionResponseUrls = {
+  'image/png': [/^gs:\/\/.*$/],
+  'image/jpeg': [/^gs:\/\/.*$/],
+  'image/webp': [/^gs:\/\/.*$/],
+  'application/pdf': [/^gs:\/\/.*$/],
+  'text/plain': [/^gs:\/\/.*$/],
+} satisfies Record<string, RegExp[]>;
+
 type GoogleGenerativeAIConfig = {
   provider: string;
   baseURL: string;
@@ -113,6 +122,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV3 {
       tools,
       toolChoice,
       providerOptions,
+      abortSignal,
     }: LanguageModelV3CallOptions,
     { isStreaming = false }: { isStreaming?: boolean } = {},
   ) {
@@ -249,9 +259,19 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV3 {
     }
 
     const { usesGemini3Features } = getGoogleModelCapabilities(this.modelId);
+    const supportedFunctionResponseUrls =
+      usesGemini3Features && isVertexProvider
+        ? googleCloudStorageFunctionResponseUrls
+        : undefined;
+    const promptWithDownloadedToolResultFiles = isVertexProvider
+      ? await downloadToolResultFiles(prompt, {
+          abortSignal,
+          supportedUrls: supportedFunctionResponseUrls,
+        })
+      : prompt;
 
     const { contents, systemInstruction } = convertToGoogleGenerativeAIMessages(
-      prompt,
+      promptWithDownloadedToolResultFiles,
       {
         isGemmaModel,
         isGemini3Model: usesGemini3Features,
@@ -259,6 +279,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV3 {
         supportsFunctionResponseParts: usesGemini3Features,
         onWarning: warning => warnings.push(warning),
         includeFunctionCallIds: !isVertexProvider,
+        supportedFunctionResponseUrls,
       },
     );
 

package/src/google-generative-ai-prompt.ts

@@ -53,9 +53,13 @@ export type GoogleGenerativeAIContentPart =
       thoughtSignature?: string;
     };
 
-export type GoogleGenerativeAIFunctionResponsePart = {
-  inlineData: { mimeType: string; data: string };
-};
+export type GoogleGenerativeAIFunctionResponsePart =
+  | {
+      inlineData: { mimeType: string; data: string };
+    }
+  | {
+      fileData: { mimeType: string; fileUri: string };
+    };
 
 export type GoogleGenerativeAIGroundingMetadata = GroundingMetadataSchema;