@ai-sdk/google 4.0.67 → 4.0.69

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.67",
+  "version": "4.0.69",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -35,8 +35,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.13",
-    "@ai-sdk/provider-utils": "5.0.39"
+    "@ai-sdk/provider": "4.0.14",
+    "@ai-sdk/provider-utils": "5.0.40"
   },
   "devDependencies": {
     "@ai-sdk/test-server": "2.0.1",

package/src/download-tool-result-files.ts

@@ -0,0 +1,124 @@
+import type {
+  LanguageModelV4Prompt,
+  LanguageModelV4ToolResultOutput,
+} from '@ai-sdk/provider';
+import {
+  detectMediaType,
+  downloadBlob,
+  isFullMediaType,
+} from '@ai-sdk/provider-utils';
+
+/**
+ * Vertex function responses only accept inline file data. Download remote tool
+ * result files before converting the prompt to the Google request format.
+ */
+export async function downloadToolResultFiles(
+  prompt: LanguageModelV4Prompt,
+  {
+    abortSignal,
+    maxBytes,
+  }: {
+    abortSignal: AbortSignal | undefined;
+    maxBytes: number;
+  },
+): Promise<LanguageModelV4Prompt> {
+  const result: LanguageModelV4Prompt = [];
+
+  for (const message of prompt) {
+    if (message.role === 'assistant') {
+      const content: typeof message.content = [];
+
+      for (const part of message.content) {
+        content.push(
+          part.type === 'tool-result'
+            ? {
+                ...part,
+                output: await downloadToolResultOutput(part.output, {
+                  abortSignal,
+                  maxBytes,
+                }),
+              }
+            : part,
+        );
+      }
+
+      result.push({ ...message, content });
+      continue;
+    }
+
+    if (message.role === 'tool') {
+      const content: typeof message.content = [];
+
+      for (const part of message.content) {
+        if (part.type !== 'tool-result') {
+          content.push(part);
+          continue;
+        }
+
+        content.push({
+          ...part,
+          output: await downloadToolResultOutput(part.output, {
+            abortSignal,
+            maxBytes,
+          }),
+        });
+      }
+
+      result.push({ ...message, content });
+      continue;
+    }
+
+    result.push(message);
+  }
+
+  return result;
+}
+
+async function downloadToolResultOutput(
+  output: LanguageModelV4ToolResultOutput,
+  {
+    abortSignal,
+    maxBytes,
+  }: {
+    abortSignal: AbortSignal | undefined;
+    maxBytes: number;
+  },
+): Promise<LanguageModelV4ToolResultOutput> {
+  if (output.type !== 'content') {
+    return output;
+  }
+
+  const value: typeof output.value = [];
+
+  for (const part of output.value) {
+    if (part.type !== 'file' || part.data.type !== 'url') {
+      value.push(part);
+      continue;
+    }
+
+    const blob = await downloadBlob(part.data.url.toString(), {
+      abortSignal,
+      maxBytes,
+    });
+    const data = new Uint8Array(await blob.arrayBuffer());
+    const detectedMediaType = detectMediaType({
+      data,
+      topLevelType: 'image',
+    });
+
+    value.push({
+      ...part,
+      data: { type: 'data' as const, data },
+      mediaType:
+        detectedMediaType ??
+        (blob.type && !isFullMediaType(part.mediaType)
+          ? blob.type
+          : part.mediaType),
+    });
+  }
+
+  return {
+    ...output,
+    value,
+  };
+}

package/src/google-batch.ts

@@ -1,6 +1,7 @@
 import {
   InvalidArgumentError,
   InvalidResponseDataError,
+  UnsupportedFunctionalityError,
   type Experimental_BatchV4 as BatchV4,
   type Experimental_BatchV4CancelResult as BatchV4CancelResult,
   type Experimental_BatchV4Error as BatchV4Error,
@@ -10,9 +11,12 @@ import {
   type Experimental_BatchV4OperationOptions as BatchV4OperationOptions,
   type Experimental_BatchV4StartResult as BatchV4StartResult,
   type Experimental_BatchV4Status as BatchV4Status,
-  type Experimental_TextBatchV4Request as TextBatchV4Request,
+  type Experimental_ImageBatchV4Request as ImageBatchV4Request,
   type Experimental_BatchV4StartOptions as BatchV4StartOptions,
   type LanguageModelV4GenerateResult,
+  type ImageModelV4Result,
+  type LanguageModelV4Prompt,
+  type SharedV4Warning,
 } from '@ai-sdk/provider';
 import {
   combineHeaders,
@@ -23,11 +27,13 @@ import {
   getFromApi,
   lazySchema,
   normalizeBatchRequestCounts,
+  parseProviderOptions,
   postJsonToApi,
   postToApi,
   resolve,
   safeValidateTypes,
   zodSchema,
+  convertToBase64,
   type InferSchema,
   type ResponseHandler,
 } from '@ai-sdk/provider-utils';
@@ -39,7 +45,12 @@ import {
   responseSchema,
   type GoogleLanguageModelConfig,
 } from './google-language-model';
-import type { GoogleModelId } from './google-language-model-options';
+import {
+  googleLanguageModelOptions,
+  type GoogleModelId,
+} from './google-language-model-options';
+import type { GoogleImageModelId } from './google-image-settings';
+import { googleImageModelOptionsSchema } from './google-image-model-options';
 
 const googleBatchInputFileMaxBytes = 2 * 1024 * 1024 * 1024;
 const googleBatchInlineCreationMaxBytes = 20_000_000;
@@ -47,7 +58,25 @@ const supportedGoogleBatchContentTypes = new Set<
   LanguageModelV4GenerateResult['content'][number]['type']
 >(['text', 'reasoning', 'source', 'tool-call', 'tool-result']);
 
-type GoogleBatchRequest = TextBatchV4Request<GoogleModelId>;
+type GoogleImageBatchRequest = ImageBatchV4Request<GoogleImageModelId>;
+type GoogleBatchModelIds = {
+  readonly text: GoogleModelId;
+  readonly image: GoogleImageModelId;
+};
+
+function assertSupportedBatchRequests(
+  requests: BatchV4StartOptions<GoogleBatchModelIds>['requests'],
+) {
+  for (const request of requests) {
+    const requestType = request.type;
+    if (requestType !== 'text' && requestType !== 'image') {
+      throw new UnsupportedFunctionalityError({
+        functionality: `batch request type: ${requestType}`,
+        message: `The Google Batch API does not support batch requests with type "${requestType}".`,
+      });
+    }
+  }
+}
 
 const googleRpcStatusSchema = z.object({
   code: z.union([z.number(), z.string()]).nullish(),
@@ -150,7 +179,7 @@ const googleBatchResponsePreviewSchema = lazySchema(() =>
   ),
 );
 
-export class GoogleBatch implements BatchV4<{ readonly text: GoogleModelId }> {
+export class GoogleBatch implements BatchV4<GoogleBatchModelIds> {
   readonly specificationVersion = 'v4' as const;
   readonly provider: string;
   readonly supportedUrls: Record<string, RegExp[]>;
@@ -169,8 +198,9 @@ export class GoogleBatch implements BatchV4<{ readonly text: GoogleModelId }> {
   }
 
   async doStartBatch(
-    options: BatchV4StartOptions<{ text: GoogleModelId }>,
+    options: BatchV4StartOptions<GoogleBatchModelIds>,
   ): Promise<BatchV4StartResult> {
+    assertSupportedBatchRequests(options.requests);
     const modelId = getGoogleBatchModelId(options.requests);
     const warnings: BatchV4StartResult['warnings'] = [];
     const displayName = `ai-sdk-batch-${this.batchGenerateId()}`;
@@ -196,11 +226,14 @@ export class GoogleBatch implements BatchV4<{ readonly text: GoogleModelId }> {
     let fileParts: string[] | undefined;
 
     for (const request of options.requests) {
-      const preparedRequest = await GoogleLanguageModel.prepareRequest({
-        modelId: request.modelId,
-        config: this.batchConfig,
-        options: request.options,
-      });
+      const preparedRequest =
+        request.type === 'text'
+          ? await GoogleLanguageModel.prepareRequest({
+              modelId: request.modelId,
+              config: this.batchConfig,
+              options: request.options,
+            })
+          : await this.prepareImageRequest(request);
       const inlinedRequest = {
         request: preparedRequest.args,
         metadata: { key: request.id },
@@ -602,6 +635,16 @@ export class GoogleBatch implements BatchV4<{ readonly text: GoogleModelId }> {
         warnings: [],
         providerOptionsNames: ['google'],
       });
+      const imageResult = convertGoogleImageBatchResult(result);
+      if (imageResult != null) {
+        yield {
+          type: 'image',
+          id: line.key,
+          status: 'succeeded',
+          result: imageResult,
+        };
+        continue;
+      }
       const unsupportedPart = result.content.find(
         part => !supportedGoogleBatchContentTypes.has(part.type),
       );
@@ -625,6 +668,108 @@ export class GoogleBatch implements BatchV4<{ readonly text: GoogleModelId }> {
     }
   }
 
+  private async prepareImageRequest(request: GoogleImageBatchRequest) {
+    const { prompt, n, size, aspectRatio, seed, files, mask, providerOptions } =
+      request.options;
+    const warnings: SharedV4Warning[] = [];
+
+    if (mask != null) {
+      throw new UnsupportedFunctionalityError({
+        functionality: 'mask-based image editing in Google batches',
+      });
+    }
+    if (n > 1) {
+      throw new UnsupportedFunctionalityError({
+        functionality: 'multiple images per Google batch request',
+      });
+    }
+    if (size != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'size',
+        details:
+          'This model does not support the `size` option. Use `aspectRatio` instead.',
+      });
+    }
+
+    const userContent: Extract<
+      LanguageModelV4Prompt[number],
+      { role: 'user' }
+    >['content'] = [];
+    if (prompt != null) userContent.push({ type: 'text', text: prompt });
+    for (const file of files ?? []) {
+      userContent.push(
+        file.type === 'url'
+          ? {
+              type: 'file',
+              data: { type: 'url', url: new URL(file.url) },
+              mediaType: 'image/*',
+            }
+          : {
+              type: 'file',
+              data: { type: 'data', data: file.data },
+              mediaType: file.mediaType,
+            },
+      );
+    }
+
+    const googleImageOptions = await parseProviderOptions({
+      provider: 'google',
+      providerOptions,
+      schema: googleImageModelOptionsSchema,
+    });
+    const {
+      responseModalities: _responseModalities,
+      imageConfig: userImageConfig,
+      ...passthroughGoogleOptions
+    } = (await parseProviderOptions({
+      provider: 'google',
+      providerOptions,
+      schema: googleLanguageModelOptions,
+    })) ?? {};
+    const preparedGoogleOptions = await parseProviderOptions({
+      provider: 'google',
+      providerOptions: {
+        google: {
+          ...passthroughGoogleOptions,
+          responseModalities: ['IMAGE'],
+          imageConfig:
+            aspectRatio != null || userImageConfig != null
+              ? {
+                  ...userImageConfig,
+                  ...(aspectRatio != null ? { aspectRatio } : {}),
+                }
+              : undefined,
+        },
+      },
+      schema: googleLanguageModelOptions,
+    });
+
+    const prepared = await GoogleLanguageModel.prepareRequest({
+      modelId: request.modelId,
+      config: this.batchConfig,
+      options: {
+        prompt: [{ role: 'user', content: userContent }],
+        seed,
+        providerOptions: {
+          google: preparedGoogleOptions ?? { responseModalities: ['IMAGE'] },
+        },
+        tools:
+          googleImageOptions?.googleSearch != null
+            ? [
+                {
+                  type: 'provider',
+                  id: 'google.google_search',
+                  name: 'google_search',
+                  args: googleImageOptions.googleSearch,
+                },
+              ]
+            : undefined,
+      },
+    });
+    return { ...prepared, warnings: [...warnings, ...prepared.warnings] };
+  }
+
   private async getHeaders(headers?: Record<string, string | undefined>) {
     return combineHeaders(
       this.batchConfig.headers
@@ -748,8 +893,8 @@ const googleUploadUrlResponseHandler: ResponseHandler<string> = async ({
 };
 
 function getGoogleBatchModelId(
-  requests: readonly GoogleBatchRequest[],
-): GoogleModelId {
+  requests: BatchV4StartOptions<GoogleBatchModelIds>['requests'],
+): GoogleModelId | GoogleImageModelId {
   const modelId = requests[0]?.modelId;
 
   if (modelId == null) {
@@ -771,3 +916,39 @@ function getGoogleBatchModelId(
 
   return modelId;
 }
+
+function convertGoogleImageBatchResult(
+  result: LanguageModelV4GenerateResult,
+): ImageModelV4Result | undefined {
+  const images = result.content.flatMap(part =>
+    part.type === 'file' &&
+    part.mediaType.startsWith('image/') &&
+    part.data.type === 'data'
+      ? [convertToBase64(part.data.data)]
+      : [],
+  );
+  if (images.length === 0) return undefined;
+
+  const googleMetadata =
+    (result.providerMetadata?.google as Record<string, unknown> | undefined) ??
+    {};
+  return {
+    images,
+    warnings: result.warnings,
+    providerMetadata: {
+      google: { ...googleMetadata, images: images.map(() => ({})) },
+    },
+    response: {
+      timestamp: new Date(),
+      modelId: result.response?.modelId ?? '',
+      headers: result.response?.headers,
+    },
+    usage: {
+      inputTokens: result.usage.inputTokens.total,
+      outputTokens: result.usage.outputTokens.total,
+      totalTokens:
+        (result.usage.inputTokens.total ?? 0) +
+        (result.usage.outputTokens.total ?? 0),
+    },
+  };
+}

package/src/google-language-model.ts

@@ -40,6 +40,7 @@ import {
 } from './convert-google-usage';
 import { convertJSONSchemaToOpenAPISchema } from './convert-json-schema-to-openapi-schema';
 import { convertToGoogleMessages } from './convert-to-google-messages';
+import { downloadToolResultFiles } from './download-tool-result-files';
 import { getModelPath } from './get-model-path';
 import { googleFailedResponseHandler } from './google-error';
 import {
@@ -76,6 +77,13 @@ export type GoogleLanguageModelConfig = {
    * The supported URLs for the model.
    */
   supportedUrls?: () => LanguageModelV4['supportedUrls'];
+
+  /**
+   * Settings for downloading remote files in tool results before conversion.
+   */
+  downloadToolResultFiles?: {
+    maxBytes: number;
+  };
 };
 
 export class GoogleLanguageModel implements LanguageModelV4 {
@@ -132,6 +140,7 @@ export class GoogleLanguageModel implements LanguageModelV4 {
       toolChoice,
       reasoning,
       providerOptions,
+      abortSignal,
     },
     isStreaming = false,
   }: {
@@ -287,14 +296,24 @@ export class GoogleLanguageModel implements LanguageModelV4 {
 
     const { usesGemini3Features } = getGoogleModelCapabilities(modelId);
 
-    const { contents, systemInstruction } = convertToGoogleMessages(prompt, {
-      isGemmaModel,
-      isGemini3Model: usesGemini3Features,
-      onWarning: warning => warnings.push(warning),
-      providerOptionsNames,
-      supportsFunctionResponseParts: usesGemini3Features,
-      includeFunctionCallIds: !isVertexProvider,
-    });
+    const promptWithDownloadedToolResultFiles = config.downloadToolResultFiles
+      ? await downloadToolResultFiles(prompt, {
+          abortSignal,
+          maxBytes: config.downloadToolResultFiles.maxBytes,
+        })
+      : prompt;
+
+    const { contents, systemInstruction } = convertToGoogleMessages(
+      promptWithDownloadedToolResultFiles,
+      {
+        isGemmaModel,
+        isGemini3Model: usesGemini3Features,
+        onWarning: warning => warnings.push(warning),
+        providerOptionsNames,
+        supportsFunctionResponseParts: usesGemini3Features,
+        includeFunctionCallIds: !isVertexProvider,
+      },
+    );
 
     const {
       tools: googleTools,

package/src/google-provider.ts

@@ -61,7 +61,10 @@ export interface GoogleProvider extends ProviderV4 {
 
   chat(modelId: GoogleModelId): LanguageModelV4;
 
-  experimental_batch(): BatchV4<{ text: GoogleModelId }>;
+  experimental_batch(): BatchV4<{
+    text: GoogleModelId;
+    image: GoogleImageModelId;
+  }>;
 
   /**
    * Creates a model for image generation.