@ai-sdk/google 4.0.0-beta.3 → 4.0.0-beta.30

package/docs/15-google-generative-ai.mdx

@@ -245,6 +245,12 @@ The following optional provider options are available for Google Generative AI m
   Optional. Defines labels used in billing reports. Available on Vertex AI only.
   See [Google Cloud labels documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/add-labels-to-api-calls).
 
+- **serviceTier** _'standard' | 'flex' | 'priority'_
+
+  Optional. The service tier to use for the request.
+  Set to 'flex' for 50% cheaper processing at the cost of increased latency.
+  Set to 'priority' for ultra-low latency at a 75-100% price premium over 'standard'.
+
 - **threshold** _string_
 
   Optional. Standalone threshold setting that can be used independently of `safetySettings`.
@@ -1131,6 +1137,28 @@ const { embedding } = await embed({
     google: {
       outputDimensionality: 512, // optional, number of dimensions for the embedding
       taskType: 'SEMANTIC_SIMILARITY', // optional, specifies the task type for generating embeddings
+      content: [[{ text: 'additional context' }]], // optional, per-value multimodal content (only 1 here, since `value` is only a single one)
+    } satisfies GoogleEmbeddingModelOptions,
+  },
+});
+```
+
+When using `embedMany`, provide per-value multimodal content via the `content` option. Each entry corresponds to a value at the same index; use `null` for text-only entries:
+
+```ts
+import { google, type GoogleEmbeddingModelOptions } from '@ai-sdk/google';
+import { embedMany } from 'ai';
+
+const { embeddings } = await embedMany({
+  model: google.embedding('gemini-embedding-2-preview'),
+  values: ['sunny day at the beach', 'rainy afternoon in the city'],
+  providerOptions: {
+    google: {
+      // content array must have the same length as values
+      content: [
+        [{ inlineData: { mimeType: 'image/png', data: '<base64>' } }], // pairs with values[0]
+        null, // text-only, pairs with values[1]
+      ],
     } satisfies GoogleEmbeddingModelOptions,
   },
 });
@@ -1155,11 +1183,16 @@ The following optional provider options are available for Google Generative AI e
   - `FACT_VERIFICATION`: Optimized for verifying factual information.
   - `CODE_RETRIEVAL_QUERY`: Optimized for retrieving code blocks based on natural language queries.
 
+- **content**: _array_
+
+  Optional. Per-value multimodal content parts for embedding non-text content (images, video, PDF, audio). Each entry corresponds to the embedding value at the same index — its parts are merged with the text value in the request. Use `null` for entries that are text-only. The array length must match the number of values being embedded. Each non-null entry is an array of parts, where each part can be either `{ text: string }` or `{ inlineData: { mimeType: string, data: string } }`. Supported by `gemini-embedding-2-preview`.
+
 ### Model Capabilities
 
-| Model                  | Default Dimensions | Custom Dimensions   |
-| ---------------------- | ------------------ | ------------------- |
-| `gemini-embedding-001` | 3072               | <Check size={18} /> |
+| Model                        | Default Dimensions | Custom Dimensions   | Multimodal          |
+| ---------------------------- | ------------------ | ------------------- | ------------------- |
+| `gemini-embedding-001`       | 3072               | <Check size={18} /> | <Cross size={18} /> |
+| `gemini-embedding-2-preview` | 3072               | <Check size={18} /> | <Check size={18} /> |
 
 ## Image Models
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.0-beta.3",
+  "version": "4.0.0-beta.30",
   "license": "Apache-2.0",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -36,8 +36,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.0-beta.0",
-    "@ai-sdk/provider-utils": "5.0.0-beta.1"
+    "@ai-sdk/provider": "4.0.0-beta.9",
+    "@ai-sdk/provider-utils": "5.0.0-beta.15"
   },
   "devDependencies": {
     "@types/node": "20.17.24",
@@ -71,9 +71,7 @@
     "build": "pnpm clean && tsup --tsconfig tsconfig.build.json",
     "build:watch": "pnpm clean && tsup --watch",
     "clean": "del-cli dist docs *.tsbuildinfo",
-    "lint": "eslint \"./**/*.ts*\"",
     "type-check": "tsc --build",
-    "prettier-check": "prettier --check \"./**/*.ts*\"",
     "test": "pnpm test:node && pnpm test:edge",
     "test:update": "pnpm test:node -u",
     "test:watch": "vitest --config vitest.node.config.js",

package/src/convert-google-generative-ai-usage.ts

@@ -1,4 +1,9 @@
-import { LanguageModelV3Usage } from '@ai-sdk/provider';
+import { LanguageModelV4Usage } from '@ai-sdk/provider';
+
+export type GoogleGenerativeAITokenDetail = {
+  modality: string;
+  tokenCount: number;
+};
 
 export type GoogleGenerativeAIUsageMetadata = {
   promptTokenCount?: number | null;
@@ -7,11 +12,13 @@ export type GoogleGenerativeAIUsageMetadata = {
   cachedContentTokenCount?: number | null;
   thoughtsTokenCount?: number | null;
   trafficType?: string | null;
+  promptTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
+  candidatesTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
 };
 
 export function convertGoogleGenerativeAIUsage(
   usage: GoogleGenerativeAIUsageMetadata | undefined | null,
-): LanguageModelV3Usage {
+): LanguageModelV4Usage {
   if (usage == null) {
     return {
       inputTokens: {

package/src/convert-to-google-generative-ai-messages.ts

@@ -1,23 +1,184 @@
 import {
-  LanguageModelV3Prompt,
+  LanguageModelV4Prompt,
   UnsupportedFunctionalityError,
 } from '@ai-sdk/provider';
+import {
+  convertToBase64,
+  isProviderReference,
+  resolveProviderReference,
+} from '@ai-sdk/provider-utils';
 import {
   GoogleGenerativeAIContent,
   GoogleGenerativeAIContentPart,
+  GoogleGenerativeAIFunctionResponsePart,
   GoogleGenerativeAIPrompt,
 } from './google-generative-ai-prompt';
-import { convertToBase64 } from '@ai-sdk/provider-utils';
+
+const dataUrlRegex = /^data:([^;,]+);base64,(.+)$/s;
+
+function parseBase64DataUrl(
+  value: string,
+): { mediaType: string; data: string } | undefined {
+  const match = dataUrlRegex.exec(value);
+  if (match == null) {
+    return undefined;
+  }
+
+  return {
+    mediaType: match[1],
+    data: match[2],
+  };
+}
+
+function convertUrlToolResultPart(
+  url: string,
+): GoogleGenerativeAIFunctionResponsePart | undefined {
+  // Per https://ai.google.dev/api/caching#FunctionResponsePart, only inline data is supported.
+  // https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/function-calling#functionresponsepart suggests that this
+  // may be different for Vertex, but this needs to be confirmed and further tested for both APIs.
+  const parsedDataUrl = parseBase64DataUrl(url);
+  if (parsedDataUrl == null) {
+    return undefined;
+  }
+
+  return {
+    inlineData: {
+      mimeType: parsedDataUrl.mediaType,
+      data: parsedDataUrl.data,
+    },
+  };
+}
+
+/*
+ * Appends tool result content parts to the message using the functionResponse
+ * format with support for multimodal parts (e.g. inline images/files alongside
+ * text). This format is supported by Gemini 3+ models.
+ */
+function appendToolResultParts(
+  parts: GoogleGenerativeAIContentPart[],
+  toolName: string,
+  outputValue: Array<{
+    type: string;
+    [key: string]: unknown;
+  }>,
+): void {
+  const functionResponseParts: GoogleGenerativeAIFunctionResponsePart[] = [];
+  const responseTextParts: string[] = [];
+
+  for (const contentPart of outputValue) {
+    switch (contentPart.type) {
+      case 'text': {
+        responseTextParts.push(contentPart.text as string);
+        break;
+      }
+      case 'image-data':
+      case 'file-data': {
+        functionResponseParts.push({
+          inlineData: {
+            mimeType: contentPart.mediaType as string,
+            data: contentPart.data as string,
+          },
+        });
+        break;
+      }
+      case 'image-url':
+      case 'file-url': {
+        const functionResponsePart = convertUrlToolResultPart(
+          contentPart.url as string,
+        );
+
+        if (functionResponsePart != null) {
+          functionResponseParts.push(functionResponsePart);
+        } else {
+          responseTextParts.push(JSON.stringify(contentPart));
+        }
+        break;
+      }
+      default: {
+        responseTextParts.push(JSON.stringify(contentPart));
+        break;
+      }
+    }
+  }
+
+  parts.push({
+    functionResponse: {
+      name: toolName,
+      response: {
+        name: toolName,
+        content:
+          responseTextParts.length > 0
+            ? responseTextParts.join('\n')
+            : 'Tool executed successfully.',
+      },
+      ...(functionResponseParts.length > 0
+        ? { parts: functionResponseParts }
+        : {}),
+    },
+  });
+}
+
+/*
+ * Appends tool result content parts using a legacy format for pre-Gemini 3
+ * models that do not support multimodal parts within functionResponse. Instead,
+ * non-text content like images is sent as separate top-level inlineData parts.
+ */
+function appendLegacyToolResultParts(
+  parts: GoogleGenerativeAIContentPart[],
+  toolName: string,
+  outputValue: Array<{
+    type: string;
+    [key: string]: unknown;
+  }>,
+): void {
+  for (const contentPart of outputValue) {
+    switch (contentPart.type) {
+      case 'text':
+        parts.push({
+          functionResponse: {
+            name: toolName,
+            response: {
+              name: toolName,
+              content: contentPart.text,
+            },
+          },
+        });
+        break;
+      case 'image-data':
+        parts.push(
+          {
+            inlineData: {
+              mimeType: String(contentPart.mediaType),
+              data: String(contentPart.data),
+            },
+          },
+          {
+            text: 'Tool executed successfully and returned this image as a response',
+          },
+        );
+        break;
+      default:
+        parts.push({ text: JSON.stringify(contentPart) });
+        break;
+    }
+  }
+}
 
 export function convertToGoogleGenerativeAIMessages(
-  prompt: LanguageModelV3Prompt,
-  options?: { isGemmaModel?: boolean; providerOptionsName?: string },
+  prompt: LanguageModelV4Prompt,
+  options?: {
+    isGemmaModel?: boolean;
+    providerOptionsName?: string;
+    supportsFunctionResponseParts?: boolean;
+  },
 ): GoogleGenerativeAIPrompt {
   const systemInstructionParts: Array<{ text: string }> = [];
   const contents: Array<GoogleGenerativeAIContent> = [];
   let systemMessagesAllowed = true;
   const isGemmaModel = options?.isGemmaModel ?? false;
   const providerOptionsName = options?.providerOptionsName ?? 'google';
+  const supportsFunctionResponseParts =
+    options?.supportsFunctionResponseParts ?? true;
 
   for (const { role, content } of prompt) {
     switch (role) {
@@ -46,25 +207,40 @@ export function convertToGoogleGenerativeAIMessages(
             }
 
             case 'file': {
-              // default to image/jpeg for unknown image/* types
               const mediaType =
                 part.mediaType === 'image/*' ? 'image/jpeg' : part.mediaType;
 
-              parts.push(
-                part.data instanceof URL
-                  ? {
-                      fileData: {
-                        mimeType: mediaType,
-                        fileUri: part.data.toString(),
-                      },
-                    }
-                  : {
-                      inlineData: {
-                        mimeType: mediaType,
-                        data: convertToBase64(part.data),
-                      },
-                    },
-              );
+              if (part.data instanceof URL) {
+                parts.push({
+                  fileData: {
+                    mimeType: mediaType,
+                    fileUri: part.data.toString(),
+                  },
+                });
+              } else if (isProviderReference(part.data)) {
+                if (providerOptionsName === 'vertex') {
+                  throw new UnsupportedFunctionalityError({
+                    functionality: 'file parts with provider references',
+                  });
+                }
+
+                parts.push({
+                  fileData: {
+                    mimeType: mediaType,
+                    fileUri: resolveProviderReference({
+                      reference: part.data,
+                      provider: 'google',
+                    }),
+                  },
+                });
+              } else {
+                parts.push({
+                  inlineData: {
+                    mimeType: mediaType,
+                    data: convertToBase64(part.data),
+                  },
+                });
+              }
 
               break;
             }
@@ -112,6 +288,24 @@ export function convertToGoogleGenerativeAIMessages(
                       };
                 }
 
+                case 'reasoning-file': {
+                  if (part.data instanceof URL) {
+                    throw new UnsupportedFunctionalityError({
+                      functionality:
+                        'File data URLs in assistant messages are not supported',
+                    });
+                  }
+
+                  return {
+                    inlineData: {
+                      mimeType: part.mediaType,
+                      data: convertToBase64(part.data),
+                    },
+                    thought: true,
+                    thoughtSignature,
+                  };
+                }
+
                 case 'file': {
                   if (part.data instanceof URL) {
                     throw new UnsupportedFunctionalityError({
@@ -120,16 +314,64 @@ export function convertToGoogleGenerativeAIMessages(
                     });
                   }
 
+                  if (isProviderReference(part.data)) {
+                    if (providerOptionsName === 'vertex') {
+                      throw new UnsupportedFunctionalityError({
+                        functionality: 'file parts with provider references',
+                      });
+                    }
+
+                    return {
+                      fileData: {
+                        mimeType: part.mediaType,
+                        fileUri: resolveProviderReference({
+                          reference: part.data,
+                          provider: 'google',
+                        }),
+                      },
+                      ...(providerOpts?.thought === true
+                        ? { thought: true }
+                        : {}),
+                      thoughtSignature,
+                    };
+                  }
+
                   return {
                     inlineData: {
                       mimeType: part.mediaType,
                       data: convertToBase64(part.data),
                     },
+                    ...(providerOpts?.thought === true
+                      ? { thought: true }
+                      : {}),
                     thoughtSignature,
                   };
                 }
 
                 case 'tool-call': {
+                  const serverToolCallId =
+                    providerOpts?.serverToolCallId != null
+                      ? String(providerOpts.serverToolCallId)
+                      : undefined;
+                  const serverToolType =
+                    providerOpts?.serverToolType != null
+                      ? String(providerOpts.serverToolType)
+                      : undefined;
+
+                  if (serverToolCallId && serverToolType) {
+                    return {
+                      toolCall: {
+                        toolType: serverToolType,
+                        args:
+                          typeof part.input === 'string'
+                            ? JSON.parse(part.input)
+                            : part.input,
+                        id: serverToolCallId,
+                      },
+                      thoughtSignature,
+                    };
+                  }
+
                   return {
                     functionCall: {
                       name: part.toolName,
@@ -138,10 +380,36 @@ export function convertToGoogleGenerativeAIMessages(
                     thoughtSignature,
                   };
                 }
+
+                case 'tool-result': {
+                  const serverToolCallId =
+                    providerOpts?.serverToolCallId != null
+                      ? String(providerOpts.serverToolCallId)
+                      : undefined;
+                  const serverToolType =
+                    providerOpts?.serverToolType != null
+                      ? String(providerOpts.serverToolType)
+                      : undefined;
+
+                  if (serverToolCallId && serverToolType) {
+                    return {
+                      toolResponse: {
+                        toolType: serverToolType,
+                        response:
+                          part.output.type === 'json' ? part.output.value : {},
+                        id: serverToolCallId,
+                      },
+                      thoughtSignature,
+                    };
+                  }
+
+                  return undefined;
+                }
               }
             })
             .filter(part => part !== undefined),
         });
+
         break;
       }
 
@@ -154,39 +422,51 @@ export function convertToGoogleGenerativeAIMessages(
           if (part.type === 'tool-approval-response') {
             continue;
           }
+
+          const partProviderOpts =
+            part.providerOptions?.[providerOptionsName] ??
+            (providerOptionsName !== 'google'
+              ? part.providerOptions?.google
+              : part.providerOptions?.vertex);
+          const serverToolCallId =
+            partProviderOpts?.serverToolCallId != null
+              ? String(partProviderOpts.serverToolCallId)
+              : undefined;
+          const serverToolType =
+            partProviderOpts?.serverToolType != null
+              ? String(partProviderOpts.serverToolType)
+              : undefined;
+
+          if (serverToolCallId && serverToolType) {
+            const serverThoughtSignature =
+              partProviderOpts?.thoughtSignature != null
+                ? String(partProviderOpts.thoughtSignature)
+                : undefined;
+
+            if (contents.length > 0) {
+              const lastContent = contents[contents.length - 1];
+              if (lastContent.role === 'model') {
+                lastContent.parts.push({
+                  toolResponse: {
+                    toolType: serverToolType,
+                    response:
+                      part.output.type === 'json' ? part.output.value : {},
+                    id: serverToolCallId,
+                  },
+                  thoughtSignature: serverThoughtSignature,
+                });
+                continue;
+              }
+            }
+          }
+
           const output = part.output;
 
           if (output.type === 'content') {
-            for (const contentPart of output.value) {
-              switch (contentPart.type) {
-                case 'text':
-                  parts.push({
-                    functionResponse: {
-                      name: part.toolName,
-                      response: {
-                        name: part.toolName,
-                        content: contentPart.text,
-                      },
-                    },
-                  });
-                  break;
-                case 'image-data':
-                  parts.push(
-                    {
-                      inlineData: {
-                        mimeType: contentPart.mediaType,
-                        data: contentPart.data,
-                      },
-                    },
-                    {
-                      text: 'Tool executed successfully and returned this image as a response',
-                    },
-                  );
-                  break;
-                default:
-                  parts.push({ text: JSON.stringify(contentPart) });
-                  break;
-              }
+            if (supportsFunctionResponseParts) {
+              appendToolResultParts(parts, part.toolName, output.value);
+            } else {
+              appendLegacyToolResultParts(parts, part.toolName, output.value);
             }
           } else {
             parts.push({

package/src/google-generative-ai-embedding-model.ts

@@ -1,5 +1,5 @@
 import {
-  EmbeddingModelV3,
+  EmbeddingModelV4,
   TooManyEmbeddingValuesForCallError,
 } from '@ai-sdk/provider';
 import {
@@ -26,8 +26,8 @@ type GoogleGenerativeAIEmbeddingConfig = {
   fetch?: FetchFunction;
 };
 
-export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
-  readonly specificationVersion = 'v3';
+export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV4 {
+  readonly specificationVersion = 'v4';
   readonly modelId: GoogleGenerativeAIEmbeddingModelId;
   readonly maxEmbeddingsPerCall = 2048;
   readonly supportsParallelCalls = true;
@@ -50,8 +50,8 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
     headers,
     abortSignal,
     providerOptions,
-  }: Parameters<EmbeddingModelV3['doEmbed']>[0]): Promise<
-    Awaited<ReturnType<EmbeddingModelV3['doEmbed']>>
+  }: Parameters<EmbeddingModelV4['doEmbed']>[0]): Promise<
+    Awaited<ReturnType<EmbeddingModelV4['doEmbed']>>
   > {
     // Parse provider options
     const googleOptions = await parseProviderOptions({
@@ -74,8 +74,26 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
       headers,
     );
 
-    // For single embeddings, use the single endpoint (ratelimits, etc.)
+    const multimodalContent = googleOptions?.content;
+
+    if (
+      multimodalContent != null &&
+      multimodalContent.length !== values.length
+    ) {
+      throw new Error(
+        `The number of multimodal content entries (${multimodalContent.length}) must match the number of values (${values.length}).`,
+      );
+    }
+
+    // For single embeddings, use the single endpoint
     if (values.length === 1) {
+      const valueParts = multimodalContent?.[0];
+      const textPart = values[0] ? [{ text: values[0] }] : [];
+      const parts =
+        valueParts != null
+          ? [...textPart, ...valueParts]
+          : [{ text: values[0] }];
+
       const {
         responseHeaders,
         value: response,
@@ -86,7 +104,7 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
         body: {
           model: `models/${this.modelId}`,
           content: {
-            parts: [{ text: values[0] }],
+            parts,
           },
           outputDimensionality: googleOptions?.outputDimensionality,
           taskType: googleOptions?.taskType,
@@ -107,6 +125,7 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
       };
     }
 
+    // For multiple values, use the batch endpoint
     const {
       responseHeaders,
       value: response,
@@ -115,12 +134,22 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
       url: `${this.config.baseURL}/models/${this.modelId}:batchEmbedContents`,
       headers: mergedHeaders,
       body: {
-        requests: values.map(value => ({
-          model: `models/${this.modelId}`,
-          content: { role: 'user', parts: [{ text: value }] },
-          outputDimensionality: googleOptions?.outputDimensionality,
-          taskType: googleOptions?.taskType,
-        })),
+        requests: values.map((value, index) => {
+          const valueParts = multimodalContent?.[index];
+          const textPart = value ? [{ text: value }] : [];
+          return {
+            model: `models/${this.modelId}`,
+            content: {
+              role: 'user',
+              parts:
+                valueParts != null
+                  ? [...textPart, ...valueParts]
+                  : [{ text: value }],
+            },
+            outputDimensionality: googleOptions?.outputDimensionality,
+            taskType: googleOptions?.taskType,
+          };
+        }),
       },
       failedResponseHandler: googleFailedResponseHandler,
       successfulResponseHandler: createJsonResponseHandler(