@ai-sdk/google 4.0.0-beta.4 → 4.0.0-beta.41

package/docs/15-google-generative-ai.mdx

@@ -245,6 +245,12 @@ The following optional provider options are available for Google Generative AI m
   Optional. Defines labels used in billing reports. Available on Vertex AI only.
   See [Google Cloud labels documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/add-labels-to-api-calls).
 
+- **serviceTier** _'standard' | 'flex' | 'priority'_
+
+  Optional. The service tier to use for the request.
+  Set to 'flex' for 50% cheaper processing at the cost of increased latency.
+  Set to 'priority' for ultra-low latency at a 75-100% price premium over 'standard'.
+
 - **threshold** _string_
 
   Optional. Standalone threshold setting that can be used independently of `safetySettings`.
@@ -1131,6 +1137,28 @@ const { embedding } = await embed({
     google: {
       outputDimensionality: 512, // optional, number of dimensions for the embedding
       taskType: 'SEMANTIC_SIMILARITY', // optional, specifies the task type for generating embeddings
+      content: [[{ text: 'additional context' }]], // optional, per-value multimodal content (only 1 here, since `value` is only a single one)
+    } satisfies GoogleEmbeddingModelOptions,
+  },
+});
+```
+
+When using `embedMany`, provide per-value multimodal content via the `content` option. Each entry corresponds to a value at the same index; use `null` for text-only entries:
+
+```ts
+import { google, type GoogleEmbeddingModelOptions } from '@ai-sdk/google';
+import { embedMany } from 'ai';
+
+const { embeddings } = await embedMany({
+  model: google.embedding('gemini-embedding-2-preview'),
+  values: ['sunny day at the beach', 'rainy afternoon in the city'],
+  providerOptions: {
+    google: {
+      // content array must have the same length as values
+      content: [
+        [{ inlineData: { mimeType: 'image/png', data: '<base64>' } }], // pairs with values[0]
+        null, // text-only, pairs with values[1]
+      ],
     } satisfies GoogleEmbeddingModelOptions,
   },
 });
@@ -1155,11 +1183,16 @@ The following optional provider options are available for Google Generative AI e
   - `FACT_VERIFICATION`: Optimized for verifying factual information.
   - `CODE_RETRIEVAL_QUERY`: Optimized for retrieving code blocks based on natural language queries.
 
+- **content**: _array_
+
+  Optional. Per-value multimodal content parts for embedding non-text content (images, video, PDF, audio). Each entry corresponds to the embedding value at the same index — its parts are merged with the text value in the request. Use `null` for entries that are text-only. The array length must match the number of values being embedded. Each non-null entry is an array of parts, where each part can be either `{ text: string }` or `{ inlineData: { mimeType: string, data: string } }`. Supported by `gemini-embedding-2-preview`.
+
 ### Model Capabilities
 
-| Model                  | Default Dimensions | Custom Dimensions   |
-| ---------------------- | ------------------ | ------------------- |
-| `gemini-embedding-001` | 3072               | <Check size={18} /> |
+| Model                        | Default Dimensions | Custom Dimensions   | Multimodal          |
+| ---------------------------- | ------------------ | ------------------- | ------------------- |
+| `gemini-embedding-001`       | 3072               | <Check size={18} /> | <Cross size={18} /> |
+| `gemini-embedding-2-preview` | 3072               | <Check size={18} /> | <Check size={18} /> |
 
 ## Image Models
 

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.0-beta.4",
+  "version": "4.0.0-beta.41",
+  "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "files": [
     "dist/**/*",
@@ -25,26 +25,25 @@
     "./package.json": "./package.json",
     ".": {
       "types": "./dist/index.d.ts",
-      "import": "./dist/index.mjs",
-      "require": "./dist/index.js"
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
     },
     "./internal": {
       "types": "./dist/internal/index.d.ts",
-      "import": "./dist/internal/index.mjs",
-      "module": "./dist/internal/index.mjs",
-      "require": "./dist/internal/index.js"
+      "import": "./dist/internal/index.js",
+      "default": "./dist/internal/index.js"
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.0-beta.0",
-    "@ai-sdk/provider-utils": "5.0.0-beta.1"
+    "@ai-sdk/provider": "4.0.0-beta.12",
+    "@ai-sdk/provider-utils": "5.0.0-beta.22"
   },
   "devDependencies": {
     "@types/node": "20.17.24",
     "tsup": "^8",
     "typescript": "5.8.3",
     "zod": "3.25.76",
-    "@ai-sdk/test-server": "2.0.0-beta.0",
+    "@ai-sdk/test-server": "2.0.0-beta.1",
     "@vercel/ai-tsconfig": "0.0.0"
   },
   "peerDependencies": {
@@ -71,9 +70,7 @@
     "build": "pnpm clean && tsup --tsconfig tsconfig.build.json",
     "build:watch": "pnpm clean && tsup --watch",
     "clean": "del-cli dist docs *.tsbuildinfo",
-    "lint": "eslint \"./**/*.ts*\"",
     "type-check": "tsc --build",
-    "prettier-check": "prettier --check \"./**/*.ts*\"",
     "test": "pnpm test:node && pnpm test:edge",
     "test:update": "pnpm test:node -u",
     "test:watch": "vitest --config vitest.node.config.js",

package/src/convert-google-generative-ai-usage.ts

@@ -1,4 +1,9 @@
-import { LanguageModelV3Usage } from '@ai-sdk/provider';
+import { LanguageModelV4Usage } from '@ai-sdk/provider';
+
+export type GoogleGenerativeAITokenDetail = {
+  modality: string;
+  tokenCount: number;
+};
 
 export type GoogleGenerativeAIUsageMetadata = {
   promptTokenCount?: number | null;
@@ -7,11 +12,13 @@ export type GoogleGenerativeAIUsageMetadata = {
   cachedContentTokenCount?: number | null;
   thoughtsTokenCount?: number | null;
   trafficType?: string | null;
+  promptTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
+  candidatesTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
 };
 
 export function convertGoogleGenerativeAIUsage(
   usage: GoogleGenerativeAIUsageMetadata | undefined | null,
-): LanguageModelV3Usage {
+): LanguageModelV4Usage {
   if (usage == null) {
     return {
       inputTokens: {

package/src/convert-to-google-generative-ai-messages.ts

@@ -1,23 +1,186 @@
 import {
-  LanguageModelV3Prompt,
+  LanguageModelV4Prompt,
   UnsupportedFunctionalityError,
 } from '@ai-sdk/provider';
+import {
+  convertToBase64,
+  isProviderReference,
+  resolveProviderReference,
+} from '@ai-sdk/provider-utils';
 import {
   GoogleGenerativeAIContent,
   GoogleGenerativeAIContentPart,
+  GoogleGenerativeAIFunctionResponsePart,
   GoogleGenerativeAIPrompt,
 } from './google-generative-ai-prompt';
-import { convertToBase64 } from '@ai-sdk/provider-utils';
+
+const dataUrlRegex = /^data:([^;,]+);base64,(.+)$/s;
+
+function parseBase64DataUrl(
+  value: string,
+): { mediaType: string; data: string } | undefined {
+  const match = dataUrlRegex.exec(value);
+  if (match == null) {
+    return undefined;
+  }
+
+  return {
+    mediaType: match[1],
+    data: match[2],
+  };
+}
+
+function convertUrlToolResultPart(
+  url: string,
+): GoogleGenerativeAIFunctionResponsePart | undefined {
+  // Per https://ai.google.dev/api/caching#FunctionResponsePart, only inline data is supported.
+  // https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/function-calling#functionresponsepart suggests that this
+  // may be different for Vertex, but this needs to be confirmed and further tested for both APIs.
+  const parsedDataUrl = parseBase64DataUrl(url);
+  if (parsedDataUrl == null) {
+    return undefined;
+  }
+
+  return {
+    inlineData: {
+      mimeType: parsedDataUrl.mediaType,
+      data: parsedDataUrl.data,
+    },
+  };
+}
+
+/*
+ * Appends tool result content parts to the message using the functionResponse
+ * format with support for multimodal parts (e.g. inline images/files alongside
+ * text). This format is supported by Gemini 3+ models.
+ */
+function appendToolResultParts(
+  parts: GoogleGenerativeAIContentPart[],
+  toolName: string,
+  outputValue: Array<{
+    type: string;
+    [key: string]: unknown;
+  }>,
+): void {
+  const functionResponseParts: GoogleGenerativeAIFunctionResponsePart[] = [];
+  const responseTextParts: string[] = [];
+
+  for (const contentPart of outputValue) {
+    switch (contentPart.type) {
+      case 'text': {
+        responseTextParts.push(contentPart.text as string);
+        break;
+      }
+      case 'file-data': {
+        functionResponseParts.push({
+          inlineData: {
+            mimeType: contentPart.mediaType as string,
+            data: contentPart.data as string,
+          },
+        });
+        break;
+      }
+      case 'file-url': {
+        const functionResponsePart = convertUrlToolResultPart(
+          contentPart.url as string,
+        );
+
+        if (functionResponsePart != null) {
+          functionResponseParts.push(functionResponsePart);
+        } else {
+          responseTextParts.push(JSON.stringify(contentPart));
+        }
+        break;
+      }
+      default: {
+        responseTextParts.push(JSON.stringify(contentPart));
+        break;
+      }
+    }
+  }
+
+  parts.push({
+    functionResponse: {
+      name: toolName,
+      response: {
+        name: toolName,
+        content:
+          responseTextParts.length > 0
+            ? responseTextParts.join('\n')
+            : 'Tool executed successfully.',
+      },
+      ...(functionResponseParts.length > 0
+        ? { parts: functionResponseParts }
+        : {}),
+    },
+  });
+}
+
+/*
+ * Appends tool result content parts using a legacy format for pre-Gemini 3
+ * models that do not support multimodal parts within functionResponse. Instead,
+ * non-text content like images is sent as separate top-level inlineData parts.
+ */
+function appendLegacyToolResultParts(
+  parts: GoogleGenerativeAIContentPart[],
+  toolName: string,
+  outputValue: Array<{
+    type: string;
+    [key: string]: unknown;
+  }>,
+): void {
+  for (const contentPart of outputValue) {
+    switch (contentPart.type) {
+      case 'text':
+        parts.push({
+          functionResponse: {
+            name: toolName,
+            response: {
+              name: toolName,
+              content: contentPart.text,
+            },
+          },
+        });
+        break;
+      case 'file-data':
+        if ((contentPart.mediaType as string).startsWith('image/')) {
+          parts.push(
+            {
+              inlineData: {
+                mimeType: contentPart.mediaType as string,
+                data: contentPart.data as string,
+              },
+            },
+            {
+              text: 'Tool executed successfully and returned this image as a response',
+            },
+          );
+        } else {
+          parts.push({ text: JSON.stringify(contentPart) });
+        }
+        break;
+      default:
+        parts.push({ text: JSON.stringify(contentPart) });
+        break;
+    }
+  }
+}
 
 export function convertToGoogleGenerativeAIMessages(
-  prompt: LanguageModelV3Prompt,
-  options?: { isGemmaModel?: boolean; providerOptionsName?: string },
+  prompt: LanguageModelV4Prompt,
+  options?: {
+    isGemmaModel?: boolean;
+    providerOptionsName?: string;
+    supportsFunctionResponseParts?: boolean;
+  },
 ): GoogleGenerativeAIPrompt {
   const systemInstructionParts: Array<{ text: string }> = [];
   const contents: Array<GoogleGenerativeAIContent> = [];
   let systemMessagesAllowed = true;
   const isGemmaModel = options?.isGemmaModel ?? false;
   const providerOptionsName = options?.providerOptionsName ?? 'google';
+  const supportsFunctionResponseParts =
+    options?.supportsFunctionResponseParts ?? true;
 
   for (const { role, content } of prompt) {
     switch (role) {
@@ -46,25 +209,40 @@ export function convertToGoogleGenerativeAIMessages(
             }
 
             case 'file': {
-              // default to image/jpeg for unknown image/* types
               const mediaType =
                 part.mediaType === 'image/*' ? 'image/jpeg' : part.mediaType;
 
-              parts.push(
-                part.data instanceof URL
-                  ? {
-                      fileData: {
-                        mimeType: mediaType,
-                        fileUri: part.data.toString(),
-                      },
-                    }
-                  : {
-                      inlineData: {
-                        mimeType: mediaType,
-                        data: convertToBase64(part.data),
-                      },
-                    },
-              );
+              if (part.data instanceof URL) {
+                parts.push({
+                  fileData: {
+                    mimeType: mediaType,
+                    fileUri: part.data.toString(),
+                  },
+                });
+              } else if (isProviderReference(part.data)) {
+                if (providerOptionsName === 'vertex') {
+                  throw new UnsupportedFunctionalityError({
+                    functionality: 'file parts with provider references',
+                  });
+                }
+
+                parts.push({
+                  fileData: {
+                    mimeType: mediaType,
+                    fileUri: resolveProviderReference({
+                      reference: part.data,
+                      provider: 'google',
+                    }),
+                  },
+                });
+              } else {
+                parts.push({
+                  inlineData: {
+                    mimeType: mediaType,
+                    data: convertToBase64(part.data),
+                  },
+                });
+              }
 
               break;
             }
@@ -112,6 +290,24 @@ export function convertToGoogleGenerativeAIMessages(
                       };
                 }
 
+                case 'reasoning-file': {
+                  if (part.data instanceof URL) {
+                    throw new UnsupportedFunctionalityError({
+                      functionality:
+                        'File data URLs in assistant messages are not supported',
+                    });
+                  }
+
+                  return {
+                    inlineData: {
+                      mimeType: part.mediaType,
+                      data: convertToBase64(part.data),
+                    },
+                    thought: true,
+                    thoughtSignature,
+                  };
+                }
+
                 case 'file': {
                   if (part.data instanceof URL) {
                     throw new UnsupportedFunctionalityError({
@@ -120,6 +316,28 @@ export function convertToGoogleGenerativeAIMessages(
                     });
                   }
 
+                  if (isProviderReference(part.data)) {
+                    if (providerOptionsName === 'vertex') {
+                      throw new UnsupportedFunctionalityError({
+                        functionality: 'file parts with provider references',
+                      });
+                    }
+
+                    return {
+                      fileData: {
+                        mimeType: part.mediaType,
+                        fileUri: resolveProviderReference({
+                          reference: part.data,
+                          provider: 'google',
+                        }),
+                      },
+                      ...(providerOpts?.thought === true
+                        ? { thought: true }
+                        : {}),
+                      thoughtSignature,
+                    };
+                  }
+
                   return {
                     inlineData: {
                       mimeType: part.mediaType,
@@ -133,6 +351,29 @@ export function convertToGoogleGenerativeAIMessages(
                 }
 
                 case 'tool-call': {
+                  const serverToolCallId =
+                    providerOpts?.serverToolCallId != null
+                      ? String(providerOpts.serverToolCallId)
+                      : undefined;
+                  const serverToolType =
+                    providerOpts?.serverToolType != null
+                      ? String(providerOpts.serverToolType)
+                      : undefined;
+
+                  if (serverToolCallId && serverToolType) {
+                    return {
+                      toolCall: {
+                        toolType: serverToolType,
+                        args:
+                          typeof part.input === 'string'
+                            ? JSON.parse(part.input)
+                            : part.input,
+                        id: serverToolCallId,
+                      },
+                      thoughtSignature,
+                    };
+                  }
+
                   return {
                     functionCall: {
                       name: part.toolName,
@@ -141,10 +382,36 @@ export function convertToGoogleGenerativeAIMessages(
                     thoughtSignature,
                   };
                 }
+
+                case 'tool-result': {
+                  const serverToolCallId =
+                    providerOpts?.serverToolCallId != null
+                      ? String(providerOpts.serverToolCallId)
+                      : undefined;
+                  const serverToolType =
+                    providerOpts?.serverToolType != null
+                      ? String(providerOpts.serverToolType)
+                      : undefined;
+
+                  if (serverToolCallId && serverToolType) {
+                    return {
+                      toolResponse: {
+                        toolType: serverToolType,
+                        response:
+                          part.output.type === 'json' ? part.output.value : {},
+                        id: serverToolCallId,
+                      },
+                      thoughtSignature,
+                    };
+                  }
+
+                  return undefined;
+                }
               }
             })
             .filter(part => part !== undefined),
         });
+
         break;
       }
 
@@ -157,39 +424,51 @@ export function convertToGoogleGenerativeAIMessages(
           if (part.type === 'tool-approval-response') {
             continue;
           }
+
+          const partProviderOpts =
+            part.providerOptions?.[providerOptionsName] ??
+            (providerOptionsName !== 'google'
+              ? part.providerOptions?.google
+              : part.providerOptions?.vertex);
+          const serverToolCallId =
+            partProviderOpts?.serverToolCallId != null
+              ? String(partProviderOpts.serverToolCallId)
+              : undefined;
+          const serverToolType =
+            partProviderOpts?.serverToolType != null
+              ? String(partProviderOpts.serverToolType)
+              : undefined;
+
+          if (serverToolCallId && serverToolType) {
+            const serverThoughtSignature =
+              partProviderOpts?.thoughtSignature != null
+                ? String(partProviderOpts.thoughtSignature)
+                : undefined;
+
+            if (contents.length > 0) {
+              const lastContent = contents[contents.length - 1];
+              if (lastContent.role === 'model') {
+                lastContent.parts.push({
+                  toolResponse: {
+                    toolType: serverToolType,
+                    response:
+                      part.output.type === 'json' ? part.output.value : {},
+                    id: serverToolCallId,
+                  },
+                  thoughtSignature: serverThoughtSignature,
+                });
+                continue;
+              }
+            }
+          }
+
           const output = part.output;
 
           if (output.type === 'content') {
-            for (const contentPart of output.value) {
-              switch (contentPart.type) {
-                case 'text':
-                  parts.push({
-                    functionResponse: {
-                      name: part.toolName,
-                      response: {
-                        name: part.toolName,
-                        content: contentPart.text,
-                      },
-                    },
-                  });
-                  break;
-                case 'image-data':
-                  parts.push(
-                    {
-                      inlineData: {
-                        mimeType: contentPart.mediaType,
-                        data: contentPart.data,
-                      },
-                    },
-                    {
-                      text: 'Tool executed successfully and returned this image as a response',
-                    },
-                  );
-                  break;
-                default:
-                  parts.push({ text: JSON.stringify(contentPart) });
-                  break;
-              }
+            if (supportsFunctionResponseParts) {
+              appendToolResultParts(parts, part.toolName, output.value);
+            } else {
+              appendLegacyToolResultParts(parts, part.toolName, output.value);
             }
           } else {
             parts.push({