@ai-sdk/google 4.0.0-beta.4 → 4.0.0-beta.40

package/src/google-generative-ai-embedding-model.ts

@@ -1,5 +1,5 @@
 import {
-  EmbeddingModelV3,
+  EmbeddingModelV4,
   TooManyEmbeddingValuesForCallError,
 } from '@ai-sdk/provider';
 import {
@@ -10,6 +10,9 @@ import {
   parseProviderOptions,
   postJsonToApi,
   resolve,
+  serializeModelOptions,
+  WORKFLOW_SERIALIZE,
+  WORKFLOW_DESERIALIZE,
   zodSchema,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
@@ -22,18 +25,35 @@ import {
 type GoogleGenerativeAIEmbeddingConfig = {
   provider: string;
   baseURL: string;
-  headers: () => Record<string, string | undefined>;
+  headers?: () => Record<string, string | undefined>;
   fetch?: FetchFunction;
 };
 
-export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
-  readonly specificationVersion = 'v3';
+export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV4 {
+  readonly specificationVersion = 'v4';
   readonly modelId: GoogleGenerativeAIEmbeddingModelId;
   readonly maxEmbeddingsPerCall = 2048;
   readonly supportsParallelCalls = true;
 
   private readonly config: GoogleGenerativeAIEmbeddingConfig;
 
+  static [WORKFLOW_SERIALIZE](model: GoogleGenerativeAIEmbeddingModel) {
+    return serializeModelOptions({
+      modelId: model.modelId,
+      config: model.config,
+    });
+  }
+
+  static [WORKFLOW_DESERIALIZE](options: {
+    modelId: string;
+    config: GoogleGenerativeAIEmbeddingConfig;
+  }) {
+    return new GoogleGenerativeAIEmbeddingModel(
+      options.modelId,
+      options.config,
+    );
+  }
+
   get provider(): string {
     return this.config.provider;
   }
@@ -50,8 +70,8 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
     headers,
     abortSignal,
     providerOptions,
-  }: Parameters<EmbeddingModelV3['doEmbed']>[0]): Promise<
-    Awaited<ReturnType<EmbeddingModelV3['doEmbed']>>
+  }: Parameters<EmbeddingModelV4['doEmbed']>[0]): Promise<
+    Awaited<ReturnType<EmbeddingModelV4['doEmbed']>>
   > {
     // Parse provider options
     const googleOptions = await parseProviderOptions({
@@ -70,12 +90,30 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
     }
 
     const mergedHeaders = combineHeaders(
-      await resolve(this.config.headers),
+      this.config.headers ? await resolve(this.config.headers) : undefined,
       headers,
     );
 
-    // For single embeddings, use the single endpoint (ratelimits, etc.)
+    const multimodalContent = googleOptions?.content;
+
+    if (
+      multimodalContent != null &&
+      multimodalContent.length !== values.length
+    ) {
+      throw new Error(
+        `The number of multimodal content entries (${multimodalContent.length}) must match the number of values (${values.length}).`,
+      );
+    }
+
+    // For single embeddings, use the single endpoint
     if (values.length === 1) {
+      const valueParts = multimodalContent?.[0];
+      const textPart = values[0] ? [{ text: values[0] }] : [];
+      const parts =
+        valueParts != null
+          ? [...textPart, ...valueParts]
+          : [{ text: values[0] }];
+
       const {
         responseHeaders,
         value: response,
@@ -86,7 +124,7 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
         body: {
           model: `models/${this.modelId}`,
           content: {
-            parts: [{ text: values[0] }],
+            parts,
           },
           outputDimensionality: googleOptions?.outputDimensionality,
           taskType: googleOptions?.taskType,
@@ -107,6 +145,7 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
       };
     }
 
+    // For multiple values, use the batch endpoint
     const {
       responseHeaders,
       value: response,
@@ -115,12 +154,22 @@ export class GoogleGenerativeAIEmbeddingModel implements EmbeddingModelV3 {
       url: `${this.config.baseURL}/models/${this.modelId}:batchEmbedContents`,
       headers: mergedHeaders,
       body: {
-        requests: values.map(value => ({
-          model: `models/${this.modelId}`,
-          content: { role: 'user', parts: [{ text: value }] },
-          outputDimensionality: googleOptions?.outputDimensionality,
-          taskType: googleOptions?.taskType,
-        })),
+        requests: values.map((value, index) => {
+          const valueParts = multimodalContent?.[index];
+          const textPart = value ? [{ text: value }] : [];
+          return {
+            model: `models/${this.modelId}`,
+            content: {
+              role: 'user',
+              parts:
+                valueParts != null
+                  ? [...textPart, ...valueParts]
+                  : [{ text: value }],
+            },
+            outputDimensionality: googleOptions?.outputDimensionality,
+            taskType: googleOptions?.taskType,
+          };
+        }),
       },
       failedResponseHandler: googleFailedResponseHandler,
       successfulResponseHandler: createJsonResponseHandler(

package/src/google-generative-ai-embedding-options.ts

@@ -7,8 +7,19 @@ import { z } from 'zod/v4';
 
 export type GoogleGenerativeAIEmbeddingModelId =
   | 'gemini-embedding-001'
+  | 'gemini-embedding-2-preview'
   | (string & {});
 
+const googleEmbeddingContentPartSchema = z.union([
+  z.object({ text: z.string() }),
+  z.object({
+    inlineData: z.object({
+      mimeType: z.string(),
+      data: z.string(),
+    }),
+  }),
+]);
+
 export const googleEmbeddingModelOptions = lazySchema(() =>
   zodSchema(
     z.object({
@@ -42,6 +53,19 @@ export const googleEmbeddingModelOptions = lazySchema(() =>
           'CODE_RETRIEVAL_QUERY',
         ])
         .optional(),
+
+      /**
+       * Optional. Per-value multimodal content parts for embedding non-text
+       * content (images, video, PDF, audio). Each entry corresponds to the
+       * embedding value at the same index and its parts are merged with the
+       * text value in the request. Use `null` for entries that are text-only.
+       *
+       * The array length must match the number of values being embedded. In
+       * the case of a single embedding, the array length must be 1.
+       */
+      content: z
+        .array(z.array(googleEmbeddingContentPartSchema).min(1).nullable())
+        .optional(),
     }),
   ),
 );

package/src/google-generative-ai-files.ts

@@ -0,0 +1,228 @@
+import {
+  AISDKError,
+  type FilesV4,
+  type FilesV4UploadFileCallOptions,
+  type FilesV4UploadFileResult,
+  type SharedV4Warning,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createJsonResponseHandler,
+  delay,
+  type FetchFunction,
+  lazySchema,
+  parseProviderOptions,
+  zodSchema,
+  getFromApi,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { googleFailedResponseHandler } from './google-error';
+
+export type GoogleFilesUploadOptions = {
+  displayName?: string | null;
+  pollIntervalMs?: number | null;
+  pollTimeoutMs?: number | null;
+
+  [key: string]: unknown;
+};
+
+interface GoogleGenerativeAIFilesConfig {
+  provider: string;
+  baseURL: string;
+  headers: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+}
+
+export class GoogleGenerativeAIFiles implements FilesV4 {
+  readonly specificationVersion = 'v4';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(private readonly config: GoogleGenerativeAIFilesConfig) {}
+
+  async uploadFile(
+    options: FilesV4UploadFileCallOptions,
+  ): Promise<FilesV4UploadFileResult> {
+    const googleOptions = (await parseProviderOptions({
+      provider: 'google',
+      providerOptions: options.providerOptions,
+      schema: googleFilesUploadOptionsSchema,
+    })) as GoogleFilesUploadOptions | undefined;
+
+    const resolvedHeaders = this.config.headers();
+    const fetchFn = this.config.fetch ?? globalThis.fetch;
+
+    const warnings: Array<SharedV4Warning> = [];
+    if (options.filename != null) {
+      warnings.push({ type: 'unsupported', feature: 'filename' });
+    }
+
+    const data = options.data;
+    const fileBytes =
+      data instanceof Uint8Array
+        ? data
+        : Uint8Array.from(atob(data), c => c.charCodeAt(0));
+
+    const mediaType = options.mediaType;
+    const displayName = googleOptions?.displayName;
+
+    const baseOrigin = this.config.baseURL.replace(/\/v1beta$/, '');
+
+    const initResponse = await fetchFn(`${baseOrigin}/upload/v1beta/files`, {
+      method: 'POST',
+      headers: {
+        ...resolvedHeaders,
+        'X-Goog-Upload-Protocol': 'resumable',
+        'X-Goog-Upload-Command': 'start',
+        'X-Goog-Upload-Header-Content-Length': String(fileBytes.length),
+        'X-Goog-Upload-Header-Content-Type': mediaType,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        file: {
+          ...(displayName != null ? { display_name: displayName } : {}),
+        },
+      }),
+    });
+
+    if (!initResponse.ok) {
+      const errorBody = await initResponse.text();
+      throw new AISDKError({
+        name: 'GOOGLE_FILES_UPLOAD_ERROR',
+        message: `Failed to initiate resumable upload: ${initResponse.status} ${errorBody}`,
+      });
+    }
+
+    const uploadUrl = initResponse.headers.get('x-goog-upload-url');
+    if (!uploadUrl) {
+      throw new AISDKError({
+        name: 'GOOGLE_FILES_UPLOAD_ERROR',
+        message: 'No upload URL returned from initiation request',
+      });
+    }
+
+    const uploadResponse = await fetchFn(uploadUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Length': String(fileBytes.length),
+        'X-Goog-Upload-Offset': '0',
+        'X-Goog-Upload-Command': 'upload, finalize',
+      },
+      body: fileBytes,
+    });
+
+    if (!uploadResponse.ok) {
+      const errorBody = await uploadResponse.text();
+      throw new AISDKError({
+        name: 'GOOGLE_FILES_UPLOAD_ERROR',
+        message: `Failed to upload file data: ${uploadResponse.status} ${errorBody}`,
+      });
+    }
+
+    const uploadResult = (await uploadResponse.json()) as {
+      file: GoogleFileResource;
+    };
+
+    let file = uploadResult.file;
+
+    const pollIntervalMs = googleOptions?.pollIntervalMs ?? 2000;
+    const pollTimeoutMs = googleOptions?.pollTimeoutMs ?? 300000;
+    const startTime = Date.now();
+
+    while (file.state === 'PROCESSING') {
+      if (Date.now() - startTime > pollTimeoutMs) {
+        throw new AISDKError({
+          name: 'GOOGLE_FILES_UPLOAD_TIMEOUT',
+          message: `File processing timed out after ${pollTimeoutMs}ms`,
+        });
+      }
+
+      await delay(pollIntervalMs);
+
+      const { value: fileStatus } = await getFromApi({
+        url: `${this.config.baseURL}/${file.name}`,
+        headers: combineHeaders(resolvedHeaders),
+        successfulResponseHandler: createJsonResponseHandler(
+          googleFileResponseSchema,
+        ),
+        failedResponseHandler: googleFailedResponseHandler,
+        fetch: this.config.fetch,
+      });
+
+      file = fileStatus;
+    }
+
+    if (file.state === 'FAILED') {
+      throw new AISDKError({
+        name: 'GOOGLE_FILES_UPLOAD_FAILED',
+        message: `File processing failed for ${file.name}`,
+      });
+    }
+
+    return {
+      warnings,
+      providerReference: { google: file.uri },
+      mediaType: file.mimeType ?? options.mediaType,
+      providerMetadata: {
+        google: {
+          name: file.name,
+          displayName: file.displayName,
+          mimeType: file.mimeType,
+          sizeBytes: file.sizeBytes,
+          state: file.state,
+          uri: file.uri,
+          ...(file.createTime != null ? { createTime: file.createTime } : {}),
+          ...(file.updateTime != null ? { updateTime: file.updateTime } : {}),
+          ...(file.expirationTime != null
+            ? { expirationTime: file.expirationTime }
+            : {}),
+          ...(file.sha256Hash != null ? { sha256Hash: file.sha256Hash } : {}),
+        },
+      },
+    };
+  }
+}
+
+type GoogleFileResource = {
+  name: string;
+  displayName?: string | null;
+  mimeType: string;
+  sizeBytes?: string | null;
+  createTime?: string | null;
+  updateTime?: string | null;
+  expirationTime?: string | null;
+  sha256Hash?: string | null;
+  uri: string;
+  state: string;
+};
+
+const googleFileResponseSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      name: z.string(),
+      displayName: z.string().nullish(),
+      mimeType: z.string(),
+      sizeBytes: z.string().nullish(),
+      createTime: z.string().nullish(),
+      updateTime: z.string().nullish(),
+      expirationTime: z.string().nullish(),
+      sha256Hash: z.string().nullish(),
+      uri: z.string(),
+      state: z.string(),
+    }),
+  ),
+);
+
+const googleFilesUploadOptionsSchema = lazySchema(() =>
+  zodSchema(
+    z
+      .object({
+        displayName: z.string().nullish(),
+        pollIntervalMs: z.number().positive().nullish(),
+        pollTimeoutMs: z.number().positive().nullish(),
+      })
+      .passthrough(),
+  ),
+);

package/src/google-generative-ai-image-model.ts

@@ -1,7 +1,7 @@
 import {
-  ImageModelV3,
-  LanguageModelV3Prompt,
-  SharedV3Warning,
+  ImageModelV4,
+  LanguageModelV4Prompt,
+  SharedV4Warning,
 } from '@ai-sdk/provider';
 import {
   combineHeaders,
@@ -15,6 +15,9 @@ import {
   postJsonToApi,
   Resolvable,
   resolve,
+  serializeModelOptions,
+  WORKFLOW_SERIALIZE,
+  WORKFLOW_DESERIALIZE,
   zodSchema,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
@@ -37,8 +40,26 @@ interface GoogleGenerativeAIImageModelConfig {
   };
 }
 
-export class GoogleGenerativeAIImageModel implements ImageModelV3 {
-  readonly specificationVersion = 'v3';
+export class GoogleGenerativeAIImageModel implements ImageModelV4 {
+  readonly specificationVersion = 'v4';
+
+  static [WORKFLOW_SERIALIZE](model: GoogleGenerativeAIImageModel) {
+    return serializeModelOptions({
+      modelId: model.modelId,
+      config: model.config,
+    });
+  }
+
+  static [WORKFLOW_DESERIALIZE](options: {
+    modelId: string;
+    config: GoogleGenerativeAIImageModelConfig;
+  }) {
+    return new GoogleGenerativeAIImageModel(
+      options.modelId,
+      {},
+      options.config,
+    );
+  }
 
   get maxImagesPerCall(): number {
     if (this.settings.maxImagesPerCall != null) {
@@ -63,8 +84,8 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
   ) {}
 
   async doGenerate(
-    options: Parameters<ImageModelV3['doGenerate']>[0],
-  ): Promise<Awaited<ReturnType<ImageModelV3['doGenerate']>>> {
+    options: Parameters<ImageModelV4['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<ImageModelV4['doGenerate']>>> {
     // Gemini image models use the language model API internally
     if (isGeminiModel(this.modelId)) {
       return this.doGenerateGemini(options);
@@ -73,8 +94,8 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
   }
 
   private async doGenerateImagen(
-    options: Parameters<ImageModelV3['doGenerate']>[0],
-  ): Promise<Awaited<ReturnType<ImageModelV3['doGenerate']>>> {
+    options: Parameters<ImageModelV4['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<ImageModelV4['doGenerate']>>> {
     const {
       prompt,
       n = 1,
@@ -87,7 +108,7 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
       files,
       mask,
     } = options;
-    const warnings: Array<SharedV3Warning> = [];
+    const warnings: Array<SharedV4Warning> = [];
 
     // Imagen API endpoints do not support image editing
     if (files != null && files.length > 0) {
@@ -151,7 +172,10 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
       predictions: Array<{ bytesBase64Encoded: string }>;
     }>({
       url: `${this.config.baseURL}/models/${this.modelId}:predict`,
-      headers: combineHeaders(await resolve(this.config.headers), headers),
+      headers: combineHeaders(
+        this.config.headers ? await resolve(this.config.headers) : undefined,
+        headers,
+      ),
       body,
       failedResponseHandler: googleFailedResponseHandler,
       successfulResponseHandler: createJsonResponseHandler(
@@ -181,8 +205,8 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
   }
 
   private async doGenerateGemini(
-    options: Parameters<ImageModelV3['doGenerate']>[0],
-  ): Promise<Awaited<ReturnType<ImageModelV3['doGenerate']>>> {
+    options: Parameters<ImageModelV4['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<ImageModelV4['doGenerate']>>> {
     const {
       prompt,
       n,
@@ -195,7 +219,7 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
       files,
       mask,
     } = options;
-    const warnings: Array<SharedV3Warning> = [];
+    const warnings: Array<SharedV4Warning> = [];
 
     // Gemini does not support mask-based inpainting
     if (mask != null) {
@@ -253,7 +277,7 @@ export class GoogleGenerativeAIImageModel implements ImageModelV3 {
       }
     }
 
-    const languageModelPrompt: LanguageModelV3Prompt = [
+    const languageModelPrompt: LanguageModelV4Prompt = [
       { role: 'user', content: userContent },
     ];