@ai-sdk/google 4.0.0-beta.7 → 4.0.0-beta.82

package/src/realtime/google-realtime-event-mapper.ts

@@ -0,0 +1,383 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4FunctionCallOutput as RealtimeModelV4FunctionCallOutput,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import { safeParseJSON } from '@ai-sdk/provider-utils';
+import { convertJSONSchemaToOpenAPISchema } from '../convert-json-schema-to-openapi-schema';
+import { getModelPath } from '../get-model-path';
+
+type GoogleRealtimeFunctionCall = {
+  id: string;
+  name: string;
+  args?: Record<string, unknown>;
+};
+
+type GoogleRealtimeServerContent = {
+  interrupted?: boolean;
+  modelTurn?: {
+    parts?: Array<{
+      inlineData?: { data?: string };
+      text?: string;
+    }>;
+  };
+  outputTranscription?: { text?: string };
+  inputTranscription?: { text?: string };
+  turnComplete?: boolean;
+};
+
+type GoogleRealtimeWireEvent = {
+  setupComplete?: unknown;
+  toolCall?: {
+    functionCalls?: GoogleRealtimeFunctionCall[];
+  };
+  toolCallCancellation?: unknown;
+  serverContent?: GoogleRealtimeServerContent;
+  inputTranscription?: { text?: string };
+};
+
+/**
+ * Stateful event mapper for Google's Gemini Live API.
+ *
+ * Unlike OpenAI/xAI, Google's events don't have response/item IDs and
+ * a single message can contain multiple pieces of data. This class
+ * tracks turn state to generate consistent synthetic IDs.
+ */
+export class GoogleRealtimeEventMapper {
+  private turnCounter = 0;
+  private hasAudio = false;
+  private hasText = false;
+  private hasTranscript = false;
+  private turnClosed = false;
+  private inputAudioRate = 16000;
+
+  private get responseId(): string {
+    return `google-resp-${this.turnCounter}`;
+  }
+
+  private get itemId(): string {
+    return `google-item-${this.turnCounter}`;
+  }
+
+  /**
+   * Rolls over to the next turn lazily, only once new model content actually
+   * arrives. `turnComplete` merely marks the current turn closed; the counter
+   * is not advanced until the next response begins. This keeps a transcript
+   * that arrives shortly after `turnComplete` attached to the turn it belongs
+   * to, since Google delivers transcription independently with no guaranteed
+   * ordering relative to `turnComplete`.
+   */
+  private beginTurnIfClosed(): void {
+    if (!this.turnClosed) return;
+    this.turnCounter++;
+    this.hasAudio = false;
+    this.hasText = false;
+    this.hasTranscript = false;
+    this.turnClosed = false;
+  }
+
+  parseServerEvent(
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const data = raw as GoogleRealtimeWireEvent;
+
+    if (data.setupComplete != null) {
+      return { type: 'session-created', raw };
+    }
+
+    if (data.toolCall != null) {
+      this.beginTurnIfClosed();
+      const functionCalls = data.toolCall.functionCalls ?? [];
+      return functionCalls.flatMap(functionCall => {
+        const args = JSON.stringify(functionCall.args ?? {});
+        return [
+          {
+            type: 'function-call-arguments-delta' as const,
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            delta: args,
+            raw,
+          },
+          {
+            type: 'function-call-arguments-done' as const,
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            name: functionCall.name,
+            arguments: args,
+            raw,
+          },
+        ];
+      });
+    }
+
+    if (data.toolCallCancellation != null) {
+      return {
+        type: 'custom',
+        rawType: 'toolCallCancellation',
+        raw,
+      };
+    }
+
+    if (data.serverContent != null) {
+      return this.parseServerContent(data.serverContent, raw);
+    }
+
+    if (data.inputTranscription?.text != null) {
+      return {
+        type: 'input-transcription-completed',
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: data.inputTranscription.text,
+        raw,
+      };
+    }
+
+    return { type: 'custom', rawType: String(Object.keys(data)[0]), raw };
+  }
+
+  private parseServerContent(
+    serverContent: GoogleRealtimeServerContent,
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const events: RealtimeModelV4ServerEvent[] = [];
+
+    if (serverContent.interrupted) {
+      events.push({
+        type: 'speech-started',
+        raw,
+      });
+    }
+
+    if (serverContent.modelTurn?.parts) {
+      // New model response content marks the start of the next turn.
+      this.beginTurnIfClosed();
+      for (const part of serverContent.modelTurn.parts) {
+        if (part.inlineData?.data) {
+          this.hasAudio = true;
+          events.push({
+            type: 'audio-delta',
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.inlineData.data,
+            raw,
+          });
+        }
+        if (part.text) {
+          this.hasText = true;
+          events.push({
+            type: 'text-delta',
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.text,
+            raw,
+          });
+        }
+      }
+    }
+
+    if (serverContent.outputTranscription?.text) {
+      this.hasTranscript = true;
+      events.push({
+        type: 'audio-transcript-delta',
+        responseId: this.responseId,
+        itemId: this.itemId,
+        delta: serverContent.outputTranscription.text,
+        raw,
+      });
+    }
+
+    if (serverContent.inputTranscription?.text) {
+      events.push({
+        type: 'input-transcription-completed',
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: serverContent.inputTranscription.text,
+        raw,
+      });
+    }
+
+    if (serverContent.turnComplete) {
+      if (this.hasAudio) {
+        events.push({
+          type: 'audio-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      if (this.hasText) {
+        events.push({
+          type: 'text-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      if (this.hasTranscript) {
+        events.push({
+          type: 'audio-transcript-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      events.push({
+        type: 'response-done',
+        responseId: this.responseId,
+        status: 'completed',
+        raw,
+      });
+      // Mark the turn closed but defer advancing the counter until the next
+      // response actually begins (see `beginTurnIfClosed`).
+      this.turnClosed = true;
+    }
+
+    if (events.length === 0) {
+      return { type: 'custom', rawType: 'serverContent', raw };
+    }
+
+    return events.length === 1 ? events[0] : events;
+  }
+
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+    modelId: string,
+  ): ReturnType<RealtimeModelV4['serializeClientEvent']> {
+    switch (event.type) {
+      case 'session-update':
+        // Capture the configured capture rate so input audio blobs advertise
+        // the real rate. Google accepts any rate as long as the blob's mimeType
+        // matches; a mismatched label corrupts custom-rate audio.
+        if (event.config.inputAudioFormat?.rate != null) {
+          this.inputAudioRate = event.config.inputAudioFormat.rate;
+        }
+        return {
+          setup: buildGoogleSessionConfig(event.config, modelId),
+        };
+
+      case 'input-audio-append':
+        return {
+          realtimeInput: {
+            audio: {
+              data: event.audio,
+              mimeType: `audio/pcm;rate=${this.inputAudioRate}`,
+            },
+          },
+        };
+
+      case 'input-audio-commit':
+      case 'input-audio-clear':
+      case 'response-create':
+      case 'response-cancel':
+      case 'conversation-item-truncate':
+        return null;
+
+      case 'conversation-item-create': {
+        const item = event.item;
+        switch (item.type) {
+          case 'text-message':
+            return {
+              realtimeInput: {
+                text: item.text,
+              },
+            };
+          case 'function-call-output':
+            return serializeFunctionCallOutput(item);
+          case 'audio-message':
+            return null;
+        }
+        break;
+      }
+    }
+
+    return null;
+  }
+}
+
+async function serializeFunctionCallOutput(
+  item: RealtimeModelV4FunctionCallOutput,
+): Promise<unknown> {
+  const parseResult = await safeParseJSON({ text: item.output });
+  const response = parseResult.success ? parseResult.value : {};
+
+  return {
+    toolResponse: {
+      functionResponses: [
+        {
+          id: item.callId,
+          name: item.name,
+          response,
+        },
+      ],
+    },
+  };
+}
+
+/**
+ * Builds a Google-specific session configuration from a normalized config.
+ * Used to construct the `bidiGenerateContentSetup` payload for auth token creation.
+ */
+export function buildGoogleSessionConfig(
+  config: RealtimeModelV4SessionConfig | undefined,
+  modelId: string,
+): Record<string, unknown> {
+  const setup: Record<string, unknown> = {
+    model: getModelPath(modelId),
+  };
+
+  const generationConfig: Record<string, unknown> = {};
+
+  if (config?.outputModalities != null) {
+    generationConfig.responseModalities = config.outputModalities.map(m =>
+      m.toUpperCase(),
+    );
+  } else {
+    generationConfig.responseModalities = ['AUDIO'];
+  }
+
+  if (config?.voice != null) {
+    generationConfig.speechConfig = {
+      voiceConfig: {
+        prebuiltVoiceConfig: {
+          voiceName: config.voice,
+        },
+      },
+    };
+  }
+
+  setup.generationConfig = generationConfig;
+
+  if (config?.instructions != null) {
+    setup.systemInstruction = {
+      parts: [{ text: config.instructions }],
+    };
+  }
+
+  if (config?.tools != null && config.tools.length > 0) {
+    setup.tools = [
+      {
+        functionDeclarations: config.tools.map(tool => ({
+          name: tool.name,
+          description: tool.description,
+          parameters: convertJSONSchemaToOpenAPISchema(tool.parameters),
+        })),
+      },
+    ];
+  }
+
+  if (config?.inputAudioTranscription != null) {
+    setup.inputAudioTranscription = {};
+  }
+
+  if (config?.outputAudioTranscription != null) {
+    setup.outputAudioTranscription = {};
+  }
+
+  if (config?.providerOptions != null) {
+    Object.assign(setup, config.providerOptions);
+  }
+
+  return setup;
+}

package/src/realtime/google-realtime-model-options.ts

@@ -0,0 +1,3 @@
+export type GoogleRealtimeModelId = string;
+
+export type GoogleRealtimeModelOptions = Record<string, never>;

package/src/realtime/google-realtime-model.ts

@@ -0,0 +1,160 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
+  Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import type { FetchFunction } from '@ai-sdk/provider-utils';
+import {
+  GoogleRealtimeEventMapper,
+  buildGoogleSessionConfig,
+} from './google-realtime-event-mapper';
+
+const realtimeWebSocketPath =
+  'google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained';
+
+function getRealtimeBaseURL(baseURL: string): URL {
+  const url = new URL(baseURL);
+  const pathSegments = url.pathname.split('/');
+  const version = pathSegments.at(-1);
+
+  if (version === 'v1beta' || version === 'v1alpha') {
+    pathSegments.pop();
+    url.pathname = pathSegments.join('/') || '/';
+  }
+
+  return url;
+}
+
+function getAuthTokensURL(baseURL: string): string {
+  const url = getRealtimeBaseURL(baseURL);
+  url.pathname = `${url.pathname.replace(/\/$/, '')}/v1alpha/auth_tokens`;
+  return url.toString();
+}
+
+function getWebSocketURL(baseURL: string): string {
+  const url = getRealtimeBaseURL(baseURL);
+  url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+  url.pathname = `${url.pathname.replace(/\/$/, '')}/ws/${realtimeWebSocketPath}`;
+  return url.toString();
+}
+
+export type GoogleRealtimeModelConfig = {
+  provider: string;
+  baseURL: string;
+  headers: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+};
+
+export class GoogleRealtimeModel implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly provider: string;
+  readonly modelId: string;
+
+  private readonly config: GoogleRealtimeModelConfig;
+  private readonly mapper = new GoogleRealtimeEventMapper();
+
+  constructor(modelId: string, config: GoogleRealtimeModelConfig) {
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+
+  async doCreateClientSecret(
+    options: RealtimeModelV4ClientSecretOptions,
+  ): Promise<RealtimeModelV4ClientSecretResult> {
+    const fetchFn = this.config.fetch ?? fetch;
+    const headers = this.config.headers();
+    const apiKey = headers['x-goog-api-key'];
+
+    if (!apiKey) {
+      throw new Error(
+        'Google Generative AI API key is required for realtime token creation.',
+      );
+    }
+
+    // `newSessionExpireTime` controls how long the token can be used to *open*
+    // a session — the window callers actually care about — so map
+    // `expiresAfterSeconds` to it (Google otherwise defaults it to ~60s).
+    // `expireTime` is the overall token lifetime and must be >=
+    // `newSessionExpireTime`, so extend it to leave room for the opened session
+    // to run.
+    const now = Date.now();
+    const openWindowMs = (options.expiresAfterSeconds ?? 60) * 1000;
+    const newSessionExpireTime = new Date(now + openWindowMs).toISOString();
+    const expireTime = new Date(
+      now + openWindowMs + 30 * 60 * 1000,
+    ).toISOString();
+
+    const setupPayload = buildGoogleSessionConfig(
+      options.sessionConfig,
+      this.modelId,
+    );
+
+    const response = await fetchFn(
+      `${getAuthTokensURL(this.config.baseURL)}?key=${encodeURIComponent(apiKey)}`,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          // `uses: 0` means no limit is applied to how many times the token can
+          // start a session (per the AuthToken spec). An unset value would
+          // default to 1, which breaks WebSocket reconnects within the session.
+          uses: 0,
+          expireTime,
+          newSessionExpireTime,
+          bidiGenerateContentSetup: setupPayload,
+        }),
+      },
+    );
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `Google realtime auth token request failed: ${response.status} ${text}`,
+      );
+    }
+
+    const data = (await response.json()) as {
+      name: string;
+      expireTime?: string;
+    };
+
+    return {
+      token: data.name,
+      url: getWebSocketURL(this.config.baseURL),
+      expiresAt: data.expireTime
+        ? Math.floor(new Date(data.expireTime).getTime() / 1000)
+        : undefined,
+    };
+  }
+
+  getWebSocketConfig(options: { token: string; url: string }): {
+    url: string;
+    protocols?: string[];
+  } {
+    return {
+      url: `${options.url}?access_token=${encodeURIComponent(options.token)}`,
+    };
+  }
+
+  parseServerEvent(
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    return this.mapper.parseServerEvent(raw);
+  }
+
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+  ): ReturnType<RealtimeModelV4['serializeClientEvent']> {
+    return this.mapper.serializeClientEvent(event, this.modelId);
+  }
+
+  buildSessionConfig(
+    config: RealtimeModelV4SessionConfig,
+  ): Record<string, unknown> {
+    return buildGoogleSessionConfig(config, this.modelId);
+  }
+}

package/src/realtime/index.ts

@@ -0,0 +1,2 @@
+export { GoogleRealtimeModel as Experimental_GoogleRealtimeModel } from './google-realtime-model';
+export type { GoogleRealtimeModelConfig as Experimental_GoogleRealtimeModelConfig } from './google-realtime-model';

package/src/tool/code-execution.ts

@@ -1,4 +1,4 @@
-import { createProviderToolFactoryWithOutputSchema } from '@ai-sdk/provider-utils';
+import { createProviderExecutedToolFactory } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
 
 /**
@@ -10,7 +10,7 @@ import { z } from 'zod/v4';
  * @see https://ai.google.dev/gemini-api/docs/code-execution (Google AI)
  * @see https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/code-execution-api (Vertex AI)
  */
-export const codeExecution = createProviderToolFactoryWithOutputSchema<
+export const codeExecution = createProviderExecutedToolFactory<
   {
     language: string;
     code: string;

package/src/tool/enterprise-web-search.ts

@@ -1,5 +1,5 @@
 import {
-  createProviderToolFactory,
+  createProviderExecutedToolFactory,
   lazySchema,
   zodSchema,
 } from '@ai-sdk/provider-utils';
@@ -7,12 +7,18 @@ import { z } from 'zod/v4';
 
 // https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/web-grounding-enterprise
 
-export const enterpriseWebSearch = createProviderToolFactory<
+export const enterpriseWebSearch = createProviderExecutedToolFactory<
   {
     // Enterprise Web Search does not have any input schema
   },
-  {}
+  {
+    // Enterprise Web Search does not have any output parameters
+  },
+  {
+    // Enterprise Web Search does not have any configuration options
+  }
 >({
   id: 'google.enterprise_web_search',
   inputSchema: lazySchema(() => zodSchema(z.object({}))),
+  outputSchema: lazySchema(() => zodSchema(z.object({}))),
 });

package/src/tool/file-search.ts

@@ -1,5 +1,5 @@
 import {
-  createProviderToolFactory,
+  createProviderExecutedToolFactory,
   lazySchema,
   zodSchema,
 } from '@ai-sdk/provider-utils';
@@ -38,14 +38,12 @@ const fileSearchArgsBaseSchema = z
 
 export type GoogleFileSearchToolArgs = z.infer<typeof fileSearchArgsBaseSchema>;
 
-const fileSearchArgsSchema = lazySchema(() =>
-  zodSchema(fileSearchArgsBaseSchema),
-);
-
-export const fileSearch = createProviderToolFactory<
+export const fileSearch = createProviderExecutedToolFactory<
+  {},
   {},
   GoogleFileSearchToolArgs
 >({
   id: 'google.file_search',
-  inputSchema: fileSearchArgsSchema,
+  inputSchema: lazySchema(() => zodSchema(z.object({}))),
+  outputSchema: lazySchema(() => zodSchema(z.object({}))),
 });

package/src/tool/google-maps.ts

@@ -1,5 +1,5 @@
 import {
-  createProviderToolFactory,
+  createProviderExecutedToolFactory,
   lazySchema,
   zodSchema,
 } from '@ai-sdk/provider-utils';
@@ -8,7 +8,8 @@ import { z } from 'zod/v4';
 // https://ai.google.dev/gemini-api/docs/maps-grounding
 // https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/grounding-with-google-maps
 
-export const googleMaps = createProviderToolFactory<{}, {}>({
+export const googleMaps = createProviderExecutedToolFactory<{}, {}, {}>({
   id: 'google.google_maps',
   inputSchema: lazySchema(() => zodSchema(z.object({}))),
+  outputSchema: lazySchema(() => zodSchema(z.object({}))),
 });

package/src/tool/google-search.ts

@@ -1,5 +1,5 @@
 import {
-  createProviderToolFactory,
+  createProviderExecutedToolFactory,
   lazySchema,
   zodSchema,
 } from '@ai-sdk/provider-utils';
@@ -9,7 +9,7 @@ import { z } from 'zod/v4';
 // https://ai.google.dev/api/generate-content#GroundingSupport
 // https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/grounding-with-google-search
 
-const googleSearchToolArgsBaseSchema = z
+export const googleSearchToolArgsBaseSchema = z
   .object({
     searchTypes: z
       .object({
@@ -31,13 +31,12 @@ export type GoogleSearchToolArgs = z.infer<
   typeof googleSearchToolArgsBaseSchema
 >;
 
-const googleSearchToolArgsSchema = lazySchema(() =>
-  zodSchema(googleSearchToolArgsBaseSchema),
-);
-
-export const googleSearch = createProviderToolFactory<{}, GoogleSearchToolArgs>(
-  {
-    id: 'google.google_search',
-    inputSchema: googleSearchToolArgsSchema,
-  },
-);
+export const googleSearch = createProviderExecutedToolFactory<
+  {},
+  {},
+  GoogleSearchToolArgs
+>({
+  id: 'google.google_search',
+  inputSchema: lazySchema(() => zodSchema(z.object({}))),
+  outputSchema: lazySchema(() => zodSchema(z.object({}))),
+});

package/src/tool/url-context.ts

@@ -1,16 +1,18 @@
 import {
-  createProviderToolFactory,
+  createProviderExecutedToolFactory,
   lazySchema,
   zodSchema,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
 
-export const urlContext = createProviderToolFactory<
+export const urlContext = createProviderExecutedToolFactory<
   {
     // Url context does not have any input schema, it will directly use the url from the prompt
   },
+  {},
   {}
 >({
   id: 'google.url_context',
   inputSchema: lazySchema(() => zodSchema(z.object({}))),
+  outputSchema: lazySchema(() => zodSchema(z.object({}))),
 });