@ai-sdk/google 4.0.0-canary.78 → 4.0.0-canary.79

package/docs/15-google.mdx

@@ -1065,6 +1065,32 @@ The following Zod features are known to not work with Google:
   available provider model ID as a string if needed.
 </Note>
 
+## Realtime Models
+
+<Note type="warning">Realtime is an experimental feature.</Note>
+
+You can create models that call the [Gemini Live API](https://ai.google.dev/gemini-api/docs/live)
+using the `.experimental_realtime()` factory method.
+
+```ts
+import { google } from '@ai-sdk/google';
+
+const model = google.experimental_realtime('gemini-3.1-flash-live-preview');
+```
+
+Realtime sessions run in the browser and require a short-lived token created on
+your server with `google.experimental_realtime.getToken()`:
+
+```ts
+const token = await google.experimental_realtime.getToken({
+  model: 'gemini-3.1-flash-live-preview',
+});
+```
+
+Google realtime models may require provider-specific audio formats, depending
+on the model and modality. See [Realtime](/docs/ai-sdk-core/realtime) for the
+complete setup and tool calling pattern.
+
 ## Interactions API
 
 The [Gemini Interactions API](https://ai.google.dev/gemini-api/docs/interactions)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.0-canary.78",
+  "version": "4.0.0-canary.79",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -35,8 +35,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.0-canary.17",
-    "@ai-sdk/provider-utils": "5.0.0-canary.45"
+    "@ai-sdk/provider": "4.0.0-canary.18",
+    "@ai-sdk/provider-utils": "5.0.0-canary.46"
   },
   "devDependencies": {
     "@types/node": "22.19.19",

package/src/google-provider.ts

@@ -5,6 +5,8 @@ import type {
   ImageModelV4,
   LanguageModelV4,
   ProviderV4,
+  Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
+  Experimental_RealtimeFactoryV4GetTokenOptions as RealtimeFactoryV4GetTokenOptions,
   SpeechModelV4,
 } from '@ai-sdk/provider';
 import {
@@ -37,6 +39,7 @@ import {
 } from './interactions/google-interactions-language-model';
 import type { GoogleInteractionsModelId } from './interactions/google-interactions-language-model-options';
 import type { GoogleInteractionsAgentName } from './interactions/google-interactions-agent';
+import { GoogleRealtimeModel } from './realtime/google-realtime-model';
 
 export interface GoogleProvider extends ProviderV4 {
   (modelId: GoogleModelId): LanguageModelV4;
@@ -115,6 +118,8 @@ export interface GoogleProvider extends ProviderV4 {
       | { managedAgent: string },
   ): LanguageModelV4;
 
+  experimental_realtime: RealtimeFactoryV4;
+
   tools: typeof googleTools;
 }
 
@@ -236,6 +241,14 @@ export function createGoogle(
       generateId: options.generateId ?? generateId,
     });
 
+  const createRealtimeModel = (modelId: string) =>
+    new GoogleRealtimeModel(modelId, {
+      provider: `${providerName}.realtime`,
+      baseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
   const createSpeechModel = (modelId: GoogleSpeechModelId) =>
     new GoogleSpeechModel(modelId, {
       provider: `${providerName}.speech`,
@@ -244,6 +257,25 @@ export function createGoogle(
       fetch: options.fetch,
     });
 
+  const experimentalRealtimeFactory = Object.assign(
+    (modelId: string) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions: RealtimeFactoryV4GetTokenOptions) => {
+        const model = createRealtimeModel(tokenOptions.model);
+        const secret = await model.doCreateClientSecret({
+          sessionConfig: tokenOptions.sessionConfig,
+          expiresAfterSeconds: tokenOptions.expiresAfterSeconds,
+        });
+
+        return {
+          token: secret.token,
+          url: secret.url,
+          expiresAt: secret.expiresAt,
+        };
+      },
+    },
+  ) as RealtimeFactoryV4;
+
   const createInteractionsModel = (
     modelIdOrAgent:
       | GoogleInteractionsModelId
@@ -283,6 +315,7 @@ export function createGoogle(
   provider.imageModel = createImageModel;
   provider.video = createVideoModel;
   provider.videoModel = createVideoModel;
+  provider.experimental_realtime = experimentalRealtimeFactory;
   provider.files = createFiles;
   provider.speech = createSpeechModel;
   provider.speechModel = createSpeechModel;

package/src/index.ts

@@ -54,5 +54,7 @@ export type {
   /** @deprecated Use `GoogleProviderSettings` instead. */
   GoogleProviderSettings as GoogleGenerativeAIProviderSettings,
 } from './google-provider';
+export { GoogleRealtimeModel as Experimental_GoogleRealtimeModel } from './realtime/google-realtime-model';
+export type { GoogleRealtimeModelConfig as Experimental_GoogleRealtimeModelConfig } from './realtime/google-realtime-model';
 
 export { VERSION } from './version';

package/src/realtime/google-realtime-event-mapper.ts

@@ -0,0 +1,383 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4FunctionCallOutput as RealtimeModelV4FunctionCallOutput,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import { safeParseJSON } from '@ai-sdk/provider-utils';
+import { convertJSONSchemaToOpenAPISchema } from '../convert-json-schema-to-openapi-schema';
+import { getModelPath } from '../get-model-path';
+
+type GoogleRealtimeFunctionCall = {
+  id: string;
+  name: string;
+  args?: Record<string, unknown>;
+};
+
+type GoogleRealtimeServerContent = {
+  interrupted?: boolean;
+  modelTurn?: {
+    parts?: Array<{
+      inlineData?: { data?: string };
+      text?: string;
+    }>;
+  };
+  outputTranscription?: { text?: string };
+  inputTranscription?: { text?: string };
+  turnComplete?: boolean;
+};
+
+type GoogleRealtimeWireEvent = {
+  setupComplete?: unknown;
+  toolCall?: {
+    functionCalls?: GoogleRealtimeFunctionCall[];
+  };
+  toolCallCancellation?: unknown;
+  serverContent?: GoogleRealtimeServerContent;
+  inputTranscription?: { text?: string };
+};
+
+/**
+ * Stateful event mapper for Google's Gemini Live API.
+ *
+ * Unlike OpenAI/xAI, Google's events don't have response/item IDs and
+ * a single message can contain multiple pieces of data. This class
+ * tracks turn state to generate consistent synthetic IDs.
+ */
+export class GoogleRealtimeEventMapper {
+  private turnCounter = 0;
+  private hasAudio = false;
+  private hasText = false;
+  private hasTranscript = false;
+  private turnClosed = false;
+  private inputAudioRate = 16000;
+
+  private get responseId(): string {
+    return `google-resp-${this.turnCounter}`;
+  }
+
+  private get itemId(): string {
+    return `google-item-${this.turnCounter}`;
+  }
+
+  /**
+   * Rolls over to the next turn lazily, only once new model content actually
+   * arrives. `turnComplete` merely marks the current turn closed; the counter
+   * is not advanced until the next response begins. This keeps a transcript
+   * that arrives shortly after `turnComplete` attached to the turn it belongs
+   * to, since Google delivers transcription independently with no guaranteed
+   * ordering relative to `turnComplete`.
+   */
+  private beginTurnIfClosed(): void {
+    if (!this.turnClosed) return;
+    this.turnCounter++;
+    this.hasAudio = false;
+    this.hasText = false;
+    this.hasTranscript = false;
+    this.turnClosed = false;
+  }
+
+  parseServerEvent(
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const data = raw as GoogleRealtimeWireEvent;
+
+    if (data.setupComplete != null) {
+      return { type: 'session-created', raw };
+    }
+
+    if (data.toolCall != null) {
+      this.beginTurnIfClosed();
+      const functionCalls = data.toolCall.functionCalls ?? [];
+      return functionCalls.flatMap(functionCall => {
+        const args = JSON.stringify(functionCall.args ?? {});
+        return [
+          {
+            type: 'function-call-arguments-delta' as const,
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            delta: args,
+            raw,
+          },
+          {
+            type: 'function-call-arguments-done' as const,
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            name: functionCall.name,
+            arguments: args,
+            raw,
+          },
+        ];
+      });
+    }
+
+    if (data.toolCallCancellation != null) {
+      return {
+        type: 'custom',
+        rawType: 'toolCallCancellation',
+        raw,
+      };
+    }
+
+    if (data.serverContent != null) {
+      return this.parseServerContent(data.serverContent, raw);
+    }
+
+    if (data.inputTranscription?.text != null) {
+      return {
+        type: 'input-transcription-completed',
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: data.inputTranscription.text,
+        raw,
+      };
+    }
+
+    return { type: 'custom', rawType: String(Object.keys(data)[0]), raw };
+  }
+
+  private parseServerContent(
+    serverContent: GoogleRealtimeServerContent,
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const events: RealtimeModelV4ServerEvent[] = [];
+
+    if (serverContent.interrupted) {
+      events.push({
+        type: 'speech-started',
+        raw,
+      });
+    }
+
+    if (serverContent.modelTurn?.parts) {
+      // New model response content marks the start of the next turn.
+      this.beginTurnIfClosed();
+      for (const part of serverContent.modelTurn.parts) {
+        if (part.inlineData?.data) {
+          this.hasAudio = true;
+          events.push({
+            type: 'audio-delta',
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.inlineData.data,
+            raw,
+          });
+        }
+        if (part.text) {
+          this.hasText = true;
+          events.push({
+            type: 'text-delta',
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.text,
+            raw,
+          });
+        }
+      }
+    }
+
+    if (serverContent.outputTranscription?.text) {
+      this.hasTranscript = true;
+      events.push({
+        type: 'audio-transcript-delta',
+        responseId: this.responseId,
+        itemId: this.itemId,
+        delta: serverContent.outputTranscription.text,
+        raw,
+      });
+    }
+
+    if (serverContent.inputTranscription?.text) {
+      events.push({
+        type: 'input-transcription-completed',
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: serverContent.inputTranscription.text,
+        raw,
+      });
+    }
+
+    if (serverContent.turnComplete) {
+      if (this.hasAudio) {
+        events.push({
+          type: 'audio-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      if (this.hasText) {
+        events.push({
+          type: 'text-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      if (this.hasTranscript) {
+        events.push({
+          type: 'audio-transcript-done',
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw,
+        });
+      }
+      events.push({
+        type: 'response-done',
+        responseId: this.responseId,
+        status: 'completed',
+        raw,
+      });
+      // Mark the turn closed but defer advancing the counter until the next
+      // response actually begins (see `beginTurnIfClosed`).
+      this.turnClosed = true;
+    }
+
+    if (events.length === 0) {
+      return { type: 'custom', rawType: 'serverContent', raw };
+    }
+
+    return events.length === 1 ? events[0] : events;
+  }
+
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+    modelId: string,
+  ): ReturnType<RealtimeModelV4['serializeClientEvent']> {
+    switch (event.type) {
+      case 'session-update':
+        // Capture the configured capture rate so input audio blobs advertise
+        // the real rate. Google accepts any rate as long as the blob's mimeType
+        // matches; a mismatched label corrupts custom-rate audio.
+        if (event.config.inputAudioFormat?.rate != null) {
+          this.inputAudioRate = event.config.inputAudioFormat.rate;
+        }
+        return {
+          setup: buildGoogleSessionConfig(event.config, modelId),
+        };
+
+      case 'input-audio-append':
+        return {
+          realtimeInput: {
+            audio: {
+              data: event.audio,
+              mimeType: `audio/pcm;rate=${this.inputAudioRate}`,
+            },
+          },
+        };
+
+      case 'input-audio-commit':
+      case 'input-audio-clear':
+      case 'response-create':
+      case 'response-cancel':
+      case 'conversation-item-truncate':
+        return null;
+
+      case 'conversation-item-create': {
+        const item = event.item;
+        switch (item.type) {
+          case 'text-message':
+            return {
+              realtimeInput: {
+                text: item.text,
+              },
+            };
+          case 'function-call-output':
+            return serializeFunctionCallOutput(item);
+          case 'audio-message':
+            return null;
+        }
+        break;
+      }
+    }
+
+    return null;
+  }
+}
+
+async function serializeFunctionCallOutput(
+  item: RealtimeModelV4FunctionCallOutput,
+): Promise<unknown> {
+  const parseResult = await safeParseJSON({ text: item.output });
+  const response = parseResult.success ? parseResult.value : {};
+
+  return {
+    toolResponse: {
+      functionResponses: [
+        {
+          id: item.callId,
+          name: item.name,
+          response,
+        },
+      ],
+    },
+  };
+}
+
+/**
+ * Builds a Google-specific session configuration from a normalized config.
+ * Used to construct the `bidiGenerateContentSetup` payload for auth token creation.
+ */
+export function buildGoogleSessionConfig(
+  config: RealtimeModelV4SessionConfig | undefined,
+  modelId: string,
+): Record<string, unknown> {
+  const setup: Record<string, unknown> = {
+    model: getModelPath(modelId),
+  };
+
+  const generationConfig: Record<string, unknown> = {};
+
+  if (config?.outputModalities != null) {
+    generationConfig.responseModalities = config.outputModalities.map(m =>
+      m.toUpperCase(),
+    );
+  } else {
+    generationConfig.responseModalities = ['AUDIO'];
+  }
+
+  if (config?.voice != null) {
+    generationConfig.speechConfig = {
+      voiceConfig: {
+        prebuiltVoiceConfig: {
+          voiceName: config.voice,
+        },
+      },
+    };
+  }
+
+  setup.generationConfig = generationConfig;
+
+  if (config?.instructions != null) {
+    setup.systemInstruction = {
+      parts: [{ text: config.instructions }],
+    };
+  }
+
+  if (config?.tools != null && config.tools.length > 0) {
+    setup.tools = [
+      {
+        functionDeclarations: config.tools.map(tool => ({
+          name: tool.name,
+          description: tool.description,
+          parameters: convertJSONSchemaToOpenAPISchema(tool.parameters),
+        })),
+      },
+    ];
+  }
+
+  if (config?.inputAudioTranscription != null) {
+    setup.inputAudioTranscription = {};
+  }
+
+  if (config?.outputAudioTranscription != null) {
+    setup.outputAudioTranscription = {};
+  }
+
+  if (config?.providerOptions != null) {
+    Object.assign(setup, config.providerOptions);
+  }
+
+  return setup;
+}

package/src/realtime/google-realtime-model-options.ts

@@ -0,0 +1,3 @@
+export type GoogleRealtimeModelId = string;
+
+export type GoogleRealtimeModelOptions = Record<string, never>;

package/src/realtime/google-realtime-model.ts

@@ -0,0 +1,160 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
+  Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import type { FetchFunction } from '@ai-sdk/provider-utils';
+import {
+  GoogleRealtimeEventMapper,
+  buildGoogleSessionConfig,
+} from './google-realtime-event-mapper';
+
+const realtimeWebSocketPath =
+  'google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained';
+
+function getRealtimeBaseURL(baseURL: string): URL {
+  const url = new URL(baseURL);
+  const pathSegments = url.pathname.split('/');
+  const version = pathSegments.at(-1);
+
+  if (version === 'v1beta' || version === 'v1alpha') {
+    pathSegments.pop();
+    url.pathname = pathSegments.join('/') || '/';
+  }
+
+  return url;
+}
+
+function getAuthTokensURL(baseURL: string): string {
+  const url = getRealtimeBaseURL(baseURL);
+  url.pathname = `${url.pathname.replace(/\/$/, '')}/v1alpha/auth_tokens`;
+  return url.toString();
+}
+
+function getWebSocketURL(baseURL: string): string {
+  const url = getRealtimeBaseURL(baseURL);
+  url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+  url.pathname = `${url.pathname.replace(/\/$/, '')}/ws/${realtimeWebSocketPath}`;
+  return url.toString();
+}
+
+export type GoogleRealtimeModelConfig = {
+  provider: string;
+  baseURL: string;
+  headers: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+};
+
+export class GoogleRealtimeModel implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly provider: string;
+  readonly modelId: string;
+
+  private readonly config: GoogleRealtimeModelConfig;
+  private readonly mapper = new GoogleRealtimeEventMapper();
+
+  constructor(modelId: string, config: GoogleRealtimeModelConfig) {
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+
+  async doCreateClientSecret(
+    options: RealtimeModelV4ClientSecretOptions,
+  ): Promise<RealtimeModelV4ClientSecretResult> {
+    const fetchFn = this.config.fetch ?? fetch;
+    const headers = this.config.headers();
+    const apiKey = headers['x-goog-api-key'];
+
+    if (!apiKey) {
+      throw new Error(
+        'Google Generative AI API key is required for realtime token creation.',
+      );
+    }
+
+    // `newSessionExpireTime` controls how long the token can be used to *open*
+    // a session — the window callers actually care about — so map
+    // `expiresAfterSeconds` to it (Google otherwise defaults it to ~60s).
+    // `expireTime` is the overall token lifetime and must be >=
+    // `newSessionExpireTime`, so extend it to leave room for the opened session
+    // to run.
+    const now = Date.now();
+    const openWindowMs = (options.expiresAfterSeconds ?? 60) * 1000;
+    const newSessionExpireTime = new Date(now + openWindowMs).toISOString();
+    const expireTime = new Date(
+      now + openWindowMs + 30 * 60 * 1000,
+    ).toISOString();
+
+    const setupPayload = buildGoogleSessionConfig(
+      options.sessionConfig,
+      this.modelId,
+    );
+
+    const response = await fetchFn(
+      `${getAuthTokensURL(this.config.baseURL)}?key=${encodeURIComponent(apiKey)}`,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          // `uses: 0` means no limit is applied to how many times the token can
+          // start a session (per the AuthToken spec). An unset value would
+          // default to 1, which breaks WebSocket reconnects within the session.
+          uses: 0,
+          expireTime,
+          newSessionExpireTime,
+          bidiGenerateContentSetup: setupPayload,
+        }),
+      },
+    );
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `Google realtime auth token request failed: ${response.status} ${text}`,
+      );
+    }
+
+    const data = (await response.json()) as {
+      name: string;
+      expireTime?: string;
+    };
+
+    return {
+      token: data.name,
+      url: getWebSocketURL(this.config.baseURL),
+      expiresAt: data.expireTime
+        ? Math.floor(new Date(data.expireTime).getTime() / 1000)
+        : undefined,
+    };
+  }
+
+  getWebSocketConfig(options: { token: string; url: string }): {
+    url: string;
+    protocols?: string[];
+  } {
+    return {
+      url: `${options.url}?access_token=${encodeURIComponent(options.token)}`,
+    };
+  }
+
+  parseServerEvent(
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    return this.mapper.parseServerEvent(raw);
+  }
+
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+  ): ReturnType<RealtimeModelV4['serializeClientEvent']> {
+    return this.mapper.serializeClientEvent(event, this.modelId);
+  }
+
+  buildSessionConfig(
+    config: RealtimeModelV4SessionConfig,
+  ): Record<string, unknown> {
+    return buildGoogleSessionConfig(config, this.modelId);
+  }
+}

package/src/realtime/index.ts

@@ -0,0 +1,2 @@
+export { GoogleRealtimeModel as Experimental_GoogleRealtimeModel } from './google-realtime-model';
+export type { GoogleRealtimeModelConfig as Experimental_GoogleRealtimeModelConfig } from './google-realtime-model';