@ai-sdk/google 4.0.111 → 4.0.113

package/src/realtime/google-realtime-event-mapper.ts

@@ -4,6 +4,7 @@ import type {
   Experimental_RealtimeModelV4FunctionCallOutput as RealtimeModelV4FunctionCallOutput,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4Usage as RealtimeModelV4Usage,
 } from '@ai-sdk/provider';
 import { isRecord, safeParseJSON } from '@ai-sdk/provider-utils';
 import { getModelPath } from '../get-model-path';
@@ -31,6 +32,22 @@ type GoogleRealtimeServerContent = {
   waitingForInput?: boolean;
 };
 
+type GoogleRealtimeTokenDetail = {
+  modality?: string;
+  tokenCount?: number;
+};
+
+type GoogleRealtimeUsageMetadata = {
+  promptTokenCount?: number;
+  responseTokenCount?: number;
+  totalTokenCount?: number;
+  toolUsePromptTokenCount?: number;
+  thoughtsTokenCount?: number;
+  cachedContentTokenCount?: number;
+  promptTokensDetails?: GoogleRealtimeTokenDetail[];
+  responseTokensDetails?: GoogleRealtimeTokenDetail[];
+};
+
 type GoogleRealtimeWireEvent = {
   setupComplete?: unknown;
   toolCall?: {
@@ -38,6 +55,7 @@ type GoogleRealtimeWireEvent = {
   };
   toolCallCancellation?: unknown;
   serverContent?: GoogleRealtimeServerContent;
+  usageMetadata?: GoogleRealtimeUsageMetadata;
   inputTranscription?: { text?: string; finished?: boolean };
   goAway?: { timeLeft?: string };
   sessionResumptionUpdate?: {
@@ -150,6 +168,37 @@ export class GoogleRealtimeEventMapper {
 
   parseServerEvent(
     raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const events = this.parseEvent(raw);
+    const usage = mapGoogleUsage(
+      (raw as GoogleRealtimeWireEvent).usageMetadata,
+    );
+    if (usage == null) return events;
+    const list = Array.isArray(events) ? events : [events];
+    // Only a shared wire frame establishes an association with a completion.
+    if (list.some(event => event.type === 'response-done')) {
+      const mapped = list.map(event =>
+        event.type === 'response-done' ? { ...event, usage } : event,
+      );
+      return Array.isArray(events) ? mapped : mapped[0];
+    }
+    const update: RealtimeModelV4ServerEvent = {
+      type: 'custom',
+      rawType: 'usageMetadata',
+      usage,
+      raw,
+    };
+    if (
+      list.length === 1 &&
+      list[0].type === 'custom' &&
+      list[0].rawType === 'usageMetadata'
+    )
+      return update;
+    return [...list, update];
+  }
+
+  private parseEvent(
+    raw: unknown,
   ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
     const data = raw as GoogleRealtimeWireEvent;
 
@@ -448,6 +497,80 @@ export class GoogleRealtimeEventMapper {
   }
 }
 
+/**
+ * Maps Gemini Live `usageMetadata` to normalized usage. Google reports usage
+ * per modality in `promptTokensDetails` / `responseTokensDetails`.
+ *
+ * Field names are Live-API-specific: the Live `UsageMetadata` uses
+ * `responseTokenCount` / `responseTokensDetails` for output, NOT the
+ * `candidatesTokenCount` / `candidatesTokensDetails` of the regular
+ * `generateContent` response (see google-language-model.ts). Do not "align"
+ * these to `candidates*` — they are different wire shapes.
+ *
+ * Gross totals preserve provider counts even when modality details are absent
+ * or contain modalities without a dedicated normalized bucket. Never guess a
+ * text/audio split from those totals.
+ */
+function mapGoogleUsage(
+  usageMetadata: GoogleRealtimeUsageMetadata | undefined,
+): RealtimeModelV4Usage | undefined {
+  if (usageMetadata == null) return undefined;
+
+  const input = sumByModality(usageMetadata.promptTokensDetails);
+  const output = sumByModality(usageMetadata.responseTokensDetails);
+
+  return compactUsage({
+    inputTokens:
+      usageMetadata.promptTokenCount != null
+        ? usageMetadata.promptTokenCount +
+          (usageMetadata.toolUsePromptTokenCount ?? 0)
+        : undefined,
+    outputTokens:
+      usageMetadata.responseTokenCount != null
+        ? usageMetadata.responseTokenCount +
+          (usageMetadata.thoughtsTokenCount ?? 0)
+        : undefined,
+    cachedInputTokens: usageMetadata.cachedContentTokenCount,
+    inputTextTokens: input.text,
+    inputAudioTokens: input.audio,
+    outputTextTokens: output.text,
+    outputAudioTokens: output.audio,
+  });
+}
+
+/** Aggregate Gemini per-modality token details into text/audio buckets. */
+function sumByModality(details: GoogleRealtimeTokenDetail[] | undefined): {
+  text?: number;
+  audio?: number;
+} {
+  if (details == null) return {};
+  const totals: { text?: number; audio?: number } = {};
+  for (const detail of details) {
+    const count = detail.tokenCount;
+    if (count == null) continue;
+    const modality = detail.modality?.toUpperCase();
+    if (modality === 'AUDIO') {
+      totals.audio = (totals.audio ?? 0) + count;
+    } else if (modality === 'TEXT') {
+      totals.text = (totals.text ?? 0) + count;
+    }
+  }
+  return totals;
+}
+
+/** Drop `undefined` fields; return `undefined` when nothing is observable. */
+function compactUsage(
+  usage: RealtimeModelV4Usage,
+): RealtimeModelV4Usage | undefined {
+  const result: RealtimeModelV4Usage = {};
+  for (const [key, value] of Object.entries(usage)) {
+    if (value != null) {
+      result[key as keyof RealtimeModelV4Usage] = value;
+    }
+  }
+  return Object.keys(result).length > 0 ? result : undefined;
+}
+
 /**
  * The value Gemini accepts for `functionResponse.response`.
  *

package/src/realtime/google-realtime-model.ts

@@ -3,11 +3,15 @@ import type {
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
   Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerConnection as RealtimeModelV4ServerConnection,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
 } from '@ai-sdk/provider';
 import {
   resolve,
+  resolveWith,
+  removeUndefinedEntries,
   type Resolvable,
   type FetchFunction,
 } from '@ai-sdk/provider-utils';
@@ -23,6 +27,13 @@ import {
 const realtimeWebSocketPath =
   'google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained';
 
+// Server-side connections use the unconstrained Bidi endpoint: a trusted server
+// holds the API key and sends the session `setup` over the socket, rather than
+// baking it into an ephemeral token at mint time (which the constrained
+// endpoint requires).
+const realtimeServerWebSocketPath =
+  'google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
+
 function getAuthTokensURL(baseURL: string): string {
   const url = getRealtimeBaseURL(baseURL);
   url.pathname = `${url.pathname.replace(/\/$/, '')}/v1alpha/auth_tokens`;
@@ -33,6 +44,13 @@ function getWebSocketURL(baseURL: string): string {
   return getRealtimeWebSocketURL(baseURL, realtimeWebSocketPath).toString();
 }
 
+function getServerWebSocketURL(baseURL: string): string {
+  const url = getRealtimeBaseURL(baseURL);
+  url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+  url.pathname = `${url.pathname.replace(/\/$/, '')}/ws/${realtimeServerWebSocketPath}`;
+  return url.toString();
+}
+
 export type GoogleRealtimeModelConfig = {
   provider: string;
   baseURL: string;
@@ -42,6 +60,11 @@ export type GoogleRealtimeModelConfig = {
 
 export class GoogleRealtimeModel implements RealtimeModelV4 {
   readonly specificationVersion = 'v4' as const;
+  readonly capabilities = {
+    conversation: 'turn-based',
+    transports: ['websocket'],
+    connections: ['client-secret-websocket', 'server-websocket'],
+  } as const;
   readonly provider: string;
   readonly modelId: string;
 
@@ -132,21 +155,56 @@ export class GoogleRealtimeModel implements RealtimeModelV4 {
     };
   }
 
+  getServerWebSocketConfig(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }):
+    | RealtimeModelV4ServerConnection
+    | PromiseLike<RealtimeModelV4ServerConnection> {
+    // Gemini Live exposes only the conversational endpoint; fail fast instead
+    // of silently downgrading unsupported intents.
+    this.assertConversationIntent(options);
+    return resolveWith(this.config.headers, headers => ({
+      url: getServerWebSocketURL(this.config.baseURL),
+      headers: removeUndefinedEntries(headers),
+    }));
+  }
+
+  private assertConversationIntent(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }): void {
+    if (options?.intent != null && options.intent !== 'conversation') {
+      throw new Error(
+        `Google realtime does not support the '${options.intent}' session intent.`,
+      );
+    }
+  }
+
   parseServerEvent(
     raw: unknown,
   ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
     return this.mapper.parseServerEvent(raw);
   }
 
+  createServerEventParser(): ReturnType<
+    NonNullable<RealtimeModelV4['createServerEventParser']>
+  > {
+    const mapper = new GoogleRealtimeEventMapper();
+    return raw => mapper.parseServerEvent(raw);
+  }
+
   serializeClientEvent(
     event: RealtimeModelV4ClientEvent,
+    options?: { intent?: RealtimeModelV4SessionIntent },
   ): ReturnType<RealtimeModelV4['serializeClientEvent']> {
+    this.assertConversationIntent(options);
     return this.mapper.serializeClientEvent(event, this.modelId);
   }
 
   buildSessionConfig(
     config: RealtimeModelV4SessionConfig,
+    options?: { intent?: RealtimeModelV4SessionIntent },
   ): Record<string, unknown> {
+    this.assertConversationIntent(options);
     return buildGoogleSessionConfig(config, this.modelId);
   }
 }