@ai-sdk/google 4.0.0-canary.78 → 4.0.0-canary.80

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @ai-sdk/google
 
+## 4.0.0-canary.80
+
+### Patch Changes
+
+- Updated dependencies [bae5e2b]
+  - @ai-sdk/provider-utils@5.0.0-canary.47
+
+## 4.0.0-canary.79
+
+### Patch Changes
+
+- ce769dd: feat(provider): add experimental Realtime API support for voice conversations
+
+  Adds first-class support for realtime (speech-to-speech) APIs:
+
+  - `Experimental_RealtimeModelV4` spec in `@ai-sdk/provider` with normalized event types and factory
+  - OpenAI, Google, and xAI realtime provider implementations
+  - `openai.experimental_realtime()` / `google.experimental_realtime()` / `xai.experimental_realtime()` work in both server and browser
+  - `.getToken()` static method on each provider for server-side ephemeral token creation
+  - `experimental_getRealtimeToolDefinitions` helper for provider session tool definitions
+  - `experimental_useRealtime` hook in `@ai-sdk/react` returning `UIMessage[]` (aligned with `useChat`), with `onToolCall` and `addToolOutput` for client-driven tool execution
+  - `inputAudioTranscription` session config for showing transcribed user audio messages when supported by the provider
+
+- Updated dependencies [ce769dd]
+  - @ai-sdk/provider@4.0.0-canary.18
+  - @ai-sdk/provider-utils@5.0.0-canary.46
+
 ## 4.0.0-canary.78
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
 import { InferSchema, FetchFunction } from '@ai-sdk/provider-utils';
-import { ProviderV4, LanguageModelV4, ImageModelV4, EmbeddingModelV4, Experimental_VideoModelV4, SpeechModelV4, FilesV4 } from '@ai-sdk/provider';
+import { ProviderV4, LanguageModelV4, ImageModelV4, EmbeddingModelV4, Experimental_VideoModelV4, SpeechModelV4, FilesV4, Experimental_RealtimeFactoryV4, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig } from '@ai-sdk/provider';
 
 declare const googleErrorDataSchema: _ai_sdk_provider_utils.LazySchema<{
     error: {
@@ -554,6 +554,7 @@ interface GoogleProvider extends ProviderV4 {
     } | {
         managedAgent: string;
     }): LanguageModelV4;
+    experimental_realtime: Experimental_RealtimeFactoryV4;
     tools: typeof googleTools;
 }
 interface GoogleProviderSettings {
@@ -595,6 +596,32 @@ declare function createGoogle(options?: GoogleProviderSettings): GoogleProvider;
  */
 declare const google: GoogleProvider;
 
+type GoogleRealtimeModelConfig = {
+    provider: string;
+    baseURL: string;
+    headers: () => Record<string, string | undefined>;
+    fetch?: FetchFunction;
+};
+declare class GoogleRealtimeModel implements Experimental_RealtimeModelV4 {
+    readonly specificationVersion: "v4";
+    readonly provider: string;
+    readonly modelId: string;
+    private readonly config;
+    private readonly mapper;
+    constructor(modelId: string, config: GoogleRealtimeModelConfig);
+    doCreateClientSecret(options: Experimental_RealtimeModelV4ClientSecretOptions): Promise<Experimental_RealtimeModelV4ClientSecretResult>;
+    getWebSocketConfig(options: {
+        token: string;
+        url: string;
+    }): {
+        url: string;
+        protocols?: string[];
+    };
+    parseServerEvent(raw: unknown): Experimental_RealtimeModelV4ServerEvent | Experimental_RealtimeModelV4ServerEvent[];
+    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): ReturnType<Experimental_RealtimeModelV4['serializeClientEvent']>;
+    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
+}
+
 declare const VERSION: string;
 
-export { type GoogleEmbeddingModelOptions, type GoogleErrorData, type GoogleFilesUploadOptions, type GoogleEmbeddingModelOptions as GoogleGenerativeAIEmbeddingProviderOptions, type GoogleImageModelOptions as GoogleGenerativeAIImageProviderOptions, type GoogleProvider as GoogleGenerativeAIProvider, type GoogleProviderMetadata as GoogleGenerativeAIProviderMetadata, type GoogleLanguageModelOptions as GoogleGenerativeAIProviderOptions, type GoogleProviderSettings as GoogleGenerativeAIProviderSettings, type GoogleVideoModelId as GoogleGenerativeAIVideoModelId, type GoogleVideoModelOptions as GoogleGenerativeAIVideoProviderOptions, type GoogleImageModelOptions, type GoogleInteractionsAgentName, type GoogleInteractionsModelId, type GoogleInteractionsProviderMetadata, type GoogleLanguageModelInteractionsOptions, type GoogleLanguageModelOptions, type GoogleProvider, type GoogleProviderMetadata, type GoogleProviderSettings, type GoogleSpeechModelId, type GoogleSpeechModelOptions, type GoogleVideoModelId, type GoogleVideoModelOptions, VERSION, createGoogle, createGoogle as createGoogleGenerativeAI, google };
+export { GoogleRealtimeModel as Experimental_GoogleRealtimeModel, type GoogleRealtimeModelConfig as Experimental_GoogleRealtimeModelConfig, type GoogleEmbeddingModelOptions, type GoogleErrorData, type GoogleFilesUploadOptions, type GoogleEmbeddingModelOptions as GoogleGenerativeAIEmbeddingProviderOptions, type GoogleImageModelOptions as GoogleGenerativeAIImageProviderOptions, type GoogleProvider as GoogleGenerativeAIProvider, type GoogleProviderMetadata as GoogleGenerativeAIProviderMetadata, type GoogleLanguageModelOptions as GoogleGenerativeAIProviderOptions, type GoogleProviderSettings as GoogleGenerativeAIProviderSettings, type GoogleVideoModelId as GoogleGenerativeAIVideoModelId, type GoogleVideoModelOptions as GoogleGenerativeAIVideoProviderOptions, type GoogleImageModelOptions, type GoogleInteractionsAgentName, type GoogleInteractionsModelId, type GoogleInteractionsProviderMetadata, type GoogleLanguageModelInteractionsOptions, type GoogleLanguageModelOptions, type GoogleProvider, type GoogleProviderMetadata, type GoogleProviderSettings, type GoogleSpeechModelId, type GoogleSpeechModelOptions, type GoogleVideoModelId, type GoogleVideoModelOptions, VERSION, createGoogle, createGoogle as createGoogleGenerativeAI, google };

package/dist/index.js

@@ -7,7 +7,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "4.0.0-canary.78" : "0.0.0-test";
+var VERSION = true ? "4.0.0-canary.80" : "0.0.0-test";
 
 // src/google-embedding-model.ts
 import {
@@ -5646,14 +5646,14 @@ async function cancelGoogleInteraction({
   baseURL,
   interactionId,
   headers,
-  fetch = getOriginalFetch()
+  fetch: fetch2 = getOriginalFetch()
 }) {
   if (interactionId == null || interactionId.length === 0) {
     return;
   }
   const url = `${baseURL}/interactions/${encodeURIComponent(interactionId)}/cancel`;
   try {
-    const response = await fetch(url, {
+    const response = await fetch2(url, {
       method: "POST",
       headers: withUserAgentSuffix(
         combineHeaders7({ "Content-Type": "application/json" }, headers),
@@ -5681,7 +5681,7 @@ async function pollGoogleInteractionUntilTerminal({
   baseURL,
   interactionId,
   headers,
-  fetch,
+  fetch: fetch2,
   abortSignal,
   initialDelayMs = DEFAULT_INITIAL_DELAY_MS,
   maxDelayMs = DEFAULT_MAX_DELAY_MS,
@@ -5695,7 +5695,7 @@ async function pollGoogleInteractionUntilTerminal({
   const startedAt = Date.now();
   let nextDelayMs = initialDelayMs;
   const url = `${baseURL}/interactions/${encodeURIComponent(interactionId)}`;
-  const cancelOnServer = () => cancelGoogleInteraction({ baseURL, interactionId, headers, fetch });
+  const cancelOnServer = () => cancelGoogleInteraction({ baseURL, interactionId, headers, fetch: fetch2 });
   try {
     while (true) {
       if (abortSignal == null ? void 0 : abortSignal.aborted) {
@@ -5720,7 +5720,7 @@ async function pollGoogleInteractionUntilTerminal({
           googleInteractionsResponseSchema
         ),
         abortSignal,
-        fetch
+        fetch: fetch2
       });
       if (isTerminalStatus(response.status)) {
         return { response, rawResponse, responseHeaders };
@@ -5896,7 +5896,7 @@ function streamGoogleInteractionEvents({
   baseURL,
   interactionId,
   headers,
-  fetch,
+  fetch: fetch2,
   abortSignal,
   maxRetries = DEFAULT_MAX_RETRIES,
   retryDelayMs = DEFAULT_RETRY_DELAY_MS
@@ -5944,7 +5944,7 @@ function streamGoogleInteractionEvents({
         googleInteractionsEventSchema
       ),
       abortSignal: effectiveSignal,
-      fetch
+      fetch: fetch2
     });
     return stream.getReader();
   }
@@ -6037,7 +6037,7 @@ function streamGoogleInteractionEvents({
             baseURL,
             interactionId,
             headers,
-            fetch
+            fetch: fetch2
           });
         }
       }
@@ -6690,6 +6690,388 @@ function pruneUndefined(obj) {
   return result;
 }
 
+// src/realtime/google-realtime-event-mapper.ts
+import { safeParseJSON } from "@ai-sdk/provider-utils";
+var GoogleRealtimeEventMapper = class {
+  constructor() {
+    this.turnCounter = 0;
+    this.hasAudio = false;
+    this.hasText = false;
+    this.hasTranscript = false;
+    this.turnClosed = false;
+    this.inputAudioRate = 16e3;
+  }
+  get responseId() {
+    return `google-resp-${this.turnCounter}`;
+  }
+  get itemId() {
+    return `google-item-${this.turnCounter}`;
+  }
+  /**
+   * Rolls over to the next turn lazily, only once new model content actually
+   * arrives. `turnComplete` merely marks the current turn closed; the counter
+   * is not advanced until the next response begins. This keeps a transcript
+   * that arrives shortly after `turnComplete` attached to the turn it belongs
+   * to, since Google delivers transcription independently with no guaranteed
+   * ordering relative to `turnComplete`.
+   */
+  beginTurnIfClosed() {
+    if (!this.turnClosed) return;
+    this.turnCounter++;
+    this.hasAudio = false;
+    this.hasText = false;
+    this.hasTranscript = false;
+    this.turnClosed = false;
+  }
+  parseServerEvent(raw) {
+    var _a, _b;
+    const data = raw;
+    if (data.setupComplete != null) {
+      return { type: "session-created", raw };
+    }
+    if (data.toolCall != null) {
+      this.beginTurnIfClosed();
+      const functionCalls = (_a = data.toolCall.functionCalls) != null ? _a : [];
+      return functionCalls.flatMap((functionCall) => {
+        var _a2;
+        const args = JSON.stringify((_a2 = functionCall.args) != null ? _a2 : {});
+        return [
+          {
+            type: "function-call-arguments-delta",
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            delta: args,
+            raw
+          },
+          {
+            type: "function-call-arguments-done",
+            responseId: this.responseId,
+            itemId: this.itemId,
+            callId: functionCall.id,
+            name: functionCall.name,
+            arguments: args,
+            raw
+          }
+        ];
+      });
+    }
+    if (data.toolCallCancellation != null) {
+      return {
+        type: "custom",
+        rawType: "toolCallCancellation",
+        raw
+      };
+    }
+    if (data.serverContent != null) {
+      return this.parseServerContent(data.serverContent, raw);
+    }
+    if (((_b = data.inputTranscription) == null ? void 0 : _b.text) != null) {
+      return {
+        type: "input-transcription-completed",
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: data.inputTranscription.text,
+        raw
+      };
+    }
+    return { type: "custom", rawType: String(Object.keys(data)[0]), raw };
+  }
+  parseServerContent(serverContent, raw) {
+    var _a, _b, _c, _d;
+    const events = [];
+    if (serverContent.interrupted) {
+      events.push({
+        type: "speech-started",
+        raw
+      });
+    }
+    if ((_a = serverContent.modelTurn) == null ? void 0 : _a.parts) {
+      this.beginTurnIfClosed();
+      for (const part of serverContent.modelTurn.parts) {
+        if ((_b = part.inlineData) == null ? void 0 : _b.data) {
+          this.hasAudio = true;
+          events.push({
+            type: "audio-delta",
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.inlineData.data,
+            raw
+          });
+        }
+        if (part.text) {
+          this.hasText = true;
+          events.push({
+            type: "text-delta",
+            responseId: this.responseId,
+            itemId: this.itemId,
+            delta: part.text,
+            raw
+          });
+        }
+      }
+    }
+    if ((_c = serverContent.outputTranscription) == null ? void 0 : _c.text) {
+      this.hasTranscript = true;
+      events.push({
+        type: "audio-transcript-delta",
+        responseId: this.responseId,
+        itemId: this.itemId,
+        delta: serverContent.outputTranscription.text,
+        raw
+      });
+    }
+    if ((_d = serverContent.inputTranscription) == null ? void 0 : _d.text) {
+      events.push({
+        type: "input-transcription-completed",
+        itemId: `google-input-${this.turnCounter}`,
+        transcript: serverContent.inputTranscription.text,
+        raw
+      });
+    }
+    if (serverContent.turnComplete) {
+      if (this.hasAudio) {
+        events.push({
+          type: "audio-done",
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw
+        });
+      }
+      if (this.hasText) {
+        events.push({
+          type: "text-done",
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw
+        });
+      }
+      if (this.hasTranscript) {
+        events.push({
+          type: "audio-transcript-done",
+          responseId: this.responseId,
+          itemId: this.itemId,
+          raw
+        });
+      }
+      events.push({
+        type: "response-done",
+        responseId: this.responseId,
+        status: "completed",
+        raw
+      });
+      this.turnClosed = true;
+    }
+    if (events.length === 0) {
+      return { type: "custom", rawType: "serverContent", raw };
+    }
+    return events.length === 1 ? events[0] : events;
+  }
+  serializeClientEvent(event, modelId) {
+    var _a;
+    switch (event.type) {
+      case "session-update":
+        if (((_a = event.config.inputAudioFormat) == null ? void 0 : _a.rate) != null) {
+          this.inputAudioRate = event.config.inputAudioFormat.rate;
+        }
+        return {
+          setup: buildGoogleSessionConfig(event.config, modelId)
+        };
+      case "input-audio-append":
+        return {
+          realtimeInput: {
+            audio: {
+              data: event.audio,
+              mimeType: `audio/pcm;rate=${this.inputAudioRate}`
+            }
+          }
+        };
+      case "input-audio-commit":
+      case "input-audio-clear":
+      case "response-create":
+      case "response-cancel":
+      case "conversation-item-truncate":
+        return null;
+      case "conversation-item-create": {
+        const item = event.item;
+        switch (item.type) {
+          case "text-message":
+            return {
+              realtimeInput: {
+                text: item.text
+              }
+            };
+          case "function-call-output":
+            return serializeFunctionCallOutput(item);
+          case "audio-message":
+            return null;
+        }
+        break;
+      }
+    }
+    return null;
+  }
+};
+async function serializeFunctionCallOutput(item) {
+  const parseResult = await safeParseJSON({ text: item.output });
+  const response = parseResult.success ? parseResult.value : {};
+  return {
+    toolResponse: {
+      functionResponses: [
+        {
+          id: item.callId,
+          name: item.name,
+          response
+        }
+      ]
+    }
+  };
+}
+function buildGoogleSessionConfig(config, modelId) {
+  const setup = {
+    model: getModelPath(modelId)
+  };
+  const generationConfig = {};
+  if ((config == null ? void 0 : config.outputModalities) != null) {
+    generationConfig.responseModalities = config.outputModalities.map(
+      (m) => m.toUpperCase()
+    );
+  } else {
+    generationConfig.responseModalities = ["AUDIO"];
+  }
+  if ((config == null ? void 0 : config.voice) != null) {
+    generationConfig.speechConfig = {
+      voiceConfig: {
+        prebuiltVoiceConfig: {
+          voiceName: config.voice
+        }
+      }
+    };
+  }
+  setup.generationConfig = generationConfig;
+  if ((config == null ? void 0 : config.instructions) != null) {
+    setup.systemInstruction = {
+      parts: [{ text: config.instructions }]
+    };
+  }
+  if ((config == null ? void 0 : config.tools) != null && config.tools.length > 0) {
+    setup.tools = [
+      {
+        functionDeclarations: config.tools.map((tool) => ({
+          name: tool.name,
+          description: tool.description,
+          parameters: convertJSONSchemaToOpenAPISchema(tool.parameters)
+        }))
+      }
+    ];
+  }
+  if ((config == null ? void 0 : config.inputAudioTranscription) != null) {
+    setup.inputAudioTranscription = {};
+  }
+  if ((config == null ? void 0 : config.outputAudioTranscription) != null) {
+    setup.outputAudioTranscription = {};
+  }
+  if ((config == null ? void 0 : config.providerOptions) != null) {
+    Object.assign(setup, config.providerOptions);
+  }
+  return setup;
+}
+
+// src/realtime/google-realtime-model.ts
+var realtimeWebSocketPath = "google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+function getRealtimeBaseURL(baseURL) {
+  const url = new URL(baseURL);
+  const pathSegments = url.pathname.split("/");
+  const version = pathSegments.at(-1);
+  if (version === "v1beta" || version === "v1alpha") {
+    pathSegments.pop();
+    url.pathname = pathSegments.join("/") || "/";
+  }
+  return url;
+}
+function getAuthTokensURL(baseURL) {
+  const url = getRealtimeBaseURL(baseURL);
+  url.pathname = `${url.pathname.replace(/\/$/, "")}/v1alpha/auth_tokens`;
+  return url.toString();
+}
+function getWebSocketURL(baseURL) {
+  const url = getRealtimeBaseURL(baseURL);
+  url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+  url.pathname = `${url.pathname.replace(/\/$/, "")}/ws/${realtimeWebSocketPath}`;
+  return url.toString();
+}
+var GoogleRealtimeModel = class {
+  constructor(modelId, config) {
+    this.specificationVersion = "v4";
+    this.mapper = new GoogleRealtimeEventMapper();
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+  async doCreateClientSecret(options) {
+    var _a, _b;
+    const fetchFn = (_a = this.config.fetch) != null ? _a : fetch;
+    const headers = this.config.headers();
+    const apiKey = headers["x-goog-api-key"];
+    if (!apiKey) {
+      throw new Error(
+        "Google Generative AI API key is required for realtime token creation."
+      );
+    }
+    const now = Date.now();
+    const openWindowMs = ((_b = options.expiresAfterSeconds) != null ? _b : 60) * 1e3;
+    const newSessionExpireTime = new Date(now + openWindowMs).toISOString();
+    const expireTime = new Date(
+      now + openWindowMs + 30 * 60 * 1e3
+    ).toISOString();
+    const setupPayload = buildGoogleSessionConfig(
+      options.sessionConfig,
+      this.modelId
+    );
+    const response = await fetchFn(
+      `${getAuthTokensURL(this.config.baseURL)}?key=${encodeURIComponent(apiKey)}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          // `uses: 0` means no limit is applied to how many times the token can
+          // start a session (per the AuthToken spec). An unset value would
+          // default to 1, which breaks WebSocket reconnects within the session.
+          uses: 0,
+          expireTime,
+          newSessionExpireTime,
+          bidiGenerateContentSetup: setupPayload
+        })
+      }
+    );
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `Google realtime auth token request failed: ${response.status} ${text}`
+      );
+    }
+    const data = await response.json();
+    return {
+      token: data.name,
+      url: getWebSocketURL(this.config.baseURL),
+      expiresAt: data.expireTime ? Math.floor(new Date(data.expireTime).getTime() / 1e3) : void 0
+    };
+  }
+  getWebSocketConfig(options) {
+    return {
+      url: `${options.url}?access_token=${encodeURIComponent(options.token)}`
+    };
+  }
+  parseServerEvent(raw) {
+    return this.mapper.parseServerEvent(raw);
+  }
+  serializeClientEvent(event) {
+    return this.mapper.serializeClientEvent(event, this.modelId);
+  }
+  buildSessionConfig(config) {
+    return buildGoogleSessionConfig(config, this.modelId);
+  }
+};
+
 // src/google-provider.ts
 function createGoogle(options = {}) {
   var _a, _b;
@@ -6756,12 +7138,35 @@ function createGoogle(options = {}) {
       generateId: (_a2 = options.generateId) != null ? _a2 : generateId2
     });
   };
+  const createRealtimeModel = (modelId) => new GoogleRealtimeModel(modelId, {
+    provider: `${providerName}.realtime`,
+    baseURL,
+    headers: getHeaders,
+    fetch: options.fetch
+  });
   const createSpeechModel = (modelId) => new GoogleSpeechModel(modelId, {
     provider: `${providerName}.speech`,
     baseURL,
     headers: getHeaders,
     fetch: options.fetch
   });
+  const experimentalRealtimeFactory = Object.assign(
+    (modelId) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions) => {
+        const model = createRealtimeModel(tokenOptions.model);
+        const secret = await model.doCreateClientSecret({
+          sessionConfig: tokenOptions.sessionConfig,
+          expiresAfterSeconds: tokenOptions.expiresAfterSeconds
+        });
+        return {
+          token: secret.token,
+          url: secret.url,
+          expiresAt: secret.expiresAt
+        };
+      }
+    }
+  );
   const createInteractionsModel = (modelIdOrAgent) => {
     var _a2;
     return new GoogleInteractionsLanguageModel(
@@ -6795,6 +7200,7 @@ function createGoogle(options = {}) {
   provider.imageModel = createImageModel;
   provider.video = createVideoModel;
   provider.videoModel = createVideoModel;
+  provider.experimental_realtime = experimentalRealtimeFactory;
   provider.files = createFiles;
   provider.speech = createSpeechModel;
   provider.speechModel = createSpeechModel;
@@ -6804,6 +7210,7 @@ function createGoogle(options = {}) {
 }
 var google = createGoogle();
 export {
+  GoogleRealtimeModel as Experimental_GoogleRealtimeModel,
   VERSION,
   createGoogle,
   createGoogle as createGoogleGenerativeAI,