@ai-sdk/xai 4.0.0-canary.70 → 4.0.0-canary.71

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @ai-sdk/xai
 
+## 4.0.0-canary.71
+
+### Patch Changes
+
+- ce769dd: feat(provider): add experimental Realtime API support for voice conversations
+
+  Adds first-class support for realtime (speech-to-speech) APIs:
+
+  - `Experimental_RealtimeModelV4` spec in `@ai-sdk/provider` with normalized event types and factory
+  - OpenAI, Google, and xAI realtime provider implementations
+  - `openai.experimental_realtime()` / `google.experimental_realtime()` / `xai.experimental_realtime()` work in both server and browser
+  - `.getToken()` static method on each provider for server-side ephemeral token creation
+  - `experimental_getRealtimeToolDefinitions` helper for provider session tool definitions
+  - `experimental_useRealtime` hook in `@ai-sdk/react` returning `UIMessage[]` (aligned with `useChat`), with `onToolCall` and `addToolOutput` for client-driven tool execution
+  - `inputAudioTranscription` session config for showing transcribed user audio messages when supported by the provider
+
+- Updated dependencies [ce769dd]
+  - @ai-sdk/provider@4.0.0-canary.18
+  - @ai-sdk/openai-compatible@3.0.0-canary.54
+  - @ai-sdk/provider-utils@5.0.0-canary.46
+
 ## 4.0.0-canary.70
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import { z } from 'zod/v4';
 import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
 import { InferSchema, FetchFunction } from '@ai-sdk/provider-utils';
-import { ProviderV4, LanguageModelV4, ImageModelV4, Experimental_VideoModelV4, FilesV4 } from '@ai-sdk/provider';
+import { ProviderV4, LanguageModelV4, ImageModelV4, Experimental_VideoModelV4, Experimental_RealtimeFactoryV4, FilesV4, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig } from '@ai-sdk/provider';
 
 type XaiChatModelId = 'grok-4.20-non-reasoning' | 'grok-4.20-reasoning' | 'grok-4.3' | 'grok-latest' | (string & {});
 declare const xaiLanguageModelChatOptions: z.ZodObject<{
@@ -419,6 +419,7 @@ interface XaiProvider extends ProviderV4 {
      * Creates an Xai video model for video generation.
      */
     videoModel(modelId: XaiVideoModelId): Experimental_VideoModelV4;
+    experimental_realtime: Experimental_RealtimeFactoryV4;
     /**
      * Returns the xAI files interface for uploading files.
      */
@@ -454,6 +455,31 @@ interface XaiProviderSettings {
 declare function createXai(options?: XaiProviderSettings): XaiProvider;
 declare const xai: XaiProvider;
 
+type XaiRealtimeModelConfig = {
+    provider: string;
+    baseURL: string;
+    headers: () => Record<string, string | undefined>;
+    fetch?: FetchFunction;
+};
+declare class XaiRealtimeModel implements Experimental_RealtimeModelV4 {
+    readonly specificationVersion: "v4";
+    readonly provider: string;
+    readonly modelId: string;
+    private readonly config;
+    constructor(modelId: string, config: XaiRealtimeModelConfig);
+    doCreateClientSecret(options: Experimental_RealtimeModelV4ClientSecretOptions): Promise<Experimental_RealtimeModelV4ClientSecretResult>;
+    getWebSocketConfig(options: {
+        token: string;
+        url: string;
+    }): {
+        url: string;
+        protocols?: string[];
+    };
+    parseServerEvent(raw: unknown): Experimental_RealtimeModelV4ServerEvent;
+    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): unknown;
+    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
+}
+
 declare const VERSION: string;
 
-export { VERSION, type XaiErrorData, type XaiFilesOptions, type XaiImageModelOptions, type XaiImageModelOptions as XaiImageProviderOptions, type XaiLanguageModelChatOptions, type XaiLanguageModelResponsesOptions, type XaiProvider, type XaiLanguageModelChatOptions as XaiProviderOptions, type XaiProviderSettings, type XaiLanguageModelResponsesOptions as XaiResponsesProviderOptions, type XaiVideoModelId, type XaiVideoModelOptions, type XaiVideoModelOptions as XaiVideoProviderOptions, codeExecution, createXai, mcpServer, viewImage, viewXVideo, webSearch, xSearch, xai, xaiTools };
+export { XaiRealtimeModel as Experimental_XaiRealtimeModel, type XaiRealtimeModelConfig as Experimental_XaiRealtimeModelConfig, VERSION, type XaiErrorData, type XaiFilesOptions, type XaiImageModelOptions, type XaiImageModelOptions as XaiImageProviderOptions, type XaiLanguageModelChatOptions, type XaiLanguageModelResponsesOptions, type XaiProvider, type XaiLanguageModelChatOptions as XaiProviderOptions, type XaiProviderSettings, type XaiLanguageModelResponsesOptions as XaiResponsesProviderOptions, type XaiVideoModelId, type XaiVideoModelOptions, type XaiVideoModelOptions as XaiVideoProviderOptions, codeExecution, createXai, mcpServer, viewImage, viewXVideo, webSearch, xSearch, xai, xaiTools };

package/dist/index.js

@@ -2969,6 +2969,375 @@ var XaiResponsesLanguageModel = class _XaiResponsesLanguageModel {
   }
 };
 
+// src/realtime/xai-realtime-event-mapper.ts
+function parseXaiRealtimeServerEvent(raw) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
+  const event = raw;
+  const type = event.type;
+  switch (type) {
+    case "session.created":
+      return {
+        type: "session-created",
+        sessionId: (_a = event.session) == null ? void 0 : _a.id,
+        raw
+      };
+    case "session.updated":
+      return { type: "session-updated", raw };
+    case "conversation.created":
+      return { type: "custom", rawType: type, raw };
+    case "input_audio_buffer.speech_started":
+      return {
+        type: "speech-started",
+        itemId: event.item_id,
+        raw
+      };
+    case "input_audio_buffer.speech_stopped":
+      return {
+        type: "speech-stopped",
+        itemId: event.item_id,
+        raw
+      };
+    case "input_audio_buffer.committed":
+      return {
+        type: "audio-committed",
+        itemId: event.item_id,
+        previousItemId: event.previous_item_id,
+        raw
+      };
+    case "conversation.item.added":
+      return {
+        type: "conversation-item-added",
+        itemId: (_c = (_b = event.item) == null ? void 0 : _b.id) != null ? _c : event.item_id,
+        item: event.item,
+        raw
+      };
+    case "conversation.item.input_audio_transcription.completed":
+      return {
+        type: "input-transcription-completed",
+        itemId: event.item_id,
+        transcript: (_d = event.transcript) != null ? _d : "",
+        raw
+      };
+    case "response.created":
+      return {
+        type: "response-created",
+        responseId: (_f = (_e = event.response) == null ? void 0 : _e.id) != null ? _f : event.response_id,
+        raw
+      };
+    case "response.done":
+      return {
+        type: "response-done",
+        responseId: (_h = (_g = event.response) == null ? void 0 : _g.id) != null ? _h : event.response_id,
+        status: (_j = (_i = event.response) == null ? void 0 : _i.status) != null ? _j : "completed",
+        raw
+      };
+    case "response.output_item.added":
+      return {
+        type: "output-item-added",
+        responseId: event.response_id,
+        itemId: (_l = (_k = event.item) == null ? void 0 : _k.id) != null ? _l : event.item_id,
+        raw
+      };
+    case "response.output_item.done":
+      return {
+        type: "output-item-done",
+        responseId: event.response_id,
+        itemId: (_n = (_m = event.item) == null ? void 0 : _m.id) != null ? _n : event.item_id,
+        raw
+      };
+    case "response.content_part.added":
+      return {
+        type: "content-part-added",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    case "response.content_part.done":
+      return {
+        type: "content-part-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    case "response.output_audio.delta":
+      return {
+        type: "audio-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.output_audio.done":
+      return {
+        type: "audio-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    case "response.output_audio_transcript.delta":
+      return {
+        type: "audio-transcript-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.output_audio_transcript.done":
+      return {
+        type: "audio-transcript-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        transcript: event.transcript,
+        raw
+      };
+    case "response.text.delta":
+      return {
+        type: "text-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.text.done":
+      return {
+        type: "text-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        text: event.text,
+        raw
+      };
+    case "response.function_call_arguments.delta":
+      return {
+        type: "function-call-arguments-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.function_call_arguments.done":
+      return {
+        type: "function-call-arguments-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        name: event.name,
+        arguments: event.arguments,
+        raw
+      };
+    case "mcp_list_tools.in_progress":
+    case "mcp_list_tools.completed":
+    case "mcp_list_tools.failed":
+    case "response.mcp_call_arguments.delta":
+    case "response.mcp_call_arguments.done":
+    case "response.mcp_call.in_progress":
+    case "response.mcp_call.completed":
+    case "response.mcp_call.failed":
+      return { type: "custom", rawType: type, raw };
+    case "error":
+      return {
+        type: "error",
+        message: (_q = (_p = (_o = event.error) == null ? void 0 : _o.message) != null ? _p : event.message) != null ? _q : "Unknown error",
+        code: (_s = (_r = event.error) == null ? void 0 : _r.code) != null ? _s : event.code,
+        raw
+      };
+    default:
+      return { type: "custom", rawType: type, raw };
+  }
+}
+function serializeXaiRealtimeClientEvent(event) {
+  switch (event.type) {
+    case "session-update":
+      return {
+        type: "session.update",
+        session: buildXaiSessionConfig(event.config)
+      };
+    case "input-audio-append":
+      return {
+        type: "input_audio_buffer.append",
+        audio: event.audio
+      };
+    case "input-audio-commit":
+      return { type: "input_audio_buffer.commit" };
+    case "input-audio-clear":
+      return { type: "input_audio_buffer.clear" };
+    case "conversation-item-create": {
+      const item = event.item;
+      switch (item.type) {
+        case "text-message":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "message",
+              role: item.role,
+              content: [{ type: "input_text", text: item.text }]
+            }
+          };
+        case "audio-message":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "message",
+              role: item.role,
+              content: [{ type: "input_audio", audio: item.audio }]
+            }
+          };
+        case "function-call-output":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "function_call_output",
+              call_id: item.callId,
+              output: item.output
+            }
+          };
+      }
+      break;
+    }
+    case "conversation-item-truncate":
+      return void 0;
+    case "response-create":
+      return {
+        type: "response.create",
+        ...event.options != null ? {
+          response: {
+            ...event.options.modalities != null ? { modalities: event.options.modalities } : {},
+            ...event.options.instructions != null ? { instructions: event.options.instructions } : {}
+          }
+        } : {}
+      };
+    case "response-cancel":
+      return { type: "response.cancel" };
+  }
+}
+function buildXaiSessionConfig(config) {
+  var _a;
+  const session = {};
+  if (config.instructions != null) {
+    session.instructions = config.instructions;
+  }
+  if (config.voice != null) {
+    session.voice = config.voice;
+  }
+  const audio = {};
+  if (config.inputAudioFormat != null) {
+    audio.input = {
+      format: {
+        type: config.inputAudioFormat.type,
+        ...config.inputAudioFormat.rate != null ? { rate: config.inputAudioFormat.rate } : {}
+      }
+    };
+  }
+  if (config.outputAudioFormat != null) {
+    audio.output = {
+      format: {
+        type: config.outputAudioFormat.type,
+        ...config.outputAudioFormat.rate != null ? { rate: config.outputAudioFormat.rate } : {}
+      }
+    };
+  }
+  if (Object.keys(audio).length > 0) {
+    session.audio = audio;
+  }
+  if (config.turnDetection != null) {
+    if (config.turnDetection.type === "disabled") {
+      session.turn_detection = null;
+    } else {
+      const td = {
+        type: "server_vad"
+      };
+      if (config.turnDetection.threshold != null) {
+        td.threshold = config.turnDetection.threshold;
+      }
+      if (config.turnDetection.silenceDurationMs != null) {
+        td.silence_duration_ms = config.turnDetection.silenceDurationMs;
+      }
+      if (config.turnDetection.prefixPaddingMs != null) {
+        td.prefix_padding_ms = config.turnDetection.prefixPaddingMs;
+      }
+      session.turn_detection = td;
+    }
+  }
+  if (config.tools != null && config.tools.length > 0) {
+    session.tools = config.tools.map((tool) => ({
+      type: tool.type,
+      name: tool.name,
+      description: tool.description,
+      parameters: tool.parameters
+    }));
+  }
+  if (config.providerOptions != null) {
+    const xaiOptions = config.providerOptions;
+    if (Array.isArray(xaiOptions.tools)) {
+      const existingTools = (_a = session.tools) != null ? _a : [];
+      session.tools = [...existingTools, ...xaiOptions.tools];
+    }
+    for (const [key, value] of Object.entries(xaiOptions)) {
+      if (key !== "tools") {
+        session[key] = value;
+      }
+    }
+  }
+  return session;
+}
+
+// src/realtime/xai-realtime-model.ts
+var XaiRealtimeModel = class {
+  constructor(modelId, config) {
+    this.specificationVersion = "v4";
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+  async doCreateClientSecret(options) {
+    var _a;
+    const fetchFn = (_a = this.config.fetch) != null ? _a : fetch;
+    const url = `${this.config.baseURL}/realtime/client_secrets`;
+    const body = {};
+    if (options.expiresAfterSeconds != null) {
+      body.expires_after = { seconds: options.expiresAfterSeconds };
+    }
+    const response = await fetchFn(url, {
+      method: "POST",
+      headers: {
+        ...this.config.headers(),
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(body)
+    });
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `xAI realtime client secret request failed: ${response.status} ${text}`
+      );
+    }
+    const data = await response.json();
+    return {
+      token: data.value,
+      // xAI selects the voice model from the `model` query parameter on the
+      // WebSocket URL. Without it the model choice is silently ignored and the
+      // server falls back to its default voice model.
+      url: `wss://${new URL(this.config.baseURL).host}/v1/realtime?model=${encodeURIComponent(this.modelId)}`,
+      expiresAt: data.expires_at
+    };
+  }
+  getWebSocketConfig(options) {
+    return {
+      url: options.url,
+      protocols: [`xai-client-secret.${options.token}`]
+    };
+  }
+  parseServerEvent(raw) {
+    return parseXaiRealtimeServerEvent(raw);
+  }
+  serializeClientEvent(event) {
+    return serializeXaiRealtimeClientEvent(event);
+  }
+  buildSessionConfig(config) {
+    return buildXaiSessionConfig(config);
+  }
+};
+
 // src/tool/code-execution.ts
 import { createProviderExecutedToolFactory as createProviderExecutedToolFactory5 } from "@ai-sdk/provider-utils";
 import { z as z12 } from "zod/v4";
@@ -3024,7 +3393,7 @@ var xaiTools = {
 };
 
 // src/version.ts
-var VERSION = true ? "4.0.0-canary.70" : "0.0.0-test";
+var VERSION = true ? "4.0.0-canary.71" : "0.0.0-test";
 
 // src/files/xai-files.ts
 import {
@@ -3527,6 +3896,31 @@ function createXai(options = {}) {
       fetch: options.fetch
     });
   };
+  const createRealtimeModel = (modelId) => {
+    return new XaiRealtimeModel(modelId, {
+      provider: "xai.realtime",
+      baseURL: baseURL != null ? baseURL : "https://api.x.ai/v1",
+      headers: getHeaders,
+      fetch: options.fetch
+    });
+  };
+  const experimentalRealtimeFactory = Object.assign(
+    (modelId) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions) => {
+        const model = createRealtimeModel(tokenOptions.model);
+        const secret = await model.doCreateClientSecret({
+          sessionConfig: tokenOptions.sessionConfig,
+          expiresAfterSeconds: tokenOptions.expiresAfterSeconds
+        });
+        return {
+          token: secret.token,
+          url: secret.url,
+          expiresAt: secret.expiresAt
+        };
+      }
+    }
+  );
   const createFiles = () => new XaiFiles({
     provider: "xai.files",
     baseURL,
@@ -3546,12 +3940,14 @@ function createXai(options = {}) {
   provider.image = createImageModel;
   provider.videoModel = createVideoModel;
   provider.video = createVideoModel;
+  provider.experimental_realtime = experimentalRealtimeFactory;
   provider.files = createFiles;
   provider.tools = xaiTools;
   return provider;
 }
 var xai = createXai();
 export {
+  XaiRealtimeModel as Experimental_XaiRealtimeModel,
   VERSION,
   codeExecution,
   createXai,