@ai-sdk/xai 4.0.0-beta.7 → 4.0.0-beta.75

package/src/realtime/xai-realtime-event-mapper.ts

@@ -0,0 +1,399 @@
+import type {
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+
+type XaiRealtimeWireEvent = {
+  type: string;
+  session?: { id?: string };
+  item?: { id?: string } & Record<string, unknown>;
+  response?: { id?: string; status?: string };
+  error?: { message?: string; code?: string };
+  item_id: string;
+  previous_item_id?: string;
+  response_id: string;
+  transcript?: string;
+  delta: string;
+  text?: string;
+  call_id: string;
+  name: string;
+  arguments: string;
+  message?: string;
+  code?: string;
+};
+
+export function parseXaiRealtimeServerEvent(
+  raw: unknown,
+): RealtimeModelV4ServerEvent {
+  const event = raw as XaiRealtimeWireEvent;
+  const type = event.type;
+
+  switch (type) {
+    case 'session.created':
+      return {
+        type: 'session-created',
+        sessionId: event.session?.id,
+        raw,
+      };
+
+    case 'session.updated':
+      return { type: 'session-updated', raw };
+
+    case 'conversation.created':
+      return { type: 'custom', rawType: type, raw };
+
+    case 'input_audio_buffer.speech_started':
+      return {
+        type: 'speech-started',
+        itemId: event.item_id,
+        raw,
+      };
+
+    case 'input_audio_buffer.speech_stopped':
+      return {
+        type: 'speech-stopped',
+        itemId: event.item_id,
+        raw,
+      };
+
+    case 'input_audio_buffer.committed':
+      return {
+        type: 'audio-committed',
+        itemId: event.item_id,
+        previousItemId: event.previous_item_id,
+        raw,
+      };
+
+    case 'conversation.item.added':
+      return {
+        type: 'conversation-item-added',
+        itemId: event.item?.id ?? event.item_id,
+        item: event.item,
+        raw,
+      };
+
+    case 'conversation.item.input_audio_transcription.completed':
+      return {
+        type: 'input-transcription-completed',
+        itemId: event.item_id,
+        transcript: event.transcript ?? '',
+        raw,
+      };
+
+    case 'response.created':
+      return {
+        type: 'response-created',
+        responseId: event.response?.id ?? event.response_id,
+        raw,
+      };
+
+    case 'response.done':
+      return {
+        type: 'response-done',
+        responseId: event.response?.id ?? event.response_id,
+        status: event.response?.status ?? 'completed',
+        raw,
+      };
+
+    case 'response.output_item.added':
+      return {
+        type: 'output-item-added',
+        responseId: event.response_id,
+        itemId: event.item?.id ?? event.item_id,
+        raw,
+      };
+
+    case 'response.output_item.done':
+      return {
+        type: 'output-item-done',
+        responseId: event.response_id,
+        itemId: event.item?.id ?? event.item_id,
+        raw,
+      };
+
+    case 'response.content_part.added':
+      return {
+        type: 'content-part-added',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw,
+      };
+
+    case 'response.content_part.done':
+      return {
+        type: 'content-part-done',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw,
+      };
+
+    case 'response.output_audio.delta':
+      return {
+        type: 'audio-delta',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw,
+      };
+
+    case 'response.output_audio.done':
+      return {
+        type: 'audio-done',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw,
+      };
+
+    case 'response.output_audio_transcript.delta':
+      return {
+        type: 'audio-transcript-delta',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw,
+      };
+
+    case 'response.output_audio_transcript.done':
+      return {
+        type: 'audio-transcript-done',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        transcript: event.transcript,
+        raw,
+      };
+
+    case 'response.text.delta':
+      return {
+        type: 'text-delta',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw,
+      };
+
+    case 'response.text.done':
+      return {
+        type: 'text-done',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        text: event.text,
+        raw,
+      };
+
+    case 'response.function_call_arguments.delta':
+      return {
+        type: 'function-call-arguments-delta',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        delta: event.delta,
+        raw,
+      };
+
+    case 'response.function_call_arguments.done':
+      return {
+        type: 'function-call-arguments-done',
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        name: event.name,
+        arguments: event.arguments,
+        raw,
+      };
+
+    case 'mcp_list_tools.in_progress':
+    case 'mcp_list_tools.completed':
+    case 'mcp_list_tools.failed':
+    case 'response.mcp_call_arguments.delta':
+    case 'response.mcp_call_arguments.done':
+    case 'response.mcp_call.in_progress':
+    case 'response.mcp_call.completed':
+    case 'response.mcp_call.failed':
+      return { type: 'custom', rawType: type, raw };
+
+    case 'error':
+      return {
+        type: 'error',
+        message: event.error?.message ?? event.message ?? 'Unknown error',
+        code: event.error?.code ?? event.code,
+        raw,
+      };
+
+    default:
+      return { type: 'custom', rawType: type, raw };
+  }
+}
+
+export function serializeXaiRealtimeClientEvent(
+  event: RealtimeModelV4ClientEvent,
+): unknown {
+  switch (event.type) {
+    case 'session-update':
+      return {
+        type: 'session.update',
+        session: buildXaiSessionConfig(event.config),
+      };
+
+    case 'input-audio-append':
+      return {
+        type: 'input_audio_buffer.append',
+        audio: event.audio,
+      };
+
+    case 'input-audio-commit':
+      return { type: 'input_audio_buffer.commit' };
+
+    case 'input-audio-clear':
+      return { type: 'input_audio_buffer.clear' };
+
+    case 'conversation-item-create': {
+      const item = event.item;
+      switch (item.type) {
+        case 'text-message':
+          return {
+            type: 'conversation.item.create',
+            item: {
+              type: 'message',
+              role: item.role,
+              content: [{ type: 'input_text', text: item.text }],
+            },
+          };
+        case 'audio-message':
+          return {
+            type: 'conversation.item.create',
+            item: {
+              type: 'message',
+              role: item.role,
+              content: [{ type: 'input_audio', audio: item.audio }],
+            },
+          };
+        case 'function-call-output':
+          return {
+            type: 'conversation.item.create',
+            item: {
+              type: 'function_call_output',
+              call_id: item.callId,
+              output: item.output,
+            },
+          };
+      }
+      break;
+    }
+
+    case 'conversation-item-truncate':
+      // xAI does not support `conversation.item.truncate` over WebSocket (it is
+      // silently ignored by the server). Barge-in still works because the SDK
+      // stops local playback when `speech_started` fires, so dropping the event
+      // here just avoids sending a no-op.
+      return undefined;
+
+    case 'response-create':
+      return {
+        type: 'response.create',
+        ...(event.options != null
+          ? {
+              response: {
+                ...(event.options.modalities != null
+                  ? { modalities: event.options.modalities }
+                  : {}),
+                ...(event.options.instructions != null
+                  ? { instructions: event.options.instructions }
+                  : {}),
+              },
+            }
+          : {}),
+      };
+
+    case 'response-cancel':
+      return { type: 'response.cancel' };
+  }
+}
+
+export function buildXaiSessionConfig(
+  config: RealtimeModelV4SessionConfig,
+): Record<string, unknown> {
+  const session: Record<string, unknown> = {};
+
+  if (config.instructions != null) {
+    session.instructions = config.instructions;
+  }
+
+  if (config.voice != null) {
+    session.voice = config.voice;
+  }
+
+  const audio: Record<string, unknown> = {};
+
+  if (config.inputAudioFormat != null) {
+    audio.input = {
+      format: {
+        type: config.inputAudioFormat.type,
+        ...(config.inputAudioFormat.rate != null
+          ? { rate: config.inputAudioFormat.rate }
+          : {}),
+      },
+    };
+  }
+
+  if (config.outputAudioFormat != null) {
+    audio.output = {
+      format: {
+        type: config.outputAudioFormat.type,
+        ...(config.outputAudioFormat.rate != null
+          ? { rate: config.outputAudioFormat.rate }
+          : {}),
+      },
+    };
+  }
+
+  if (Object.keys(audio).length > 0) {
+    session.audio = audio;
+  }
+
+  if (config.turnDetection != null) {
+    if (config.turnDetection.type === 'disabled') {
+      session.turn_detection = null;
+    } else {
+      const td: Record<string, unknown> = {
+        type: 'server_vad',
+      };
+      if (config.turnDetection.threshold != null) {
+        td.threshold = config.turnDetection.threshold;
+      }
+      if (config.turnDetection.silenceDurationMs != null) {
+        td.silence_duration_ms = config.turnDetection.silenceDurationMs;
+      }
+      if (config.turnDetection.prefixPaddingMs != null) {
+        td.prefix_padding_ms = config.turnDetection.prefixPaddingMs;
+      }
+      session.turn_detection = td;
+    }
+  }
+
+  if (config.tools != null && config.tools.length > 0) {
+    session.tools = config.tools.map(tool => ({
+      type: tool.type,
+      name: tool.name,
+      description: tool.description,
+      parameters: tool.parameters,
+    }));
+  }
+
+  if (config.providerOptions != null) {
+    const xaiOptions = config.providerOptions as Record<string, unknown>;
+
+    if (Array.isArray(xaiOptions.tools)) {
+      const existingTools = (session.tools as unknown[]) ?? [];
+      session.tools = [...existingTools, ...xaiOptions.tools];
+    }
+
+    for (const [key, value] of Object.entries(xaiOptions)) {
+      if (key !== 'tools') {
+        session[key] = value;
+      }
+    }
+  }
+
+  return session;
+}

package/src/realtime/xai-realtime-model-options.ts

@@ -0,0 +1,3 @@
+export type XaiRealtimeModelId = string;
+
+export type XaiRealtimeModelOptions = Record<string, never>;

package/src/realtime/xai-realtime-model.ts

@@ -0,0 +1,101 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
+  Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import type { FetchFunction } from '@ai-sdk/provider-utils';
+import {
+  buildXaiSessionConfig,
+  parseXaiRealtimeServerEvent,
+  serializeXaiRealtimeClientEvent,
+} from './xai-realtime-event-mapper';
+
+export type XaiRealtimeModelConfig = {
+  provider: string;
+  baseURL: string;
+  headers: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+};
+
+export class XaiRealtimeModel implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly provider: string;
+  readonly modelId: string;
+
+  private readonly config: XaiRealtimeModelConfig;
+
+  constructor(modelId: string, config: XaiRealtimeModelConfig) {
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+
+  async doCreateClientSecret(
+    options: RealtimeModelV4ClientSecretOptions,
+  ): Promise<RealtimeModelV4ClientSecretResult> {
+    const fetchFn = this.config.fetch ?? fetch;
+    const url = `${this.config.baseURL}/realtime/client_secrets`;
+
+    const body: Record<string, unknown> = {};
+    if (options.expiresAfterSeconds != null) {
+      body.expires_after = { seconds: options.expiresAfterSeconds };
+    }
+
+    const response = await fetchFn(url, {
+      method: 'POST',
+      headers: {
+        ...this.config.headers(),
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `xAI realtime client secret request failed: ${response.status} ${text}`,
+      );
+    }
+
+    const data = (await response.json()) as {
+      value: string;
+      expires_at?: number;
+    };
+
+    return {
+      token: data.value,
+      // xAI selects the voice model from the `model` query parameter on the
+      // WebSocket URL. Without it the model choice is silently ignored and the
+      // server falls back to its default voice model.
+      url: `wss://${new URL(this.config.baseURL).host}/v1/realtime?model=${encodeURIComponent(this.modelId)}`,
+      expiresAt: data.expires_at,
+    };
+  }
+
+  getWebSocketConfig(options: { token: string; url: string }): {
+    url: string;
+    protocols?: string[];
+  } {
+    return {
+      url: options.url,
+      protocols: [`xai-client-secret.${options.token}`],
+    };
+  }
+
+  parseServerEvent(raw: unknown): RealtimeModelV4ServerEvent {
+    return parseXaiRealtimeServerEvent(raw);
+  }
+
+  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
+    return serializeXaiRealtimeClientEvent(event);
+  }
+
+  buildSessionConfig(
+    config: RealtimeModelV4SessionConfig,
+  ): Record<string, unknown> {
+    return buildXaiSessionConfig(config);
+  }
+}

package/src/remove-additional-properties.ts

@@ -0,0 +1,24 @@
+/**
+ * Recursively removes `additionalProperties: false` entries from a JSON
+ * schema.
+ * Used to sanitize tool input schemas before sending them to the xAI API.
+ * https://docs.x.ai/developers/model-capabilities/text/structured-outputs#supported-types
+ */
+export function removeAdditionalPropertiesFalse(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(removeAdditionalPropertiesFalse);
+  }
+
+  if (value == null || typeof value !== 'object') {
+    return value;
+  }
+
+  const result: Record<string, unknown> = {};
+  for (const [key, propertyValue] of Object.entries(value)) {
+    if (key === 'additionalProperties' && propertyValue === false) {
+      continue;
+    }
+    result[key] = removeAdditionalPropertiesFalse(propertyValue);
+  }
+  return result;
+}

package/src/responses/convert-to-xai-responses-input.ts

@@ -1,10 +1,15 @@
 import {
-  SharedV3Warning,
-  LanguageModelV3Message,
   UnsupportedFunctionalityError,
+  type SharedV4Warning,
+  type LanguageModelV4Message,
 } from '@ai-sdk/provider';
-import { convertToBase64 } from '@ai-sdk/provider-utils';
 import {
+  convertToBase64,
+  getTopLevelMediaType,
+  resolveFullMediaType,
+  resolveProviderReference,
+} from '@ai-sdk/provider-utils';
+import type {
   XaiResponsesInput,
   XaiResponsesUserMessageContentPart,
 } from './xai-responses-api';
@@ -12,14 +17,14 @@ import {
 export async function convertToXaiResponsesInput({
   prompt,
 }: {
-  prompt: LanguageModelV3Message[];
+  prompt: LanguageModelV4Message[];
   store?: boolean;
 }): Promise<{
   input: XaiResponsesInput;
-  inputWarnings: SharedV3Warning[];
+  inputWarnings: SharedV4Warning[];
 }> {
   const input: XaiResponsesInput = [];
-  const inputWarnings: SharedV3Warning[] = [];
+  const inputWarnings: SharedV4Warning[] = [];
 
   for (const message of prompt) {
     switch (message.role) {
@@ -42,22 +47,51 @@ export async function convertToXaiResponsesInput({
             }
 
             case 'file': {
-              if (block.mediaType.startsWith('image/')) {
-                const mediaType =
-                  block.mediaType === 'image/*'
-                    ? 'image/jpeg'
-                    : block.mediaType;
-
-                const imageUrl =
-                  block.data instanceof URL
-                    ? block.data.toString()
-                    : `data:${mediaType};base64,${convertToBase64(block.data)}`;
-
-                contentParts.push({ type: 'input_image', image_url: imageUrl });
-              } else {
-                throw new UnsupportedFunctionalityError({
-                  functionality: `file part media type ${block.mediaType}`,
-                });
+              switch (block.data.type) {
+                case 'reference': {
+                  contentParts.push({
+                    type: 'input_file',
+                    file_id: resolveProviderReference({
+                      reference: block.data.reference,
+                      provider: 'xai',
+                    }),
+                  });
+                  break;
+                }
+                case 'text': {
+                  throw new UnsupportedFunctionalityError({
+                    functionality: 'text file parts',
+                  });
+                }
+                case 'url':
+                case 'data': {
+                  if (getTopLevelMediaType(block.mediaType) === 'image') {
+                    const imageUrl =
+                      block.data.type === 'url'
+                        ? block.data.url.toString()
+                        : `data:${resolveFullMediaType({ part: block })};base64,${convertToBase64(block.data.data)}`;
+
+                    contentParts.push({
+                      type: 'input_image',
+                      image_url: imageUrl,
+                    });
+                  } else if (block.data.type === 'url') {
+                    // xAI's Responses API accepts non-image documents (PDF, text, CSV, etc.)
+                    // via `{ type: 'input_file', file_url }`. See
+                    // https://docs.x.ai/docs/guides/chat-with-files. Inline bytes for
+                    // non-image files are not supported by xAI; callers must upload via
+                    // the Files API and pass a provider reference (file_id) instead.
+                    contentParts.push({
+                      type: 'input_file',
+                      file_url: block.data.url.toString(),
+                    });
+                  } else {
+                    throw new UnsupportedFunctionalityError({
+                      functionality: `file part media type ${block.mediaType} as inline data (xAI Responses requires a URL or a Files API reference for non-image files)`,
+                    });
+                  }
+                  break;
+                }
               }
               break;
             }
@@ -123,7 +157,50 @@ export async function convertToXaiResponsesInput({
               break;
             }
 
-            case 'reasoning':
+            case 'reasoning': {
+              const itemId =
+                typeof part.providerOptions?.xai?.itemId === 'string'
+                  ? part.providerOptions.xai.itemId
+                  : undefined;
+              const encryptedContent =
+                typeof part.providerOptions?.xai?.reasoningEncryptedContent ===
+                'string'
+                  ? part.providerOptions.xai.reasoningEncryptedContent
+                  : undefined;
+
+              if (itemId != null || encryptedContent != null) {
+                const summaryParts: Array<{
+                  type: 'summary_text';
+                  text: string;
+                }> = [];
+                if (part.text.length > 0) {
+                  summaryParts.push({
+                    type: 'summary_text',
+                    text: part.text,
+                  });
+                }
+
+                input.push({
+                  type: 'reasoning',
+                  id: itemId ?? '',
+                  summary: summaryParts,
+                  status: 'completed',
+                  ...(encryptedContent != null && {
+                    encrypted_content: encryptedContent,
+                  }),
+                });
+              } else {
+                inputWarnings.push({
+                  type: 'other',
+                  message:
+                    'Reasoning parts without itemId or encrypted content cannot be sent back to xAI. Skipping.',
+                });
+              }
+              break;
+            }
+
+            case 'reasoning-file':
+            case 'custom':
             case 'file': {
               inputWarnings.push({
                 type: 'other',

package/src/responses/convert-xai-responses-usage.ts

@@ -1,9 +1,9 @@
-import { LanguageModelV3Usage } from '@ai-sdk/provider';
-import { XaiResponsesUsage } from './xai-responses-api';
+import type { LanguageModelV4Usage } from '@ai-sdk/provider';
+import type { XaiResponsesUsage } from './xai-responses-api';
 
 export function convertXaiResponsesUsage(
   usage: XaiResponsesUsage,
-): LanguageModelV3Usage {
+): LanguageModelV4Usage {
   const cacheReadTokens = usage.input_tokens_details?.cached_tokens ?? 0;
   const reasoningTokens = usage.output_tokens_details?.reasoning_tokens ?? 0;