@ai-sdk/xai 4.0.5 → 4.0.6

package/docs/01-xai.mdx

@@ -725,11 +725,38 @@ const result = await transcribe({
 
   Includes filler words such as `uh` and `um` in the transcript.
 
+- **streaming** _object_
+
+  Options for streaming speech-to-text over WebSocket. Use with `experimental_streamTranscribe`.
+  - **interimResults** _boolean_
+
+    Emits partial transcripts while speech is being processed.
+
+  - **endpointing** _number_
+
+    Silence duration in milliseconds before an utterance-final event. Range: 0-5000.
+
+  - **smartTurn** _number_
+
+    End-of-turn detection threshold. When set, enables Smart Turn. Range: 0.0-1.0.
+
+  - **smartTurnTimeout** _number_
+
+    Maximum silence duration in milliseconds before forcing `speech_final`. Range: 1-5000.
+
 ### Model Capabilities
 
-| Model     | Word Timestamps     | Diarization         | Multichannel        |
-| --------- | ------------------- | ------------------- | ------------------- |
-| `default` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
+| Model     | Request/Response    | Streaming           | Word Timestamps     | Diarization         | Multichannel        |
+| --------- | ------------------- | ------------------- | ------------------- | ------------------- | ------------------- |
+| `default` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
+
+<Note>
+  Word timestamps, diarization results, and segments are only returned on the
+  request/response path (`transcribe`). Streaming transcription
+  (`experimental_streamTranscribe`) emits partial and final transcript text with
+  utterance-level timing, but does not surface per-word timestamps, speaker
+  labels, or segments.
+</Note>
 
 ## Image Models
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/xai",
-  "version": "4.0.5",
+  "version": "4.0.6",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -29,17 +29,17 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.1",
-    "@ai-sdk/provider-utils": "5.0.4",
-    "@ai-sdk/openai-compatible": "3.0.4"
+    "@ai-sdk/openai-compatible": "3.0.5",
+    "@ai-sdk/provider": "4.0.2",
+    "@ai-sdk/provider-utils": "5.0.5"
   },
   "devDependencies": {
     "@types/node": "22.19.19",
     "tsup": "^8.5.1",
     "typescript": "5.8.3",
     "zod": "3.25.76",
-    "@vercel/ai-tsconfig": "0.0.0",
-    "@ai-sdk/test-server": "2.0.0"
+    "@ai-sdk/test-server": "2.0.0",
+    "@vercel/ai-tsconfig": "0.0.0"
   },
   "peerDependencies": {
     "zod": "^3.25.76 || ^4.1.8"

package/src/xai-provider.ts

@@ -16,6 +16,7 @@ import {
   withoutTrailingSlash,
   withUserAgentSuffix,
   type FetchFunction,
+  type WebSocketConstructor,
 } from '@ai-sdk/provider-utils';
 import { XaiChatLanguageModel } from './xai-chat-language-model';
 import type { XaiChatModelId } from './xai-chat-language-model-options';
@@ -129,6 +130,12 @@ export interface XaiProviderSettings {
    * or to provide a custom fetch implementation for e.g. testing.
    */
   fetch?: FetchFunction;
+
+  /**
+   * Custom WebSocket implementation. Required in runtimes whose native
+   * WebSocket constructor does not support headers for xAI streaming STT.
+   */
+  webSocket?: WebSocketConstructor;
 }
 
 export function createXai(options: XaiProviderSettings = {}): XaiProvider {
@@ -210,6 +217,7 @@ export function createXai(options: XaiProviderSettings = {}): XaiProvider {
       baseURL,
       headers: getHeaders,
       fetch: options.fetch,
+      webSocket: options.webSocket,
     });
   };
 

package/src/xai-transcription-model-options.ts

@@ -61,6 +61,33 @@ export const xaiTranscriptionModelOptionsSchema = lazySchema(() =>
        * Include filler words such as "uh" and "um" in the transcript.
        */
       fillerWords: z.boolean().nullish(),
+
+      /**
+       * Options for streaming speech-to-text over WebSocket.
+       */
+      streaming: z
+        .object({
+          /**
+           * Emit interim transcript results while speech is being processed.
+           */
+          interimResults: z.boolean().optional(),
+
+          /**
+           * Silence duration in milliseconds before an utterance-final event.
+           */
+          endpointing: z.number().int().min(0).max(5000).optional(),
+
+          /**
+           * End-of-turn detection threshold. When set, enables Smart Turn.
+           */
+          smartTurn: z.number().min(0).max(1).optional(),
+
+          /**
+           * Maximum silence duration in milliseconds before forcing speech_final.
+           */
+          smartTurnTimeout: z.number().int().min(1).max(5000).optional(),
+        })
+        .optional(),
     }),
   ),
 );

package/src/xai-transcription-model.ts

@@ -1,30 +1,56 @@
-import type { SharedV4Warning, TranscriptionModelV4 } from '@ai-sdk/provider';
+import {
+  InvalidArgumentError,
+  type Experimental_TranscriptionModelV4StreamOptions as TranscriptionModelV4StreamOptions,
+  type SharedV4Warning,
+  type TranscriptionModelV4,
+} from '@ai-sdk/provider';
 import {
   combineHeaders,
   convertBase64ToUint8Array,
   createJsonResponseHandler,
+  getWebSocketConstructor,
   mediaTypeToExtension,
   parseProviderOptions,
   postFormDataToApi,
+  readWebSocketMessageText,
+  safeParseJSON,
   serializeModelOptions,
+  toWebSocketUrl,
   WORKFLOW_DESERIALIZE,
   WORKFLOW_SERIALIZE,
   type FetchFunction,
+  type WebSocketConstructor,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
 import { xaiFailedResponseHandler } from './xai-error';
-import { xaiTranscriptionModelOptionsSchema } from './xai-transcription-model-options';
+import {
+  xaiTranscriptionModelOptionsSchema,
+  type XaiTranscriptionModelOptions,
+} from './xai-transcription-model-options';
 
 interface XaiTranscriptionModelConfig {
   provider: string;
   baseURL: string | undefined;
   headers?: () => Record<string, string | undefined>;
   fetch?: FetchFunction;
+  webSocket?: WebSocketConstructor;
   _internal?: {
     currentDate?: () => Date;
   };
 }
 
+type XaiStreamingTranscriptionEvent = {
+  type?: string;
+  text?: string;
+  words?: Array<{ text?: string; start?: number; end?: number }>;
+  is_final?: boolean;
+  speech_final?: boolean;
+  start?: number;
+  duration?: number;
+  channel_index?: number;
+  message?: string;
+};
+
 export class XaiTranscriptionModel implements TranscriptionModelV4 {
   readonly specificationVersion = 'v4';
 
@@ -148,6 +174,359 @@ export class XaiTranscriptionModel implements TranscriptionModelV4 {
       },
     };
   }
+
+  async doStream(
+    options: TranscriptionModelV4StreamOptions,
+  ): Promise<
+    Awaited<ReturnType<NonNullable<TranscriptionModelV4['doStream']>>>
+  > {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const warnings: SharedV4Warning[] = [];
+    const xaiOptions = await parseProviderOptions({
+      provider: 'xai',
+      providerOptions: options.providerOptions,
+      schema: xaiTranscriptionModelOptionsSchema,
+    });
+
+    if (xaiOptions?.multichannel === true && xaiOptions.channels == null) {
+      throw new InvalidArgumentError({
+        argument: 'providerOptions',
+        message:
+          'providerOptions.xai.channels is required when providerOptions.xai.multichannel is true',
+      });
+    }
+
+    if (xaiOptions?.format != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'providerOptions.xai.format',
+        details: 'xAI streaming transcription does not support format.',
+      });
+    }
+
+    if (
+      xaiOptions?.audioFormat == null &&
+      !isKnownInputAudioFormat(options.inputAudioFormat.type)
+    ) {
+      warnings.push({
+        type: 'other',
+        message:
+          `Unrecognized inputAudioFormat.type "${options.inputAudioFormat.type}"; ` +
+          `falling back to raw PCM encoding. ` +
+          `Use audio/pcm, audio/pcmu, or audio/pcma, ` +
+          `or set providerOptions.xai.audioFormat explicitly.`,
+      });
+    }
+
+    const url = buildXaiStreamingTranscriptionUrl({
+      baseURL: this.config.baseURL ?? 'https://api.x.ai/v1',
+      inputAudioFormat: options.inputAudioFormat,
+      providerOptions: xaiOptions,
+    });
+    const headers = combineHeaders(this.config.headers?.(), options.headers);
+
+    return {
+      request: { body: url.toString() },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+      },
+      stream: createXaiStreamingTranscriptionStream({
+        webSocket: this.config.webSocket,
+        url,
+        headers,
+        warnings,
+        language: xaiOptions?.language ?? undefined,
+        expectedDoneCount:
+          xaiOptions?.multichannel === true ? xaiOptions.channels! : 1,
+        audio: options.audio,
+        abortSignal: options.abortSignal,
+        includeRawChunks: options.includeRawChunks,
+      }),
+    };
+  }
+}
+
+function createXaiStreamingTranscriptionStream({
+  webSocket,
+  url,
+  headers,
+  warnings,
+  language,
+  expectedDoneCount,
+  audio,
+  abortSignal,
+  includeRawChunks,
+}: {
+  webSocket: WebSocketConstructor | undefined;
+  url: URL;
+  headers: Record<string, string | undefined>;
+  warnings: SharedV4Warning[];
+  language: string | undefined;
+  expectedDoneCount: number;
+  audio: ReadableStream<Uint8Array | string>;
+  abortSignal: AbortSignal | undefined;
+  includeRawChunks: boolean | undefined;
+}) {
+  let finished = false;
+  let cleanup: (closeCode?: number) => void = () => {};
+
+  return new ReadableStream({
+    start: controller => {
+      const WebSocketConstructor = getWebSocketConstructor(webSocket);
+      const ws = new WebSocketConstructor(url, undefined, { headers });
+      const doneTexts = new Map<number, string>();
+      let doneDuration: number | undefined;
+      let audioReader:
+        | ReadableStreamDefaultReader<Uint8Array | string>
+        | undefined;
+
+      cleanup = (closeCode?: number) => {
+        abortSignal?.removeEventListener('abort', abort);
+        void audioReader?.cancel().catch(() => {});
+        try {
+          ws.close(closeCode);
+        } catch {}
+      };
+
+      const finishWithError = (error: unknown) => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        controller.error(error);
+      };
+
+      const maybeFinish = () => {
+        if (finished || doneTexts.size < expectedDoneCount) return;
+        finished = true;
+        const text = [...doneTexts.entries()]
+          .sort(([a], [b]) => a - b)
+          .map(([, value]) => value)
+          .join('\n');
+        controller.enqueue({
+          type: 'finish',
+          text,
+          segments: [],
+          language,
+          durationInSeconds: doneDuration,
+        });
+        controller.close();
+        cleanup(1000);
+      };
+
+      const abort = () => {
+        finishWithError(abortSignal?.reason ?? new Error('Aborted'));
+      };
+      if (abortSignal?.aborted) {
+        abort();
+        return;
+      }
+      abortSignal?.addEventListener('abort', abort, { once: true });
+
+      const sendAudio = async () => {
+        audioReader = audio.getReader();
+        try {
+          while (true) {
+            const { done, value } = await audioReader.read();
+            if (done || finished) break;
+            ws.send(
+              value instanceof Uint8Array
+                ? value
+                : convertBase64ToUint8Array(value),
+            );
+          }
+        } finally {
+          audioReader.releaseLock();
+        }
+        if (!finished) {
+          ws.send(JSON.stringify({ type: 'audio.done' }));
+        }
+      };
+
+      ws.onmessage = event => {
+        void readWebSocketMessageText(event.data)
+          .then(async text => {
+            const parsed = await safeParseJSON({ text });
+            if (!parsed.success) return;
+            const raw = parsed.value as XaiStreamingTranscriptionEvent;
+
+            if (includeRawChunks) {
+              controller.enqueue({ type: 'raw', rawValue: raw });
+            }
+
+            switch (raw.type) {
+              case 'transcript.created': {
+                controller.enqueue({ type: 'stream-start', warnings });
+                void sendAudio().catch(finishWithError);
+                break;
+              }
+
+              case 'transcript.partial': {
+                const id = channelId(raw.channel_index);
+                const timing = timingFromXaiEvent(raw);
+                if (raw.is_final) {
+                  controller.enqueue({
+                    type: 'transcript-final',
+                    id,
+                    text: raw.text ?? '',
+                    ...timing,
+                    channelIndex: raw.channel_index,
+                  });
+                } else {
+                  controller.enqueue({
+                    type: 'transcript-partial',
+                    id,
+                    text: raw.text ?? '',
+                    startSecond: raw.start,
+                    durationInSeconds: raw.duration,
+                    channelIndex: raw.channel_index,
+                  });
+                }
+                break;
+              }
+
+              case 'transcript.done': {
+                const channelIndex = raw.channel_index ?? 0;
+                doneTexts.set(channelIndex, raw.text ?? '');
+                doneDuration = raw.duration ?? doneDuration;
+                maybeFinish();
+                break;
+              }
+
+              case 'error': {
+                // xAI STT errors are terminal: surface the server message
+                // instead of letting the socket close mask it.
+                finishWithError(new Error(raw.message ?? 'xAI STT error'));
+                break;
+              }
+            }
+          })
+          .catch(finishWithError);
+      };
+
+      ws.onerror = () => {
+        finishWithError(
+          new Error(
+            'xAI streaming transcription error.' +
+              (webSocket == null
+                ? ' Note: the native WebSocket implementation in browsers,' +
+                  ' Node.js, Deno, and Bun cannot send the Authorization' +
+                  ' header required by xAI. Pass a header-capable WebSocket' +
+                  " implementation (e.g. the 'ws' package) via" +
+                  ' createXai({ webSocket }).'
+                : ''),
+          ),
+        );
+      };
+
+      ws.onclose = () => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        controller.close();
+      };
+    },
+
+    cancel: () => {
+      if (finished) return;
+      finished = true;
+      cleanup();
+    },
+  });
+}
+
+function buildXaiStreamingTranscriptionUrl({
+  baseURL,
+  inputAudioFormat,
+  providerOptions,
+}: {
+  baseURL: string;
+  inputAudioFormat: TranscriptionModelV4StreamOptions['inputAudioFormat'];
+  providerOptions: XaiTranscriptionModelOptions | undefined;
+}) {
+  const url = toWebSocketUrl(`${baseURL}/stt`);
+
+  appendSearchParam(
+    url,
+    'sample_rate',
+    providerOptions?.sampleRate ?? inputAudioFormat.rate,
+  );
+  appendSearchParam(
+    url,
+    'encoding',
+    providerOptions?.audioFormat ??
+      encodingFromInputAudioFormat(inputAudioFormat.type),
+  );
+  appendSearchParam(url, 'language', providerOptions?.language);
+  appendSearchParam(url, 'diarize', providerOptions?.diarize);
+  appendSearchParam(url, 'filler_words', providerOptions?.fillerWords);
+  appendSearchParam(url, 'multichannel', providerOptions?.multichannel);
+  appendSearchParam(url, 'channels', providerOptions?.channels);
+  appendSearchParam(
+    url,
+    'interim_results',
+    providerOptions?.streaming?.interimResults,
+  );
+  appendSearchParam(
+    url,
+    'endpointing',
+    providerOptions?.streaming?.endpointing,
+  );
+  appendSearchParam(url, 'smart_turn', providerOptions?.streaming?.smartTurn);
+  appendSearchParam(
+    url,
+    'smart_turn_timeout',
+    providerOptions?.streaming?.smartTurnTimeout,
+  );
+
+  if (providerOptions?.keyterm != null) {
+    const keyterms = Array.isArray(providerOptions.keyterm)
+      ? providerOptions.keyterm
+      : [providerOptions.keyterm];
+    for (const keyterm of keyterms) {
+      url.searchParams.append('keyterm', keyterm);
+    }
+  }
+
+  return url;
+}
+
+function appendSearchParam(
+  url: URL,
+  key: string,
+  value: string | number | boolean | null | undefined,
+) {
+  if (value != null) {
+    url.searchParams.set(key, String(value));
+  }
+}
+
+function isKnownInputAudioFormat(type: string): boolean {
+  return type === 'audio/pcm' || type === 'audio/pcmu' || type === 'audio/pcma';
+}
+
+function encodingFromInputAudioFormat(type: string): 'pcm' | 'mulaw' | 'alaw' {
+  switch (type) {
+    case 'audio/pcmu':
+      return 'mulaw';
+    case 'audio/pcma':
+      return 'alaw';
+    default:
+      return 'pcm';
+  }
+}
+
+function channelId(channelIndex: number | undefined): string | undefined {
+  return channelIndex == null ? undefined : `channel-${channelIndex}`;
+}
+
+function timingFromXaiEvent(event: XaiStreamingTranscriptionEvent) {
+  return {
+    ...(event.start != null ? { startSecond: event.start } : {}),
+    ...(event.start != null && event.duration != null
+      ? { endSecond: event.start + event.duration }
+      : {}),
+  };
 }
 
 const xaiTranscriptionResponseSchema = z.object({