@ai-sdk/xai 4.0.39 → 4.0.40

package/docs/01-xai.mdx

@@ -746,6 +746,45 @@ const result = await generateSpeech({
 
   Whether to normalize written-form input text before synthesizing speech.
 
+- **withTimestamps** _boolean_
+
+  Return character-level timing metadata alongside the audio. The timing data
+  and total duration are exposed via `providerMetadata.xai` (see below).
+
+- **replace** _Record<string, string>_
+
+  Map of phrases to spoken substitutions applied before synthesis. Values may
+  be respellings (`{ 'Acme Mobile': 'Acme Mobull' }`) or IPA phonetics
+  (`{ nginx: '/ˈɛndʒɪn ˈɛks/' }`).
+
+### Provider Metadata
+
+xAI speech results include provider-specific metadata under
+`providerMetadata.xai`:
+
+- **traceId** _string_ — the xAI trace ID for the request, useful for
+  debugging with xAI support.
+- **duration** _number_ — total audio duration in seconds (only with
+  `withTimestamps`).
+- **contentType** _string_ — MIME type of the decoded audio, e.g.
+  `'audio/mpeg'` (only with `withTimestamps`).
+- **audioTimestamps** `{ graphChars: string[]; graphTimes: [number, number][] }` —
+  per-character alignment data (only with `withTimestamps`). `graphChars[i]`
+  is the character spoken during the `[start, end]` interval
+  `graphTimes[i]`, in seconds.
+
+```ts
+const result = await generateSpeech({
+  model: xai.speech(),
+  text: 'Hello world.',
+  providerOptions: {
+    xai: { withTimestamps: true } satisfies XaiSpeechModelOptions,
+  },
+});
+
+const { traceId, duration, audioTimestamps } = result.providerMetadata.xai;
+```
+
 ### Model Capabilities
 
 | Model     | Language  | Speed     | Output Formats             |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/xai",
-  "version": "4.0.39",
+  "version": "4.0.40",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/xai-error.ts

@@ -15,15 +15,25 @@ const responsesErrorSchema = z.object({
   error: z.string(),
 });
 
+// Text to Speech error shape, e.g. {"error":"speed must be between 0.7 and 1.5"}
+const speechErrorSchema = z.object({
+  error: z.string(),
+});
+
 export const xaiErrorDataSchema = z.union([
   chatCompletionsErrorSchema,
   responsesErrorSchema,
+  speechErrorSchema,
 ]);
 
 export type XaiErrorData = z.infer<typeof xaiErrorDataSchema>;
 
 export const xaiFailedResponseHandler = createJsonErrorResponseHandler({
   errorSchema: xaiErrorDataSchema,
-  errorToMessage: data =>
-    'code' in data ? `${data.code}: ${data.error}` : data.error.message,
+  errorToMessage: data => {
+    if (typeof data.error === 'string') {
+      return 'code' in data ? `${data.code}: ${data.error}` : data.error;
+    }
+    return data.error.message;
+  },
 });

package/src/xai-speech-model-options.ts

@@ -46,6 +46,20 @@ export const xaiSpeechModelOptionsSchema = lazySchema(() =>
        * Normalize written-form text into spoken-form text before synthesis.
        */
       textNormalization: z.boolean().nullish(),
+
+      /**
+       * Return character-level timing metadata alongside the audio. When
+       * enabled, the response carries per-character start/end times and the
+       * total duration, exposed via `providerMetadata.xai`.
+       */
+      withTimestamps: z.boolean().nullish(),
+
+      /**
+       * Map of phrases to spoken substitutions applied before synthesis.
+       * Values may be respellings (`{ 'Acme Mobile': 'Acme Mobull' }`) or IPA
+       * phonetics (`{ nginx: '/ˈɛndʒɪn ˈɛks/' }`).
+       */
+      replace: z.record(z.string(), z.string()).nullish(),
     }),
   ),
 );

package/src/xai-speech-model.ts

@@ -1,7 +1,9 @@
 import type { SharedV4Warning, SpeechModelV4 } from '@ai-sdk/provider';
 import {
   combineHeaders,
+  convertBase64ToUint8Array,
   createBinaryResponseHandler,
+  createJsonResponseHandler,
   parseProviderOptions,
   postJsonToApi,
   resolve,
@@ -11,6 +13,7 @@ import {
   type FetchFunction,
   type Resolvable,
 } from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
 import { xaiFailedResponseHandler } from './xai-error';
 import { xaiSpeechModelOptionsSchema } from './xai-speech-model-options';
 
@@ -122,22 +125,30 @@ export class XaiSpeechModel implements SpeechModelV4 {
       speed,
       optimize_streaming_latency: xaiOptions?.optimizeStreamingLatency,
       text_normalization: xaiOptions?.textNormalization,
+      with_timestamps: xaiOptions?.withTimestamps,
+      replace: xaiOptions?.replace,
     };
 
-    return { requestBody, warnings };
+    return {
+      requestBody,
+      warnings,
+      withTimestamps: xaiOptions?.withTimestamps === true,
+    };
   }
 
   async doGenerate(
     options: Parameters<SpeechModelV4['doGenerate']>[0],
   ): Promise<Awaited<ReturnType<SpeechModelV4['doGenerate']>>> {
     const currentDate = this.config._internal?.currentDate?.() ?? new Date();
-    const { requestBody, warnings } = await this.getArgs(options);
-
-    const {
-      value: audio,
-      responseHeaders,
-      rawValue: rawResponse,
-    } = await postJsonToApi({
+    const { requestBody, warnings, withTimestamps } =
+      await this.getArgs(options);
+
+    // With `with_timestamps` the API returns a JSON envelope carrying
+    // base64-encoded audio plus character-level timings instead of raw
+    // audio bytes.
+    const { value, responseHeaders, rawValue } = await postJsonToApi<
+      Uint8Array | XaiSpeechTimestampsResponse
+    >({
       url: `${this.config.baseURL}/tts`,
       headers: combineHeaders(
         this.config.headers ? await resolve(this.config.headers) : undefined,
@@ -145,11 +156,30 @@ export class XaiSpeechModel implements SpeechModelV4 {
       ),
       body: requestBody,
       failedResponseHandler: xaiFailedResponseHandler,
-      successfulResponseHandler: createBinaryResponseHandler(),
+      successfulResponseHandler: withTimestamps
+        ? createJsonResponseHandler(xaiSpeechTimestampsResponseSchema)
+        : createBinaryResponseHandler(),
       abortSignal: options.abortSignal,
       fetch: this.config.fetch,
     });
 
+    let audio: Uint8Array;
+    let envelope: XaiSpeechTimestampsResponse | undefined;
+    if (value instanceof Uint8Array) {
+      audio = value;
+    } else {
+      envelope = value;
+      // Empty audio is returned as-is so the core layer throws
+      // NoSpeechGeneratedError.
+      audio =
+        envelope.audio != null
+          ? convertBase64ToUint8Array(envelope.audio)
+          : new Uint8Array(0);
+    }
+
+    // xAI returns a trace id on every response (success and error).
+    const traceId = responseHeaders?.['x-trace-id'];
+
     return {
       audio,
       warnings,
@@ -160,8 +190,46 @@ export class XaiSpeechModel implements SpeechModelV4 {
         timestamp: currentDate,
         modelId: this.modelId,
         headers: responseHeaders,
-        body: rawResponse,
+        body: rawValue,
+      },
+      providerMetadata: {
+        xai: {
+          ...(traceId != null ? { traceId } : {}),
+          ...(envelope?.duration != null
+            ? { duration: envelope.duration }
+            : {}),
+          ...(envelope?.content_type != null
+            ? { contentType: envelope.content_type }
+            : {}),
+          ...(envelope?.audio_timestamps != null
+            ? {
+                audioTimestamps: {
+                  graphChars: envelope.audio_timestamps.graph_chars,
+                  graphTimes: envelope.audio_timestamps.graph_times,
+                },
+              }
+            : {}),
+        },
       },
     };
   }
 }
+
+// Minimal schema for the `with_timestamps` JSON envelope: only the fields
+// the implementation reads, with `.nullish()` so provider API changes don't
+// break parsing.
+const xaiSpeechTimestampsResponseSchema = z.object({
+  audio: z.string().nullish(),
+  content_type: z.string().nullish(),
+  duration: z.number().nullish(),
+  audio_timestamps: z
+    .object({
+      graph_chars: z.array(z.string()),
+      graph_times: z.array(z.tuple([z.number(), z.number()])),
+    })
+    .nullish(),
+});
+
+type XaiSpeechTimestampsResponse = z.infer<
+  typeof xaiSpeechTimestampsResponseSchema
+>;