@ai-sdk/xai 4.0.35 → 4.0.36

package/docs/01-xai.mdx

@@ -1223,12 +1223,64 @@ const { video } = await generateVideo({
 });
 ```
 
+`inputReferences` accepts image references only. A reference with a non-image
+media type (for example a video or audio clip) is ignored with a warning, and a
+reference with no media type is treated as an image. If no image reference
+remains, reference-to-video is not selected and no reference images are sent.
+
+#### Reference Audio
+
+Reference-to-video can also give the subject a voice. `referenceVoiceIds` takes
+up to 3 xAI **preset** voice ids — you cannot upload your own audio clips.
+Reference the voices from the prompt with `<AUDIO_0>`, `<AUDIO_1>`, and
+`<AUDIO_2>`, in the order the voices are passed.
+
+```ts
+import { xai, type XaiVideoModelOptions } from '@ai-sdk/xai';
+import { experimental_generateVideo as generateVideo } from 'ai';
+
+const { video } = await generateVideo({
+  model: xai.video('grok-imagine-video-1.5'),
+  prompt:
+    'The person from <IMAGE_0> stands in the room from <IMAGE_1> and speaks ' +
+    'to the camera with the voice from <AUDIO_0>.',
+  aspectRatio: '9:16',
+  duration: 10,
+  providerOptions: {
+    xai: {
+      mode: 'reference-to-video',
+      referenceImageUrls: [
+        'https://example.com/person.png',
+        'https://example.com/room.png',
+      ],
+      referenceVoiceIds: ['eve'],
+      resolution: '720p',
+      pollTimeoutMs: 600000,
+    } satisfies XaiVideoModelOptions,
+  },
+});
+```
+
+Valid voice ids come from the xAI
+[text-to-speech voice roster](https://docs.x.ai/developers/model-capabilities/audio/text-to-speech#voices).
+Ids are case-insensitive, and an unknown id returns a `400` response listing the
+available voices. `referenceVoiceIds` is ignored with a warning when the
+resolved operation is not reference-to-video.
+
+<Note>
+  xAI documents reference audio as available in the United States only, for
+  trusted partners. Requests from accounts without access are rejected by the
+  xAI API.
+</Note>
+
 <Note>
   Reference-to-video supports `duration`, `aspectRatio`, and `resolution`. Use
   `mode` to select the operation — each mode is mutually exclusive. When both
   are provided, `frameImages` takes precedence over `inputReferences`, and
   `inputReferences` takes precedence over the legacy `referenceImageUrls`
-  provider option. Reference-to-video requires the `grok-imagine-video` model.
+  provider option. Reference-to-video is supported by the `grok-imagine-video`
+  and `grok-imagine-video-1.5` models; native 1080p requires
+  `grok-imagine-video-1.5`.
 </Note>
 
 ### Video Provider Options
@@ -1244,11 +1296,14 @@ You can validate the provider options using the `XaiVideoModelOptions` type.
 
   Maximum wait time in milliseconds for video generation. Defaults to 600000 (10 minutes).
 
-- **resolution** _'480p' | '720p'_
+- **resolution** _'480p' | '720p' | '1080p'_
 
   Video resolution. When using the SDK's standard `resolution` parameter,
-  `1280x720` maps to `720p` and `854x480` maps to `480p`.
-  Use this provider option to pass the native format directly.
+  `1920x1080` maps to `1080p`, `1280x720` maps to `720p`, and `854x480` maps
+  to `480p`. Use this provider option to pass the native format directly.
+  `1080p` requires the `grok-imagine-video-1.5` model and is available for
+  text-to-video and image-to-video; reference-to-video is capped at `720p`
+  (a `1080p` request is downgraded with a warning).
 
 - **user** _string_
 
@@ -1280,6 +1335,17 @@ You can validate the provider options using the `XaiVideoModelOptions` type.
   `<IMAGE_1>`, `<IMAGE_2>`, etc. in the prompt to reference specific
   images. Used with `mode: 'reference-to-video'`.
 
+- **referenceVoiceIds** _string[]_
+
+  Up to 3 xAI preset voice ids that give the subject a voice in
+  reference-to-video (R2V) generation. Preset voices only — audio clips
+  cannot be uploaded. Ids are case-insensitive and come from the
+  [text-to-speech voice roster](https://docs.x.ai/developers/model-capabilities/audio/text-to-speech#voices);
+  an unknown id returns a `400` listing the available voices. Use `<AUDIO_0>`,
+  `<AUDIO_1>`, and `<AUDIO_2>` tags in the prompt to reference the voices.
+  Ignored with a warning outside reference-to-video. Reference audio is
+  documented as US-only and limited to trusted partners.
+
 <Note>
   Video generation is an asynchronous process that can take several minutes.
   Consider setting `pollTimeoutMs` to at least 10 minutes (600000ms) for
@@ -1290,7 +1356,9 @@ You can validate the provider options using the `XaiVideoModelOptions` type.
 ### Aspect Ratio and Resolution
 
 For **text-to-video**, you can specify both `aspectRatio` and `resolution`.
-The default aspect ratio is `16:9` and the default resolution is `480p`.
+The default aspect ratio is `16:9` and the default resolution is `480p`. The
+`grok-imagine-video-1.5` model additionally supports native `1080p` for
+text-to-video and image-to-video.
 
 For **image-to-video**, the output defaults to the input image's aspect ratio.
 If you specify `aspectRatio`, it will override this and stretch the image to the
@@ -1306,13 +1374,18 @@ from the source video. `duration` is supported and controls only the
 extension length.
 
 For **reference-to-video (R2V)**, you can specify `duration`, `aspectRatio`,
-and `resolution` just like text-to-video.
+and `resolution`. Unlike text-to-video, R2V is capped at `720p` — a `1080p`
+request is downgraded to `720p` with a warning.
 
 ### Video Model Capabilities
 
-| Model                | Duration | Aspect Ratios                                     | Resolution     | Image-to-Video | Editing   | Extension | R2V       |
-| -------------------- | -------- | ------------------------------------------------- | -------------- | -------------- | --------- | --------- | --------- |
-| `grok-imagine-video` | 1–15s    | `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3` | `480p`, `720p` | <Check />      | <Check /> | <Check /> | <Check /> |
+| Model                    | Duration | Aspect Ratios                                     | Resolution                | Image-to-Video | Editing   | Extension | R2V       |
+| ------------------------ | -------- | ------------------------------------------------- | ------------------------- | -------------- | --------- | --------- | --------- |
+| `grok-imagine-video`     | 1–15s    | `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3` | `480p`, `720p`            | <Check />      | <Check /> | <Check /> | <Check /> |
+| `grok-imagine-video-1.5` | 1–15s    | `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3` | `480p`, `720p`, `1080p`\* | <Check />      | <Check /> | <Check /> | <Check /> |
+
+\* Native `1080p` applies to text-to-video and image-to-video. Reference-to-video
+is capped at `720p` — a `1080p` request is downgraded with a warning.
 
 <Note>
   You can also pass any available provider model ID as a string if needed.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/xai",
-  "version": "4.0.35",
+  "version": "4.0.36",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/xai-video-model-options.ts

@@ -2,7 +2,7 @@ import { lazySchema, zodSchema } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
 
 const nonEmptyStringSchema = z.string().min(1);
-const resolutionSchema = z.enum(['480p', '720p']);
+const resolutionSchema = z.enum(['480p', '720p', '1080p']);
 const modeSchema = z.enum(['edit-video', 'extend-video', 'reference-to-video']);
 
 export type XaiVideoMode = z.infer<typeof modeSchema>;
@@ -48,6 +48,10 @@ interface XaiVideoReferenceToVideoOptions
   mode: 'reference-to-video';
   /** Reference image URLs (1-7) for R2V generation. */
   referenceImageUrls: string[];
+  /**
+   * Preset voice ids (up to 3) that give the subject a voice.
+   */
+  referenceVoiceIds?: string[];
 }
 
 interface XaiVideoGenerationOptions
@@ -75,6 +79,10 @@ interface XaiLegacyReferenceToVideoOptions
    */
   mode?: undefined;
   referenceImageUrls: string[];
+  /**
+   * Preset voice ids (up to 3) that give the subject a voice.
+   */
+  referenceVoiceIds?: string[];
 }
 
 /**
@@ -106,52 +114,11 @@ const baseFields = {
   resolution: resolutionSchema.nullish(),
 };
 
-const userField = {
-  user: z.string().optional(),
-};
-
-const editVideoSchema = z.object({
-  ...baseFields,
-  ...userField,
-  mode: z.literal('edit-video'),
-  videoUrl: nonEmptyStringSchema,
-  referenceImageUrls: z.undefined().optional(),
-});
-
-const extendVideoSchema = z.object({
-  ...baseFields,
-  mode: z.literal('extend-video'),
-  videoUrl: nonEmptyStringSchema,
-  referenceImageUrls: z.undefined().optional(),
-});
-
-const referenceToVideoSchema = z.object({
-  ...baseFields,
-  ...userField,
-  mode: z.literal('reference-to-video'),
-  referenceImageUrls: z.array(nonEmptyStringSchema).min(1).max(7),
-  videoUrl: z.undefined().optional(),
-});
-
-const autoDetectSchema = z.object({
-  ...baseFields,
-  ...userField,
-  mode: z.undefined().optional(),
-  videoUrl: nonEmptyStringSchema.optional(),
-  referenceImageUrls: z.array(nonEmptyStringSchema).min(1).max(7).optional(),
-});
-
-export const xaiVideoModelOptions = z.union([
-  editVideoSchema,
-  extendVideoSchema,
-  referenceToVideoSchema,
-  autoDetectSchema,
-]);
-
 const runtimeSchema = z.looseObject({
   mode: modeSchema.optional(),
   videoUrl: nonEmptyStringSchema.optional(),
   referenceImageUrls: z.array(nonEmptyStringSchema).min(1).max(7).optional(),
+  referenceVoiceIds: z.array(nonEmptyStringSchema).max(3).optional(),
   user: z.string().optional(),
   ...baseFields,
 });

package/src/xai-video-model.ts

@@ -40,6 +40,7 @@ interface XaiVideoModelConfig {
 }
 
 const RESOLUTION_MAP: Record<string, string> = {
+  '1920x1080': '1080p',
   '1280x720': '720p',
   '854x480': '480p',
   '640x480': '480p',
@@ -70,7 +71,12 @@ function resolveStartImage(
 const isVideoFile = (file: VideoModelV4File): boolean =>
   file.mediaType != null && getTopLevelMediaType(file.mediaType) === 'video';
 
-function fileToXaiImageUrl(file: VideoModelV4File): string {
+// References without a media type (only possible for URLs) are treated as
+// images, matching the legacy `referenceImageUrls` behavior.
+const isImageReference = (file: VideoModelV4File): boolean =>
+  file.mediaType == null || getTopLevelMediaType(file.mediaType) === 'image';
+
+function fileToXaiUrl(file: VideoModelV4File): string {
   if (file.type === 'url') {
     return file.url;
   }
@@ -84,32 +90,37 @@ function fileToXaiImageUrl(file: VideoModelV4File): string {
 
 // Resolves the reference images for R2V generation. First-class
 // `inputReferences` win over the legacy `referenceImageUrls` provider option.
-// Video references are not supported for reference-to-video and are skipped
-// with a warning.
-function resolveReferenceImages(
+// Non-image references (video or audio) are not supported for
+// reference-to-video and are skipped with a warning.
+function resolveReferences(
   options: XaiVideoCallOptions,
   xaiOptions: XaiParsedVideoModelOptions | undefined,
   warnings: SharedV4Warning[],
 ): Array<{ url: string }> | undefined {
   if (options.inputReferences != null && options.inputReferences.length > 0) {
-    const imageReferences = options.inputReferences.filter(reference => {
-      if (isVideoFile(reference)) {
+    const imageFiles: VideoModelV4File[] = [];
+
+    for (const reference of options.inputReferences) {
+      if (!isImageReference(reference)) {
         warnings.push({
           type: 'unsupported',
           feature: 'inputReferences',
-          details:
-            'xAI reference-to-video accepts image references only. The video ' +
-            'reference was ignored. Use providerOptions.xai.mode ' +
-            '"extend-video" to continue from a video.',
+          details: isVideoFile(reference)
+            ? 'xAI reference-to-video accepts image references only. The ' +
+              'video reference was ignored. Use providerOptions.xai.mode ' +
+              '"extend-video" to continue from a video.'
+            : 'xAI reference-to-video accepts image references only. The ' +
+              'non-image reference was ignored.',
         });
-        return false;
+        continue;
       }
-      return true;
-    });
 
-    return imageReferences.map(reference => ({
-      url: fileToXaiImageUrl(reference),
-    }));
+      imageFiles.push(reference);
+    }
+
+    return imageFiles.length > 0
+      ? imageFiles.map(reference => ({ url: fileToXaiUrl(reference) }))
+      : undefined;
   }
 
   if (
@@ -122,6 +133,11 @@ function resolveReferenceImages(
   return undefined;
 }
 
+// True when at least one reference would survive as an image.
+function hasImageInputReference(options: XaiVideoCallOptions): boolean {
+  return options.inputReferences?.some(isImageReference) ?? false;
+}
+
 function resolveVideoMode(
   options: XaiVideoCallOptions,
   xaiOptions: XaiParsedVideoModelOptions | undefined,
@@ -138,13 +154,17 @@ function resolveVideoMode(
   // only auto-select reference-to-video when no frame images are provided.
   const hasFrameImages =
     options.frameImages != null && options.frameImages.length > 0;
-  const hasInputReferences =
-    options.inputReferences != null && options.inputReferences.length > 0;
   const hasLegacyReferenceUrls =
     xaiOptions?.referenceImageUrls != null &&
     xaiOptions.referenceImageUrls.length > 0;
 
-  if (!hasFrameImages && (hasInputReferences || hasLegacyReferenceUrls)) {
+  // Reference-to-video needs at least one image reference. An audio-only (or
+  // video-only) `inputReferences` array must not flip a text- or
+  // image-to-video request into R2V.
+  if (
+    !hasFrameImages &&
+    (hasImageInputReference(options) || hasLegacyReferenceUrls)
+  ) {
     return 'reference-to-video';
   }
 
@@ -290,7 +310,8 @@ export class XaiVideoModel implements VideoModelV4 {
           feature: 'resolution',
           details:
             `Unrecognized resolution "${options.resolution}". ` +
-            'Use providerOptions.xai.resolution with "480p" or "720p" instead.',
+            'Use providerOptions.xai.resolution with "480p", "720p", or ' +
+            '"1080p" instead.',
         });
       }
     }
@@ -320,7 +341,7 @@ export class XaiVideoModel implements VideoModelV4 {
             'continue from a video instead.',
         });
       } else {
-        body.image = { url: fileToXaiImageUrl(startImage) };
+        body.image = { url: fileToXaiUrl(startImage) };
       }
     }
 
@@ -342,18 +363,58 @@ export class XaiVideoModel implements VideoModelV4 {
 
     // Reference images for R2V (reference-to-video) generation
     if (hasReferenceImages) {
-      const referenceImages = resolveReferenceImages(
-        options,
-        xaiOptions,
-        warnings,
-      );
+      const referenceImages = resolveReferences(options, xaiOptions, warnings);
+
       if (referenceImages != null) {
         body.reference_images = referenceImages;
+      } else {
+        // Explicit R2V with no usable image references would silently send
+        // a plain generations request; tell the user it is no longer R2V.
+        warnings.push({
+          type: 'unsupported',
+          feature: 'referenceImages',
+          details:
+            'xAI reference-to-video requires at least one image reference. ' +
+            'The video will be generated without reference images.',
+        });
+      }
+
+      const referenceVoiceIds = xaiOptions?.referenceVoiceIds;
+      if (referenceVoiceIds != null && referenceVoiceIds.length > 0) {
+        body.reference_audios = referenceVoiceIds.map(voiceId => ({
+          voice_id: voiceId,
+        }));
+      }
+
+      // Reference-to-video is capped at 720p; downgrade a 1080p request.
+      if (body.resolution === '1080p') {
+        warnings.push({
+          type: 'unsupported',
+          feature: 'resolution',
+          details:
+            'xAI reference-to-video is limited to 720p. The request was ' +
+            'downgraded from 1080p to 720p.',
+        });
+        body.resolution = '720p';
       }
     }
 
-    // Warn when reference images were provided but cannot be used in the
-    // resolved mode (e.g. alongside frameImages, or in edit/extend modes).
+    // 1080p requires grok-imagine-video-1.5; the original grok-imagine-video
+    // rejects it. Warn, but send the request as the user asked.
+    if (body.resolution === '1080p' && this.modelId === 'grok-imagine-video') {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'resolution',
+        details:
+          'xAI model "grok-imagine-video" does not support 1080p. Use ' +
+          '"grok-imagine-video-1.5" for 1080p, or a lower resolution. The ' +
+          'request was sent with 1080p.',
+      });
+    }
+
+    // Warn when references were provided but cannot be used in the resolved
+    // mode (e.g. alongside frameImages, in edit/extend modes, or when the
+    // references carried no usable image to drive reference-to-video).
     if (
       options.inputReferences != null &&
       options.inputReferences.length > 0 &&
@@ -362,9 +423,26 @@ export class XaiVideoModel implements VideoModelV4 {
       warnings.push({
         type: 'unsupported',
         feature: 'inputReferences',
+        details: hasImageInputReference(options)
+          ? 'xAI only supports inputReferences for reference-to-video ' +
+            'generation. The reference images were ignored.'
+          : 'xAI reference-to-video requires at least one image reference. ' +
+            'The references were ignored.',
+      });
+    }
+
+    // Preset reference voices only apply to reference-to-video generation.
+    if (
+      xaiOptions?.referenceVoiceIds != null &&
+      xaiOptions.referenceVoiceIds.length > 0 &&
+      !hasReferenceImages
+    ) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'referenceVoiceIds',
         details:
-          'xAI only supports inputReferences for reference-to-video ' +
-          'generation. The reference images were ignored.',
+          'xAI only supports reference voices for reference-to-video ' +
+          'generation. The reference voices were ignored.',
       });
     }
 
@@ -382,6 +460,7 @@ export class XaiVideoModel implements VideoModelV4 {
             'resolution',
             'videoUrl',
             'referenceImageUrls',
+            'referenceVoiceIds',
             'user',
           ].includes(key)
         ) {

package/src/xai-video-settings.ts

@@ -1 +1,4 @@
-export type XaiVideoModelId = 'grok-imagine-video' | (string & {});
+export type XaiVideoModelId =
+  | 'grok-imagine-video'
+  | 'grok-imagine-video-1.5'
+  | (string & {});