@ai-sdk/xai 4.0.1 → 4.0.3

package/docs/01-xai.mdx

@@ -907,6 +907,37 @@ const { video } = await generateVideo({
 });
 ```
 
+You can also pass the starting frame via the top-level `frameImages` option using the `first_frame` role. This is provider-agnostic and accepts file data (e.g. a `Buffer`) or a `{ type: 'url', url }` object:
+
+```ts
+import { xai, type XaiVideoModelOptions } from '@ai-sdk/xai';
+import { experimental_generateVideo as generateVideo } from 'ai';
+import fs from 'node:fs';
+
+const { video } = await generateVideo({
+  model: xai.video('grok-imagine-video'),
+  prompt: 'The cat slowly turns its head and blinks',
+  frameImages: [
+    {
+      image: fs.readFileSync('./start-frame.png'),
+      frameType: 'first_frame',
+    },
+  ],
+  duration: 5,
+  providerOptions: {
+    xai: {
+      pollTimeoutMs: 600000, // 10 minutes
+    } satisfies XaiVideoModelOptions,
+  },
+});
+```
+
+<Note>
+  xAI does not support first-last-frame interpolation. A `last_frame` entry in
+  `frameImages` is ignored with a warning — use the `extend-video` mode to
+  continue from a video's last frame instead.
+</Note>
+
 ### Video Editing
 
 Edit an existing video using a text prompt by providing a source video URL via provider options:
@@ -1064,9 +1095,38 @@ const { video } = await generateVideo({
 
 Use `<IMAGE_1>`, `<IMAGE_2>`, etc. in your prompt to reference specific images. Up to 7 reference images are supported per request.
 
+Alternatively, use the provider-agnostic top-level `inputReferences` option. Providing it automatically selects reference-to-video mode, and it accepts file data (e.g. a `Buffer`) or `{ type: 'url', url }` objects:
+
+```ts
+import { xai, type XaiVideoModelOptions } from '@ai-sdk/xai';
+import { experimental_generateVideo as generateVideo } from 'ai';
+import fs from 'node:fs';
+
+const { video } = await generateVideo({
+  model: xai.video('grok-imagine-video'),
+  prompt:
+    'The comic cat and the comic dog are having a playful chase ' +
+    'through a sunlit park. Cinematic slow-motion, warm afternoon light.',
+  inputReferences: [
+    fs.readFileSync('./comic-cat.png'),
+    fs.readFileSync('./comic-dog.png'),
+  ],
+  duration: 8,
+  aspectRatio: '16:9',
+  providerOptions: {
+    xai: {
+      pollTimeoutMs: 600000,
+    } satisfies XaiVideoModelOptions,
+  },
+});
+```
+
 <Note>
   Reference-to-video supports `duration`, `aspectRatio`, and `resolution`. Use
-  `mode` to select the operation — each mode is mutually exclusive.
+  `mode` to select the operation — each mode is mutually exclusive. When both
+  are provided, `frameImages` takes precedence over `inputReferences`, and
+  `inputReferences` takes precedence over the legacy `referenceImageUrls`
+  provider option. Reference-to-video requires the `grok-imagine-video` model.
 </Note>
 
 ### Video Provider Options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/xai",
-  "version": "4.0.1",
+  "version": "4.0.3",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -29,9 +29,9 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/openai-compatible": "3.0.0",
-    "@ai-sdk/provider": "4.0.0",
-    "@ai-sdk/provider-utils": "5.0.0"
+    "@ai-sdk/openai-compatible": "3.0.2",
+    "@ai-sdk/provider": "4.0.1",
+    "@ai-sdk/provider-utils": "5.0.2"
   },
   "devDependencies": {
     "@types/node": "22.19.19",

package/src/xai-video-model.ts

@@ -1,6 +1,7 @@
 import {
   AISDKError,
   type Experimental_VideoModelV4,
+  type Experimental_VideoModelV4File,
   type SharedV4Warning,
 } from '@ai-sdk/provider';
 import {
@@ -37,21 +38,87 @@ const RESOLUTION_MAP: Record<string, string> = {
   '640x480': '480p',
 };
 
+type XaiVideoDoGenerateOptions = Parameters<
+  Experimental_VideoModelV4['doGenerate']
+>[0];
+
+function getFirstFrameImage(
+  options: XaiVideoDoGenerateOptions,
+): Experimental_VideoModelV4File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'first_frame')
+    ?.image;
+}
+
+function getLastFrameImage(
+  options: XaiVideoDoGenerateOptions,
+): Experimental_VideoModelV4File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'last_frame')
+    ?.image;
+}
+
+function resolveStartImage(
+  options: XaiVideoDoGenerateOptions,
+): Experimental_VideoModelV4File | undefined {
+  return getFirstFrameImage(options) ?? options.image;
+}
+
+function fileToXaiImageUrl(file: Experimental_VideoModelV4File): string {
+  if (file.type === 'url') {
+    return file.url;
+  }
+
+  const base64Data =
+    typeof file.data === 'string'
+      ? file.data
+      : convertUint8ArrayToBase64(file.data);
+  return `data:${file.mediaType ?? 'image/png'};base64,${base64Data}`;
+}
+
+// Resolves the reference images for R2V generation. First-class
+// `inputReferences` win over the legacy `referenceImageUrls` provider option.
+function resolveReferenceImages(
+  options: XaiVideoDoGenerateOptions,
+  xaiOptions: XaiParsedVideoModelOptions | undefined,
+): Array<{ url: string }> | undefined {
+  if (options.inputReferences != null && options.inputReferences.length > 0) {
+    return options.inputReferences.map(reference => ({
+      url: fileToXaiImageUrl(reference),
+    }));
+  }
+
+  if (
+    xaiOptions?.referenceImageUrls != null &&
+    xaiOptions.referenceImageUrls.length > 0
+  ) {
+    return xaiOptions.referenceImageUrls.map(url => ({ url }));
+  }
+
+  return undefined;
+}
+
 function resolveVideoMode(
-  options: XaiParsedVideoModelOptions | undefined,
+  options: XaiVideoDoGenerateOptions,
+  xaiOptions: XaiParsedVideoModelOptions | undefined,
 ): XaiParsedVideoModelOptions['mode'] | undefined {
-  if (options?.mode != null) {
-    return options.mode;
+  if (xaiOptions?.mode != null) {
+    return xaiOptions.mode;
   }
 
-  if (options?.videoUrl != null) {
+  if (xaiOptions?.videoUrl != null) {
     return 'edit-video';
   }
 
-  if (
-    options?.referenceImageUrls != null &&
-    options.referenceImageUrls.length > 0
-  ) {
+  // frameImages (first/last frame) take precedence over reference images, so
+  // only auto-select reference-to-video when no frame images are provided.
+  const hasFrameImages =
+    options.frameImages != null && options.frameImages.length > 0;
+  const hasInputReferences =
+    options.inputReferences != null && options.inputReferences.length > 0;
+  const hasLegacyReferenceUrls =
+    xaiOptions?.referenceImageUrls != null &&
+    xaiOptions.referenceImageUrls.length > 0;
+
+  if (!hasFrameImages && (hasInputReferences || hasLegacyReferenceUrls)) {
     return 'reference-to-video';
   }
 
@@ -83,7 +150,7 @@ export class XaiVideoModel implements Experimental_VideoModelV4 {
       schema: xaiVideoModelOptionsSchema,
     })) as XaiParsedVideoModelOptions | undefined;
 
-    const effectiveMode = resolveVideoMode(xaiOptions);
+    const effectiveMode = resolveVideoMode(options, xaiOptions);
 
     const isEdit = effectiveMode === 'edit-video';
     const isExtension = effectiveMode === 'extend-video';
@@ -207,26 +274,47 @@ export class XaiVideoModel implements Experimental_VideoModelV4 {
       body.video = { url: xaiOptions!.videoUrl };
     }
 
-    // Convert SDK image input to the nested xAI request image object
-    if (options.image != null) {
-      if (options.image.type === 'url') {
-        body.image = { url: options.image.url };
-      } else {
-        const base64Data =
-          typeof options.image.data === 'string'
-            ? options.image.data
-            : convertUint8ArrayToBase64(options.image.data);
-        body.image = {
-          url: `data:${options.image.mediaType};base64,${base64Data}`,
-        };
-      }
+    // Convert the start image (first_frame or image-to-video input) to the
+    // nested xAI request image object.
+    const startImage = resolveStartImage(options);
+    if (startImage != null) {
+      body.image = { url: fileToXaiImageUrl(startImage) };
+    }
+
+    // xAI has no first-last-frame interpolation; warn and ignore last_frame.
+    if (getLastFrameImage(options) != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'frameImages',
+        details:
+          'xAI video models do not support last_frame. Use ' +
+          'providerOptions.xai.mode "extend-video" to continue from a ' +
+          "video's last frame. The last frame image was ignored.",
+      });
     }
 
     // Reference images for R2V (reference-to-video) generation
     if (hasReferenceImages) {
-      body.reference_images = xaiOptions!.referenceImageUrls!.map(url => ({
-        url,
-      }));
+      const referenceImages = resolveReferenceImages(options, xaiOptions);
+      if (referenceImages != null) {
+        body.reference_images = referenceImages;
+      }
+    }
+
+    // Warn when reference images were provided but cannot be used in the
+    // resolved mode (e.g. alongside frameImages, or in edit/extend modes).
+    if (
+      options.inputReferences != null &&
+      options.inputReferences.length > 0 &&
+      !hasReferenceImages
+    ) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'inputReferences',
+        details:
+          'xAI only supports inputReferences for reference-to-video ' +
+          'generation. The reference images were ignored.',
+      });
     }
 
     if (xaiOptions != null) {