ai 7.0.7 → 7.0.8

package/dist/internal/index.js

@@ -83,7 +83,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "7.0.7" : "0.0.0-test";
+var VERSION = true ? "7.0.8" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({

package/docs/03-ai-sdk-core/38-video-generation.mdx

@@ -171,6 +171,46 @@ const { video } = await generateVideo({
 });
 ```
 
+### First and Last Frame
+
+Some video models support first-last-frame generation, where you provide the
+starting and/or ending frames of the video. Use the `frameImages` option to pass
+role-tagged images in a provider-agnostic way:
+
+```tsx highlight={"5-16"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The cat walks across the scene and transforms into a dog by the end',
+  frameImages: [
+    {
+      image: 'https://example.com/first-frame.png',
+      frameType: 'first_frame',
+    },
+    {
+      image: 'https://example.com/last-frame.png',
+      frameType: 'last_frame',
+    },
+  ],
+});
+```
+
+### Reference Images
+
+Some video models support reference-to-video generation, where you provide one or
+more reference images that the model incorporates into the generated video. Use the `inputReferences` option to pass
+these images in a provider-agnostic way:
+
+```tsx highlight={"5-8"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The two characters meet in a bustling market',
+  inputReferences: [
+    'https://example.com/character-1.png',
+    'https://example.com/character-2.png',
+  ],
+});
+```
+
 ### Providing a Seed
 
 You can provide a seed to the `experimental_generateVideo` function to control the output of the video generation process.

package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx

@@ -110,6 +110,18 @@ console.log(videos);
       isOptional: true,
       description: 'Seed for the video generation.',
     },
+    {
+      name: 'frameImages',
+      type: 'Array<{ image: DataContent; frameType: "first_frame" | "last_frame" }>',
+      isOptional: true,
+      description: 'Role-tagged image inputs for first-last-frame generation.',
+    },
+    {
+      name: 'inputReferences',
+      type: 'Array<DataContent>',
+      isOptional: true,
+      description: 'Reference image inputs for reference-to-video generation.',
+    },
     {
       name: 'generateAudio',
       type: 'boolean',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.7",
+  "version": "7.0.8",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,9 +42,9 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.5",
-    "@ai-sdk/provider": "4.0.0",
-    "@ai-sdk/provider-utils": "5.0.1"
+    "@ai-sdk/gateway": "4.0.6",
+    "@ai-sdk/provider": "4.0.1",
+    "@ai-sdk/provider-utils": "5.0.2"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",

package/src/generate-video/generate-video.ts

@@ -2,6 +2,8 @@ import type {
   Experimental_VideoModelV4,
   Experimental_VideoModelV4CallOptions,
   Experimental_VideoModelV4File,
+  Experimental_VideoModelV4FrameImage,
+  Experimental_VideoModelV4FrameType,
   SharedV4ProviderMetadata,
 } from '@ai-sdk/provider';
 import {
@@ -26,6 +28,7 @@ import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
 import type { GenerateVideoResult } from './generate-video-result';
 import { splitDataUrl } from '../prompt/split-data-url';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
 
 export type GenerateVideoPrompt =
   | string
@@ -45,6 +48,8 @@ export type GenerateVideoPrompt =
  * @param duration - Duration of the video in seconds.
  * @param fps - Frames per second for the video.
  * @param seed - Seed for the video generation.
+ * @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
+ * @param inputReferences - Reference image inputs for reference-to-video generation.
  * @param generateAudio - Whether the model should generate audio alongside the video.
  * @param providerOptions - Additional provider-specific options that are passed through to the provider
  * as body parameters.
@@ -66,6 +71,8 @@ export async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -118,6 +125,26 @@ export async function experimental_generateVideo({
    */
   seed?: number;
 
+  /**
+   * Role-tagged image inputs for image-to-video and first-last-frame generation.
+   */
+  frameImages?: Array<{
+    /**
+     * The image for this frame.
+     */
+    image: DataContent;
+
+    /**
+     * Which frame this image represents.
+     */
+    frameType: Experimental_VideoModelV4FrameType;
+  }>;
+
+  /**
+   * Reference image inputs for reference-to-video generation.
+   */
+  inputReferences?: Array<DataContent>;
+
   /**
    * Whether the model should generate audio alongside the video.
    */
@@ -172,6 +199,55 @@ export async function experimental_generateVideo({
 
   const { prompt, image } = normalizePrompt(promptArg);
 
+  const normalizedFrameImages:
+    | Array<Experimental_VideoModelV4FrameImage>
+    | undefined = frameImages?.map(frame => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType,
+  }));
+
+  const normalizedInputReferences:
+    | Array<Experimental_VideoModelV4File>
+    | undefined = inputReferences?.map(reference =>
+    normalizeImageData(reference),
+  );
+
+  const effectiveInputReferences =
+    normalizedFrameImages != null && normalizedFrameImages.length > 0
+      ? undefined
+      : normalizedInputReferences;
+
+  const warnings: Array<Warning> = [];
+
+  if (
+    normalizedFrameImages != null &&
+    normalizedFrameImages.length > 0 &&
+    normalizedInputReferences != null &&
+    normalizedInputReferences.length > 0
+  ) {
+    warnings.push({
+      type: 'other',
+      message:
+        'inputReferences were ignored because frameImages were provided; ' +
+        'frameImages and inputReferences cannot be combined.',
+    });
+  }
+
+  const firstFrameImage = normalizedFrameImages?.find(
+    frame => frame.frameType === 'first_frame',
+  )?.image;
+
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: 'other',
+      message:
+        'prompt.image was ignored because a first_frame frameImage was provided; ' +
+        'the first_frame frameImage takes precedence as the start image.',
+    });
+  }
+
+  const resolvedImage = firstFrameImage ?? image;
+
   const maxVideosPerCallWithDefault =
     maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;
 
@@ -194,8 +270,10 @@ export async function experimental_generateVideo({
             duration,
             fps,
             seed,
+            image: resolvedImage,
+            frameImages: normalizedFrameImages,
+            inputReferences: effectiveInputReferences,
             generateAudio,
-            image,
             providerOptions: providerOptions ?? {},
             headers: headersWithUserAgent,
             abortSignal,
@@ -206,7 +284,6 @@ export async function experimental_generateVideo({
 
   // collect result videos, warnings, and response metadata
   const videos: Array<GeneratedFile> = [];
-  const warnings: Array<Warning> = [];
   const responses: Array<VideoModelResponseMetadata> = [];
   const providerMetadata: SharedV4ProviderMetadata = {};
 
@@ -342,59 +419,55 @@ function normalizePrompt(promptArg: GenerateVideoPrompt): {
     };
   }
 
-  let image: Experimental_VideoModelV4File | undefined;
-
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-
-    if (typeof dataContent === 'string') {
-      if (
-        dataContent.startsWith('http://') ||
-        dataContent.startsWith('https://')
-      ) {
-        image = {
-          type: 'url',
-          url: dataContent,
-        };
-      } else if (dataContent.startsWith('data:')) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: 'file',
-          mediaType: mediaType ?? 'image/png',
-          data: convertBase64ToUint8Array(base64Content ?? ''),
-        };
-      } else {
-        const bytes = convertBase64ToUint8Array(dataContent);
-        const mediaType =
-          detectMediaType({
-            data: bytes,
-            topLevelType: 'image',
-          }) ?? 'image/png';
-
-        image = {
-          type: 'file',
-          mediaType,
-          data: bytes,
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType =
-        detectMediaType({
-          data: dataContent,
-          topLevelType: 'image',
-        }) ?? 'image/png';
-
-      image = {
+  return {
+    prompt: promptArg.text,
+    image:
+      promptArg.image != null ? normalizeImageData(promptArg.image) : undefined,
+  };
+}
+
+/**
+ * Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV4File}.
+ * Accepts a URL string, a data URL, a base64 string, or binary image data.
+ */
+function normalizeImageData(
+  dataContent: DataContent,
+): Experimental_VideoModelV4File {
+  if (typeof dataContent === 'string') {
+    if (
+      dataContent.startsWith('http://') ||
+      dataContent.startsWith('https://')
+    ) {
+      return {
+        type: 'url',
+        url: dataContent,
+      };
+    }
+
+    if (dataContent.startsWith('data:')) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: 'file',
-        mediaType,
-        data: dataContent,
+        mediaType: mediaType ?? 'image/png',
+        data: convertBase64ToUint8Array(base64Content ?? ''),
       };
     }
+
+    const bytes = convertBase64ToUint8Array(dataContent);
+    return {
+      type: 'file',
+      mediaType:
+        detectMediaType({ data: bytes, topLevelType: 'image' }) ?? 'image/png',
+      data: bytes,
+    };
   }
 
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image,
+    type: 'file',
+    mediaType:
+      detectMediaType({ data: bytes, topLevelType: 'image' }) ?? 'image/png',
+    data: bytes,
   };
 }