ai 6.0.216 → 6.0.217

package/dist/internal/index.js

@@ -152,7 +152,7 @@ function detectMediaType({
 var import_provider_utils2 = require("@ai-sdk/provider-utils");
 
 // src/version.ts
-var VERSION = true ? "6.0.216" : "0.0.0-test";
+var VERSION = true ? "6.0.217" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({

package/dist/internal/index.mjs

@@ -132,7 +132,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "6.0.216" : "0.0.0-test";
+var VERSION = true ? "6.0.217" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({

package/docs/02-foundations/02-providers-and-models.mdx

@@ -133,6 +133,7 @@ Here are the capabilities of popular models:
 | [OpenAI](/providers/ai-sdk-providers/openai)       | `gpt-5.1`                                   | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
 | [OpenAI](/providers/ai-sdk-providers/openai)       | `gpt-5-codex`                               | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
 | [OpenAI](/providers/ai-sdk-providers/openai)       | `gpt-5-chat-latest`                         | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
+| [Anthropic](/providers/ai-sdk-providers/anthropic) | `claude-sonnet-5`                           | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
 | [Anthropic](/providers/ai-sdk-providers/anthropic) | `claude-fable-5`                            | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
 | [Anthropic](/providers/ai-sdk-providers/anthropic) | `claude-opus-4-8`                           | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
 | [Anthropic](/providers/ai-sdk-providers/anthropic) | `claude-opus-4-7`                           | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |

package/docs/03-ai-sdk-core/38-video-generation.mdx

@@ -171,6 +171,46 @@ const { video } = await generateVideo({
 });
 ```
 
+### First and Last Frame
+
+Some video models support first-last-frame generation, where you provide the
+starting and/or ending frames of the video. Use the `frameImages` option to pass
+role-tagged images in a provider-agnostic way:
+
+```tsx highlight={"5-16"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The cat walks across the scene and transforms into a dog by the end',
+  frameImages: [
+    {
+      image: 'https://example.com/first-frame.png',
+      frameType: 'first_frame',
+    },
+    {
+      image: 'https://example.com/last-frame.png',
+      frameType: 'last_frame',
+    },
+  ],
+});
+```
+
+### Reference Images
+
+Some video models support reference-to-video generation, where you provide one or
+more reference images that the model incorporates into the generated video. Use the `inputReferences` option to pass
+these images in a provider-agnostic way:
+
+```tsx highlight={"5-8"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The two characters meet in a bustling market',
+  inputReferences: [
+    'https://example.com/character-1.png',
+    'https://example.com/character-2.png',
+  ],
+});
+```
+
 ### Providing a Seed
 
 You can provide a seed to the `experimental_generateVideo` function to control the output of the video generation process.

package/docs/03-ai-sdk-core/40-middleware.mdx

@@ -271,15 +271,15 @@ Find more examples at this [link](https://github.com/minpeter/ai-sdk-tool-call-m
 <Note>
   Implementing language model middleware is advanced functionality and requires
   a solid understanding of the [language model
-  specification](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+  specification](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
 </Note>
 
 You can implement any of the following three function to modify the behavior of the language model:
 
 1. `transformParams`: Transforms the parameters before they are passed to the language model, for both `doGenerate` and `doStream`.
-2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
    You can modify the parameters, call the language model, and modify the result.
-3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
    You can modify the parameters, call the language model, and modify the result.
 
 Here are some examples of how to implement language model middleware:

package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx

@@ -110,6 +110,18 @@ console.log(videos);
       isOptional: true,
       description: 'Seed for the video generation.',
     },
+    {
+      name: 'frameImages',
+      type: 'Array<{ image: DataContent; frameType: "first_frame" | "last_frame" }>',
+      isOptional: true,
+      description: 'Role-tagged image inputs for first-last-frame generation.',
+    },
+    {
+      name: 'inputReferences',
+      type: 'Array<DataContent>',
+      isOptional: true,
+      description: 'Reference image inputs for reference-to-video generation.',
+    },
     {
       name: 'generateAudio',
       type: 'boolean',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "6.0.216",
+  "version": "6.0.217",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
-    "@ai-sdk/gateway": "3.0.140",
-    "@ai-sdk/provider": "3.0.12",
-    "@ai-sdk/provider-utils": "4.0.33"
+    "@ai-sdk/gateway": "3.0.141",
+    "@ai-sdk/provider": "3.0.13",
+    "@ai-sdk/provider-utils": "4.0.34"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",

package/src/generate-video/generate-video.ts

@@ -2,6 +2,8 @@ import type {
   Experimental_VideoModelV3,
   Experimental_VideoModelV3CallOptions,
   Experimental_VideoModelV3File,
+  Experimental_VideoModelV3FrameImage,
+  Experimental_VideoModelV3FrameType,
   SharedV3ProviderMetadata,
 } from '@ai-sdk/provider';
 import {
@@ -30,6 +32,7 @@ import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
 import type { GenerateVideoResult } from './generate-video-result';
 import { splitDataUrl } from '../prompt/split-data-url';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
 
 export type GenerateVideoPrompt =
   | string
@@ -49,6 +52,8 @@ export type GenerateVideoPrompt =
  * @param duration - Duration of the video in seconds.
  * @param fps - Frames per second for the video.
  * @param seed - Seed for the video generation.
+ * @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
+ * @param inputReferences - Reference image inputs for reference-to-video generation.
  * @param generateAudio - Whether the model should generate audio alongside the video.
  * @param providerOptions - Additional provider-specific options that are passed through to the provider
  * as body parameters.
@@ -70,6 +75,8 @@ export async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -122,6 +129,26 @@ export async function experimental_generateVideo({
    */
   seed?: number;
 
+  /**
+   * Role-tagged image inputs for image-to-video and first-last-frame generation.
+   */
+  frameImages?: Array<{
+    /**
+     * The image for this frame.
+     */
+    image: DataContent;
+
+    /**
+     * Which frame this image represents.
+     */
+    frameType: Experimental_VideoModelV3FrameType;
+  }>;
+
+  /**
+   * Reference image inputs for reference-to-video generation.
+   */
+  inputReferences?: Array<DataContent>;
+
   /**
    * Whether the model should generate audio alongside the video.
    */
@@ -176,6 +203,55 @@ export async function experimental_generateVideo({
 
   const { prompt, image } = normalizePrompt(promptArg);
 
+  const normalizedFrameImages:
+    | Array<Experimental_VideoModelV3FrameImage>
+    | undefined = frameImages?.map(frame => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType,
+  }));
+
+  const normalizedInputReferences:
+    | Array<Experimental_VideoModelV3File>
+    | undefined = inputReferences?.map(reference =>
+    normalizeImageData(reference),
+  );
+
+  const effectiveInputReferences =
+    normalizedFrameImages != null && normalizedFrameImages.length > 0
+      ? undefined
+      : normalizedInputReferences;
+
+  const warnings: Array<Warning> = [];
+
+  if (
+    normalizedFrameImages != null &&
+    normalizedFrameImages.length > 0 &&
+    normalizedInputReferences != null &&
+    normalizedInputReferences.length > 0
+  ) {
+    warnings.push({
+      type: 'other',
+      message:
+        'inputReferences were ignored because frameImages were provided; ' +
+        'frameImages and inputReferences cannot be combined.',
+    });
+  }
+
+  const firstFrameImage = normalizedFrameImages?.find(
+    frame => frame.frameType === 'first_frame',
+  )?.image;
+
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: 'other',
+      message:
+        'prompt.image was ignored because a first_frame frameImage was provided; ' +
+        'the first_frame frameImage takes precedence as the start image.',
+    });
+  }
+
+  const resolvedImage = firstFrameImage ?? image;
+
   const maxVideosPerCallWithDefault =
     maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;
 
@@ -187,29 +263,31 @@ export async function experimental_generateVideo({
   });
 
   const results = await Promise.all(
-    callVideoCounts.map(async callVideoCount =>
-      retry(() =>
-        model.doGenerate({
-          prompt,
-          n: callVideoCount,
-          aspectRatio,
-          resolution,
-          duration,
-          fps,
-          seed,
-          generateAudio,
-          image,
-          providerOptions: providerOptions ?? {},
-          headers: headersWithUserAgent,
-          abortSignal,
-        } satisfies Experimental_VideoModelV3CallOptions),
-      ),
+    callVideoCounts.map(
+      async callVideoCount =>
+        await retry(() =>
+          model.doGenerate({
+            prompt,
+            n: callVideoCount,
+            aspectRatio,
+            resolution,
+            duration,
+            fps,
+            seed,
+            image: resolvedImage,
+            frameImages: normalizedFrameImages,
+            inputReferences: effectiveInputReferences,
+            generateAudio,
+            providerOptions: providerOptions ?? {},
+            headers: headersWithUserAgent,
+            abortSignal,
+          } satisfies Experimental_VideoModelV3CallOptions),
+        ),
     ),
   );
 
   // collect result videos, warnings, and response metadata
   const videos: Array<GeneratedFile> = [];
-  const warnings: Array<Warning> = [];
   const responses: Array<VideoModelResponseMetadata> = [];
   const providerMetadata: SharedV3ProviderMetadata = {};
 
@@ -345,59 +423,59 @@ function normalizePrompt(promptArg: GenerateVideoPrompt): {
     };
   }
 
-  let image: Experimental_VideoModelV3File | undefined;
-
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-
-    if (typeof dataContent === 'string') {
-      if (
-        dataContent.startsWith('http://') ||
-        dataContent.startsWith('https://')
-      ) {
-        image = {
-          type: 'url',
-          url: dataContent,
-        };
-      } else if (dataContent.startsWith('data:')) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: 'file',
-          mediaType: mediaType ?? 'image/png',
-          data: convertBase64ToUint8Array(base64Content ?? ''),
-        };
-      } else {
-        const bytes = convertBase64ToUint8Array(dataContent);
-        const mediaType =
-          detectMediaType({
-            data: bytes,
-            signatures: imageMediaTypeSignatures,
-          }) ?? 'image/png';
-
-        image = {
-          type: 'file',
-          mediaType,
-          data: bytes,
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType =
-        detectMediaType({
-          data: dataContent,
-          signatures: imageMediaTypeSignatures,
-        }) ?? 'image/png';
+  return {
+    prompt: promptArg.text,
+    image:
+      promptArg.image != null ? normalizeImageData(promptArg.image) : undefined,
+  };
+}
+
+/**
+ * Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV3File}.
+ * Accepts a URL string, a data URL, a base64 string, or binary image data.
+ */
+function normalizeImageData(
+  dataContent: DataContent,
+): Experimental_VideoModelV3File {
+  if (typeof dataContent === 'string') {
+    if (
+      dataContent.startsWith('http://') ||
+      dataContent.startsWith('https://')
+    ) {
+      return {
+        type: 'url',
+        url: dataContent,
+      };
+    }
 
-      image = {
+    if (dataContent.startsWith('data:')) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: 'file',
-        mediaType,
-        data: dataContent,
+        mediaType: mediaType ?? 'image/png',
+        data: convertBase64ToUint8Array(base64Content ?? ''),
       };
     }
+
+    const bytes = convertBase64ToUint8Array(dataContent);
+    return {
+      type: 'file',
+      mediaType:
+        detectMediaType({
+          data: bytes,
+          signatures: imageMediaTypeSignatures,
+        }) ?? 'image/png',
+      data: bytes,
+    };
   }
 
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image,
+    type: 'file',
+    mediaType:
+      detectMediaType({ data: bytes, signatures: imageMediaTypeSignatures }) ??
+      'image/png',
+    data: bytes,
   };
 }