ai 7.0.6 → 7.0.8

package/dist/internal/index.js

@@ -83,7 +83,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "7.0.6" : "0.0.0-test";
+var VERSION = true ? "7.0.8" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({

package/docs/03-ai-sdk-core/38-video-generation.mdx

@@ -171,6 +171,46 @@ const { video } = await generateVideo({
 });
 ```
 
+### First and Last Frame
+
+Some video models support first-last-frame generation, where you provide the
+starting and/or ending frames of the video. Use the `frameImages` option to pass
+role-tagged images in a provider-agnostic way:
+
+```tsx highlight={"5-16"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The cat walks across the scene and transforms into a dog by the end',
+  frameImages: [
+    {
+      image: 'https://example.com/first-frame.png',
+      frameType: 'first_frame',
+    },
+    {
+      image: 'https://example.com/last-frame.png',
+      frameType: 'last_frame',
+    },
+  ],
+});
+```
+
+### Reference Images
+
+Some video models support reference-to-video generation, where you provide one or
+more reference images that the model incorporates into the generated video. Use the `inputReferences` option to pass
+these images in a provider-agnostic way:
+
+```tsx highlight={"5-8"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The two characters meet in a bustling market',
+  inputReferences: [
+    'https://example.com/character-1.png',
+    'https://example.com/character-2.png',
+  ],
+});
+```
+
 ### Providing a Seed
 
 You can provide a seed to the `experimental_generateVideo` function to control the output of the video generation process.

package/docs/03-ai-sdk-core/40-middleware.mdx

@@ -271,15 +271,15 @@ Find more examples at this [link](https://github.com/minpeter/ai-sdk-tool-call-m
 <Note>
   Implementing language model middleware is advanced functionality and requires
   a solid understanding of the [language model
-  specification](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+  specification](https://github.com/vercel/ai/blob/main/packages/provider/src/language-model/v4/language-model-v4.ts).
 </Note>
 
 You can implement any of the following three function to modify the behavior of the language model:
 
 1. `transformParams`: Transforms the parameters before they are passed to the language model, for both `doGenerate` and `doStream`.
-2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/main/packages/provider/src/language-model/v4/language-model-v4.ts).
    You can modify the parameters, call the language model, and modify the result.
-3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/main/packages/provider/src/language-model/v4/language-model-v4.ts).
    You can modify the parameters, call the language model, and modify the result.
 
 Here are some examples of how to implement language model middleware:
@@ -307,9 +307,13 @@ export const yourLogMiddleware: LanguageModelV4Middleware = {
     console.log(`params: ${JSON.stringify(params, null, 2)}`);
 
     const result = await doGenerate();
+    const generatedText = result.content
+      .filter(part => part.type === 'text')
+      .map(part => part.text)
+      .join('');
 
     console.log('doGenerate finished');
-    console.log(`generated text: ${result.text}`);
+    console.log(`generated text: ${generatedText}`);
 
     return result;
   },
@@ -437,12 +441,16 @@ import type { LanguageModelV4Middleware } from '@ai-sdk/provider';
 
 export const yourGuardrailMiddleware: LanguageModelV4Middleware = {
   wrapGenerate: async ({ doGenerate }) => {
-    const { text, ...rest } = await doGenerate();
+    const result = await doGenerate();
 
     // filtering approach, e.g. for PII or other sensitive information:
-    const cleanedText = text?.replace(/badword/g, '<REDACTED>');
+    const content = result.content.map(part =>
+      part.type === 'text'
+        ? { ...part, text: part.text.replace(/badword/g, '<REDACTED>') }
+        : part,
+    );
 
-    return { text: cleanedText, ...rest };
+    return { ...result, content };
   },
 
   // here you would implement the guardrail logic for streaming

package/docs/04-ai-sdk-ui/03-chatbot-message-persistence.mdx

@@ -359,11 +359,20 @@ By default, message IDs are generated client-side:
 - AI response message IDs are generated by `streamText` on the server
 
 For applications without persistence, client-side ID generation works perfectly.
-However, **for persistence, you need server-side generated IDs** to ensure consistency across sessions and prevent ID conflicts when messages are stored and retrieved.
+However, **for persistence, you should use IDs that are stable before messages
+are stored** to ensure consistency across sessions and prevent ID conflicts when
+messages are restored.
+
+The server-side options below control IDs for generated assistant response
+messages. User message IDs are created by `useChat` before the request is sent
+to your API route, so keep those client-generated IDs when saving incoming
+request messages, or generate and persist your own user message IDs before
+sending/storing them.
 
 ### Setting Up Server-side ID Generation
 
-When implementing persistence, you have two options for generating server-side IDs:
+When implementing persistence, you have two options for generating server-side
+IDs for assistant response messages:
 
 1. **Using `generateMessageId` in `toUIMessageStream`**
 2. **Setting IDs in your start message part with `createUIMessageStream`**

package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx

@@ -110,6 +110,18 @@ console.log(videos);
       isOptional: true,
       description: 'Seed for the video generation.',
     },
+    {
+      name: 'frameImages',
+      type: 'Array<{ image: DataContent; frameType: "first_frame" | "last_frame" }>',
+      isOptional: true,
+      description: 'Role-tagged image inputs for first-last-frame generation.',
+    },
+    {
+      name: 'inputReferences',
+      type: 'Array<DataContent>',
+      isOptional: true,
+      description: 'Reference image inputs for reference-to-video generation.',
+    },
     {
       name: 'generateAudio',
       type: 'boolean',

package/docs/07-reference/01-ai-sdk-core/65-language-model-v2-middleware.mdx

@@ -28,8 +28,8 @@ See [Language Model Middleware](/docs/ai-sdk-core/middleware) for more informati
   content={[
     {
       name: 'specificationVersion',
-      type: "'v3'",
-      description: 'The specification version of the middleware. Must be "v3".',
+      type: "'v4'",
+      description: 'The specification version of the middleware. Must be "v4".',
     },
     {
       name: 'transformParams',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.6",
+  "version": "7.0.8",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,9 +42,9 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.5",
-    "@ai-sdk/provider-utils": "5.0.1",
-    "@ai-sdk/provider": "4.0.0"
+    "@ai-sdk/gateway": "4.0.6",
+    "@ai-sdk/provider": "4.0.1",
+    "@ai-sdk/provider-utils": "5.0.2"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",

package/src/generate-video/generate-video.ts

@@ -2,6 +2,8 @@ import type {
   Experimental_VideoModelV4,
   Experimental_VideoModelV4CallOptions,
   Experimental_VideoModelV4File,
+  Experimental_VideoModelV4FrameImage,
+  Experimental_VideoModelV4FrameType,
   SharedV4ProviderMetadata,
 } from '@ai-sdk/provider';
 import {
@@ -26,6 +28,7 @@ import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
 import type { GenerateVideoResult } from './generate-video-result';
 import { splitDataUrl } from '../prompt/split-data-url';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
 
 export type GenerateVideoPrompt =
   | string
@@ -45,6 +48,8 @@ export type GenerateVideoPrompt =
  * @param duration - Duration of the video in seconds.
  * @param fps - Frames per second for the video.
  * @param seed - Seed for the video generation.
+ * @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
+ * @param inputReferences - Reference image inputs for reference-to-video generation.
  * @param generateAudio - Whether the model should generate audio alongside the video.
  * @param providerOptions - Additional provider-specific options that are passed through to the provider
  * as body parameters.
@@ -66,6 +71,8 @@ export async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -118,6 +125,26 @@ export async function experimental_generateVideo({
    */
   seed?: number;
 
+  /**
+   * Role-tagged image inputs for image-to-video and first-last-frame generation.
+   */
+  frameImages?: Array<{
+    /**
+     * The image for this frame.
+     */
+    image: DataContent;
+
+    /**
+     * Which frame this image represents.
+     */
+    frameType: Experimental_VideoModelV4FrameType;
+  }>;
+
+  /**
+   * Reference image inputs for reference-to-video generation.
+   */
+  inputReferences?: Array<DataContent>;
+
   /**
    * Whether the model should generate audio alongside the video.
    */
@@ -172,6 +199,55 @@ export async function experimental_generateVideo({
 
   const { prompt, image } = normalizePrompt(promptArg);
 
+  const normalizedFrameImages:
+    | Array<Experimental_VideoModelV4FrameImage>
+    | undefined = frameImages?.map(frame => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType,
+  }));
+
+  const normalizedInputReferences:
+    | Array<Experimental_VideoModelV4File>
+    | undefined = inputReferences?.map(reference =>
+    normalizeImageData(reference),
+  );
+
+  const effectiveInputReferences =
+    normalizedFrameImages != null && normalizedFrameImages.length > 0
+      ? undefined
+      : normalizedInputReferences;
+
+  const warnings: Array<Warning> = [];
+
+  if (
+    normalizedFrameImages != null &&
+    normalizedFrameImages.length > 0 &&
+    normalizedInputReferences != null &&
+    normalizedInputReferences.length > 0
+  ) {
+    warnings.push({
+      type: 'other',
+      message:
+        'inputReferences were ignored because frameImages were provided; ' +
+        'frameImages and inputReferences cannot be combined.',
+    });
+  }
+
+  const firstFrameImage = normalizedFrameImages?.find(
+    frame => frame.frameType === 'first_frame',
+  )?.image;
+
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: 'other',
+      message:
+        'prompt.image was ignored because a first_frame frameImage was provided; ' +
+        'the first_frame frameImage takes precedence as the start image.',
+    });
+  }
+
+  const resolvedImage = firstFrameImage ?? image;
+
   const maxVideosPerCallWithDefault =
     maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;
 
@@ -194,8 +270,10 @@ export async function experimental_generateVideo({
             duration,
             fps,
             seed,
+            image: resolvedImage,
+            frameImages: normalizedFrameImages,
+            inputReferences: effectiveInputReferences,
             generateAudio,
-            image,
             providerOptions: providerOptions ?? {},
             headers: headersWithUserAgent,
             abortSignal,
@@ -206,7 +284,6 @@ export async function experimental_generateVideo({
 
   // collect result videos, warnings, and response metadata
   const videos: Array<GeneratedFile> = [];
-  const warnings: Array<Warning> = [];
   const responses: Array<VideoModelResponseMetadata> = [];
   const providerMetadata: SharedV4ProviderMetadata = {};
 
@@ -342,59 +419,55 @@ function normalizePrompt(promptArg: GenerateVideoPrompt): {
     };
   }
 
-  let image: Experimental_VideoModelV4File | undefined;
-
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-
-    if (typeof dataContent === 'string') {
-      if (
-        dataContent.startsWith('http://') ||
-        dataContent.startsWith('https://')
-      ) {
-        image = {
-          type: 'url',
-          url: dataContent,
-        };
-      } else if (dataContent.startsWith('data:')) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: 'file',
-          mediaType: mediaType ?? 'image/png',
-          data: convertBase64ToUint8Array(base64Content ?? ''),
-        };
-      } else {
-        const bytes = convertBase64ToUint8Array(dataContent);
-        const mediaType =
-          detectMediaType({
-            data: bytes,
-            topLevelType: 'image',
-          }) ?? 'image/png';
-
-        image = {
-          type: 'file',
-          mediaType,
-          data: bytes,
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType =
-        detectMediaType({
-          data: dataContent,
-          topLevelType: 'image',
-        }) ?? 'image/png';
-
-      image = {
+  return {
+    prompt: promptArg.text,
+    image:
+      promptArg.image != null ? normalizeImageData(promptArg.image) : undefined,
+  };
+}
+
+/**
+ * Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV4File}.
+ * Accepts a URL string, a data URL, a base64 string, or binary image data.
+ */
+function normalizeImageData(
+  dataContent: DataContent,
+): Experimental_VideoModelV4File {
+  if (typeof dataContent === 'string') {
+    if (
+      dataContent.startsWith('http://') ||
+      dataContent.startsWith('https://')
+    ) {
+      return {
+        type: 'url',
+        url: dataContent,
+      };
+    }
+
+    if (dataContent.startsWith('data:')) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: 'file',
-        mediaType,
-        data: dataContent,
+        mediaType: mediaType ?? 'image/png',
+        data: convertBase64ToUint8Array(base64Content ?? ''),
       };
     }
+
+    const bytes = convertBase64ToUint8Array(dataContent);
+    return {
+      type: 'file',
+      mediaType:
+        detectMediaType({ data: bytes, topLevelType: 'image' }) ?? 'image/png',
+      data: bytes,
+    };
   }
 
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image,
+    type: 'file',
+    mediaType:
+      detectMediaType({ data: bytes, topLevelType: 'image' }) ?? 'image/png',
+    data: bytes,
   };
 }
 

package/src/ui/convert-to-model-messages.ts

@@ -280,10 +280,12 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
               }
             }
 
-            modelMessages.push({
-              role: 'assistant',
-              content,
-            });
+            if (content.length > 0) {
+              modelMessages.push({
+                role: 'assistant',
+                content,
+              });
+            }
 
             // check if there are tool invocations with results in the block
             // Include non-provider-executed tools, OR provider-executed tools with approval responses