ai 6.0.216 → 6.0.218

package/docs/03-ai-sdk-core/38-video-generation.mdx

@@ -171,6 +171,46 @@ const { video } = await generateVideo({
 });
 ```
 
+### First and Last Frame
+
+Some video models support first-last-frame generation, where you provide the
+starting and/or ending frames of the video. Use the `frameImages` option to pass
+role-tagged images in a provider-agnostic way:
+
+```tsx highlight={"5-16"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The cat walks across the scene and transforms into a dog by the end',
+  frameImages: [
+    {
+      image: 'https://example.com/first-frame.png',
+      frameType: 'first_frame',
+    },
+    {
+      image: 'https://example.com/last-frame.png',
+      frameType: 'last_frame',
+    },
+  ],
+});
+```
+
+### Reference Images
+
+Some video models support reference-to-video generation, where you provide one or
+more reference images that the model incorporates into the generated video. Use the `inputReferences` option to pass
+these images in a provider-agnostic way:
+
+```tsx highlight={"5-8"}
+const { video } = await generateVideo({
+  model: __VIDEO_MODEL__,
+  prompt: 'The two characters meet in a bustling market',
+  inputReferences: [
+    'https://example.com/character-1.png',
+    'https://example.com/character-2.png',
+  ],
+});
+```
+
 ### Providing a Seed
 
 You can provide a seed to the `experimental_generateVideo` function to control the output of the video generation process.

package/docs/03-ai-sdk-core/40-middleware.mdx

@@ -271,15 +271,15 @@ Find more examples at this [link](https://github.com/minpeter/ai-sdk-tool-call-m
 <Note>
   Implementing language model middleware is advanced functionality and requires
   a solid understanding of the [language model
-  specification](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+  specification](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
 </Note>
 
 You can implement any of the following three function to modify the behavior of the language model:
 
 1. `transformParams`: Transforms the parameters before they are passed to the language model, for both `doGenerate` and `doStream`.
-2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+2. `wrapGenerate`: Wraps the `doGenerate` method of the [language model](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
    You can modify the parameters, call the language model, and modify the result.
-3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/v5/packages/provider/src/language-model/v2/language-model-v2.ts).
+3. `wrapStream`: Wraps the `doStream` method of the [language model](https://github.com/vercel/ai/blob/release-v6.0/packages/provider/src/language-model/v3/language-model-v3.ts).
    You can modify the parameters, call the language model, and modify the result.
 
 Here are some examples of how to implement language model middleware:

package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx

@@ -110,6 +110,18 @@ console.log(videos);
       isOptional: true,
       description: 'Seed for the video generation.',
     },
+    {
+      name: 'frameImages',
+      type: 'Array<{ image: DataContent; frameType: "first_frame" | "last_frame" }>',
+      isOptional: true,
+      description: 'Role-tagged image inputs for first-last-frame generation.',
+    },
+    {
+      name: 'inputReferences',
+      type: 'Array<DataContent>',
+      isOptional: true,
+      description: 'Reference image inputs for reference-to-video generation.',
+    },
     {
       name: 'generateAudio',
       type: 'boolean',

package/docs/07-reference/01-ai-sdk-core/23-create-mcp-client.mdx

@@ -146,6 +146,13 @@ It currently does not support accepting notifications from an MCP server, and cu
               isOptional: true,
               description: 'Handler for uncaught errors',
             },
+            {
+              name: 'maxRetries',
+              type: 'number',
+              isOptional: true,
+              description:
+                'Maximum number of retries for transient MCP tool call failures. Set to 0 to disable retries. Defaults to 0. Retries are opt-in and apply to tools/call requests.',
+            },
             {
               name: 'capabilities',
               type: 'ClientCapabilities',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "6.0.216",
+  "version": "6.0.218",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
-    "@ai-sdk/gateway": "3.0.140",
-    "@ai-sdk/provider": "3.0.12",
-    "@ai-sdk/provider-utils": "4.0.33"
+    "@ai-sdk/gateway": "3.0.142",
+    "@ai-sdk/provider": "3.0.13",
+    "@ai-sdk/provider-utils": "4.0.35"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",

package/src/generate-video/generate-video.ts

@@ -2,6 +2,8 @@ import type {
   Experimental_VideoModelV3,
   Experimental_VideoModelV3CallOptions,
   Experimental_VideoModelV3File,
+  Experimental_VideoModelV3FrameImage,
+  Experimental_VideoModelV3FrameType,
   SharedV3ProviderMetadata,
 } from '@ai-sdk/provider';
 import {
@@ -30,6 +32,7 @@ import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
 import type { GenerateVideoResult } from './generate-video-result';
 import { splitDataUrl } from '../prompt/split-data-url';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
 
 export type GenerateVideoPrompt =
   | string
@@ -49,6 +52,8 @@ export type GenerateVideoPrompt =
  * @param duration - Duration of the video in seconds.
  * @param fps - Frames per second for the video.
  * @param seed - Seed for the video generation.
+ * @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
+ * @param inputReferences - Reference image inputs for reference-to-video generation.
  * @param generateAudio - Whether the model should generate audio alongside the video.
  * @param providerOptions - Additional provider-specific options that are passed through to the provider
  * as body parameters.
@@ -70,6 +75,8 @@ export async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -122,6 +129,26 @@ export async function experimental_generateVideo({
    */
   seed?: number;
 
+  /**
+   * Role-tagged image inputs for image-to-video and first-last-frame generation.
+   */
+  frameImages?: Array<{
+    /**
+     * The image for this frame.
+     */
+    image: DataContent;
+
+    /**
+     * Which frame this image represents.
+     */
+    frameType: Experimental_VideoModelV3FrameType;
+  }>;
+
+  /**
+   * Reference image inputs for reference-to-video generation.
+   */
+  inputReferences?: Array<DataContent>;
+
   /**
    * Whether the model should generate audio alongside the video.
    */
@@ -176,6 +203,55 @@ export async function experimental_generateVideo({
 
   const { prompt, image } = normalizePrompt(promptArg);
 
+  const normalizedFrameImages:
+    | Array<Experimental_VideoModelV3FrameImage>
+    | undefined = frameImages?.map(frame => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType,
+  }));
+
+  const normalizedInputReferences:
+    | Array<Experimental_VideoModelV3File>
+    | undefined = inputReferences?.map(reference =>
+    normalizeImageData(reference),
+  );
+
+  const effectiveInputReferences =
+    normalizedFrameImages != null && normalizedFrameImages.length > 0
+      ? undefined
+      : normalizedInputReferences;
+
+  const warnings: Array<Warning> = [];
+
+  if (
+    normalizedFrameImages != null &&
+    normalizedFrameImages.length > 0 &&
+    normalizedInputReferences != null &&
+    normalizedInputReferences.length > 0
+  ) {
+    warnings.push({
+      type: 'other',
+      message:
+        'inputReferences were ignored because frameImages were provided; ' +
+        'frameImages and inputReferences cannot be combined.',
+    });
+  }
+
+  const firstFrameImage = normalizedFrameImages?.find(
+    frame => frame.frameType === 'first_frame',
+  )?.image;
+
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: 'other',
+      message:
+        'prompt.image was ignored because a first_frame frameImage was provided; ' +
+        'the first_frame frameImage takes precedence as the start image.',
+    });
+  }
+
+  const resolvedImage = firstFrameImage ?? image;
+
   const maxVideosPerCallWithDefault =
     maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;
 
@@ -187,29 +263,31 @@ export async function experimental_generateVideo({
   });
 
   const results = await Promise.all(
-    callVideoCounts.map(async callVideoCount =>
-      retry(() =>
-        model.doGenerate({
-          prompt,
-          n: callVideoCount,
-          aspectRatio,
-          resolution,
-          duration,
-          fps,
-          seed,
-          generateAudio,
-          image,
-          providerOptions: providerOptions ?? {},
-          headers: headersWithUserAgent,
-          abortSignal,
-        } satisfies Experimental_VideoModelV3CallOptions),
-      ),
+    callVideoCounts.map(
+      async callVideoCount =>
+        await retry(() =>
+          model.doGenerate({
+            prompt,
+            n: callVideoCount,
+            aspectRatio,
+            resolution,
+            duration,
+            fps,
+            seed,
+            image: resolvedImage,
+            frameImages: normalizedFrameImages,
+            inputReferences: effectiveInputReferences,
+            generateAudio,
+            providerOptions: providerOptions ?? {},
+            headers: headersWithUserAgent,
+            abortSignal,
+          } satisfies Experimental_VideoModelV3CallOptions),
+        ),
     ),
   );
 
   // collect result videos, warnings, and response metadata
   const videos: Array<GeneratedFile> = [];
-  const warnings: Array<Warning> = [];
   const responses: Array<VideoModelResponseMetadata> = [];
   const providerMetadata: SharedV3ProviderMetadata = {};
 
@@ -345,59 +423,59 @@ function normalizePrompt(promptArg: GenerateVideoPrompt): {
     };
   }
 
-  let image: Experimental_VideoModelV3File | undefined;
-
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-
-    if (typeof dataContent === 'string') {
-      if (
-        dataContent.startsWith('http://') ||
-        dataContent.startsWith('https://')
-      ) {
-        image = {
-          type: 'url',
-          url: dataContent,
-        };
-      } else if (dataContent.startsWith('data:')) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: 'file',
-          mediaType: mediaType ?? 'image/png',
-          data: convertBase64ToUint8Array(base64Content ?? ''),
-        };
-      } else {
-        const bytes = convertBase64ToUint8Array(dataContent);
-        const mediaType =
-          detectMediaType({
-            data: bytes,
-            signatures: imageMediaTypeSignatures,
-          }) ?? 'image/png';
-
-        image = {
-          type: 'file',
-          mediaType,
-          data: bytes,
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType =
-        detectMediaType({
-          data: dataContent,
-          signatures: imageMediaTypeSignatures,
-        }) ?? 'image/png';
+  return {
+    prompt: promptArg.text,
+    image:
+      promptArg.image != null ? normalizeImageData(promptArg.image) : undefined,
+  };
+}
+
+/**
+ * Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV3File}.
+ * Accepts a URL string, a data URL, a base64 string, or binary image data.
+ */
+function normalizeImageData(
+  dataContent: DataContent,
+): Experimental_VideoModelV3File {
+  if (typeof dataContent === 'string') {
+    if (
+      dataContent.startsWith('http://') ||
+      dataContent.startsWith('https://')
+    ) {
+      return {
+        type: 'url',
+        url: dataContent,
+      };
+    }
 
-      image = {
+    if (dataContent.startsWith('data:')) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: 'file',
-        mediaType,
-        data: dataContent,
+        mediaType: mediaType ?? 'image/png',
+        data: convertBase64ToUint8Array(base64Content ?? ''),
       };
     }
+
+    const bytes = convertBase64ToUint8Array(dataContent);
+    return {
+      type: 'file',
+      mediaType:
+        detectMediaType({
+          data: bytes,
+          signatures: imageMediaTypeSignatures,
+        }) ?? 'image/png',
+      data: bytes,
+    };
   }
 
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image,
+    type: 'file',
+    mediaType:
+      detectMediaType({ data: bytes, signatures: imageMediaTypeSignatures }) ??
+      'image/png',
+    data: bytes,
   };
 }
 

package/src/util/prepare-retries.ts

@@ -1,8 +1,6 @@
 import { InvalidArgumentError } from '../error/invalid-argument-error';
-import {
-  retryWithExponentialBackoffRespectingRetryHeaders,
-  type RetryFunction,
-} from '../util/retry-with-exponential-backoff';
+import type { RetryFunction } from '@ai-sdk/provider-utils';
+import { retryWithExponentialBackoffRespectingRetryHeaders } from '../util/retry-with-exponential-backoff';
 
 /**
  * Validate and prepare retries.

package/src/util/retry-with-exponential-backoff.ts

@@ -1,12 +1,11 @@
 import { APICallError } from '@ai-sdk/provider';
 import { GatewayError } from '@ai-sdk/gateway';
-import { delay, getErrorMessage, isAbortError } from '@ai-sdk/provider-utils';
+import {
+  retryWithExponentialBackoff,
+  type RetryFunction,
+} from '@ai-sdk/provider-utils';
 import { RetryError } from './retry-error';
 
-export type RetryFunction = <OUTPUT>(
-  fn: () => PromiseLike<OUTPUT>,
-) => PromiseLike<OUTPUT>;
-
 function getRetryDelayInMs({
   error,
   exponentialBackoffDelay,
@@ -62,98 +61,31 @@ function getRetryDelayInMs({
  * while respecting rate limit headers (retry-after-ms and retry-after) if they are provided and reasonable (0-60 seconds).
  * You can configure the maximum number of retries, the initial delay, and the backoff factor.
  */
-export const retryWithExponentialBackoffRespectingRetryHeaders =
-  ({
-    maxRetries = 2,
-    initialDelayInMs = 2000,
-    backoffFactor = 2,
-    abortSignal,
-  }: {
-    maxRetries?: number;
-    initialDelayInMs?: number;
-    backoffFactor?: number;
-    abortSignal?: AbortSignal;
-  } = {}): RetryFunction =>
-  async <OUTPUT>(f: () => PromiseLike<OUTPUT>) =>
-    _retryWithExponentialBackoff(f, {
-      maxRetries,
-      delayInMs: initialDelayInMs,
-      backoffFactor,
-      abortSignal,
-    });
-
-async function _retryWithExponentialBackoff<OUTPUT>(
-  f: () => PromiseLike<OUTPUT>,
-  {
+export const retryWithExponentialBackoffRespectingRetryHeaders = ({
+  maxRetries = 2,
+  initialDelayInMs = 2000,
+  backoffFactor = 2,
+  abortSignal,
+}: {
+  maxRetries?: number;
+  initialDelayInMs?: number;
+  backoffFactor?: number;
+  abortSignal?: AbortSignal;
+} = {}): RetryFunction =>
+  retryWithExponentialBackoff({
     maxRetries,
-    delayInMs,
+    initialDelayInMs,
     backoffFactor,
     abortSignal,
-  }: {
-    maxRetries: number;
-    delayInMs: number;
-    backoffFactor: number;
-    abortSignal: AbortSignal | undefined;
-  },
-  errors: unknown[] = [],
-): Promise<OUTPUT> {
-  try {
-    return await f();
-  } catch (error) {
-    if (isAbortError(error)) {
-      throw error; // don't retry when the request was aborted
-    }
-
-    if (maxRetries === 0) {
-      throw error; // don't wrap the error when retries are disabled
-    }
-
-    const errorMessage = getErrorMessage(error);
-    const newErrors = [...errors, error];
-    const tryNumber = newErrors.length;
-
-    if (tryNumber > maxRetries) {
-      throw new RetryError({
-        message: `Failed after ${tryNumber} attempts. Last error: ${errorMessage}`,
-        reason: 'maxRetriesExceeded',
-        errors: newErrors,
-      });
-    }
-
-    if (
+    shouldRetry: error =>
       error instanceof Error &&
       ((APICallError.isInstance(error) && error.isRetryable === true) ||
-        (GatewayError.isInstance(error) && error.isRetryable === true)) &&
-      tryNumber <= maxRetries
-    ) {
-      await delay(
-        getRetryDelayInMs({
-          error: error as APICallError | GatewayError,
-          exponentialBackoffDelay: delayInMs,
-        }),
-        { abortSignal },
-      );
-
-      return _retryWithExponentialBackoff(
-        f,
-        {
-          maxRetries,
-          delayInMs: backoffFactor * delayInMs,
-          backoffFactor,
-          abortSignal,
-        },
-        newErrors,
-      );
-    }
-
-    if (tryNumber === 1) {
-      throw error; // don't wrap the error when a non-retryable error occurs on the first try
-    }
-
-    throw new RetryError({
-      message: `Failed after ${tryNumber} attempts with non-retryable error: '${errorMessage}'`,
-      reason: 'errorNotRetryable',
-      errors: newErrors,
-    });
-  }
-}
+        (GatewayError.isInstance(error) && error.isRetryable === true)),
+    getDelayInMs: ({ error, exponentialBackoffDelay }) =>
+      getRetryDelayInMs({
+        error: error as APICallError | GatewayError,
+        exponentialBackoffDelay,
+      }),
+    createRetryError: ({ message, reason, errors }) =>
+      new RetryError({ message, reason, errors }),
+  });