ai 6.0.216 → 6.0.217

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # ai
 
+## 6.0.217
+
+### Patch Changes
+
+- fa850e6: feat (video): add first-class `frameImages` and `inputReferences` call options for video generation
+- Updated dependencies [95bc6fd]
+- Updated dependencies [f6b2159]
+- Updated dependencies [fa850e6]
+  - @ai-sdk/gateway@3.0.141
+  - @ai-sdk/provider@3.0.13
+  - @ai-sdk/provider-utils@4.0.34
+
 ## 6.0.216
 
 ### Patch Changes

package/dist/index.d.mts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV3Embedding, EmbeddingModelV3Middleware, ImageModelV3, ImageModelV2, ImageModelV3ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV3Middleware, JSONValue as JSONValue$1, LanguageModelV3, LanguageModelV2, SharedV3Warning, LanguageModelV3Source, LanguageModelV3Middleware, RerankingModelV3, SharedV3ProviderMetadata, SpeechModelV3, SpeechModelV2, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV3Usage, AISDKError, LanguageModelV3ToolCall, JSONSchema7, LanguageModelV3CallOptions, LanguageModelV3ToolChoice, ProviderV3, JSONParseError, TypeValidationError, Experimental_VideoModelV3, EmbeddingModelV3CallOptions, ProviderV2, NoSuchModelError } from '@ai-sdk/provider';
+import { EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV3Embedding, EmbeddingModelV3Middleware, ImageModelV3, ImageModelV2, ImageModelV3ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV3Middleware, JSONValue as JSONValue$1, LanguageModelV3, LanguageModelV2, SharedV3Warning, LanguageModelV3Source, LanguageModelV3Middleware, RerankingModelV3, SharedV3ProviderMetadata, SpeechModelV3, SpeechModelV2, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV3Usage, AISDKError, LanguageModelV3ToolCall, JSONSchema7, LanguageModelV3CallOptions, LanguageModelV3ToolChoice, ProviderV3, JSONParseError, TypeValidationError, Experimental_VideoModelV3, Experimental_VideoModelV3FrameType, EmbeddingModelV3CallOptions, ProviderV2, NoSuchModelError } from '@ai-sdk/provider';
 export { AISDKError, APICallError, EmptyResponseBodyError, InvalidPromptError, InvalidResponseDataError, JSONParseError, JSONSchema7, LoadAPIKeyError, LoadSettingError, NoContentGeneratedError, NoSuchModelError, TooManyEmbeddingValuesForCallError, TypeValidationError, UnsupportedFunctionalityError } from '@ai-sdk/provider';
 import { GatewayModelId } from '@ai-sdk/gateway';
 export { GatewayModelId, createGateway, gateway } from '@ai-sdk/gateway';
@@ -5863,7 +5863,7 @@ type GenerateVideoPrompt = string | {
     image: DataContent;
     text?: string;
 };
-declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
+declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, frameImages, inputReferences, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
     /**
      * The video model to use.
      */
@@ -5900,6 +5900,23 @@ declare function experimental_generateVideo({ model: modelArg, prompt: promptArg
      * Seed for the video generation.
      */
     seed?: number;
+    /**
+     * Role-tagged image inputs for image-to-video and first-last-frame generation.
+     */
+    frameImages?: Array<{
+        /**
+         * The image for this frame.
+         */
+        image: DataContent;
+        /**
+         * Which frame this image represents.
+         */
+        frameType: Experimental_VideoModelV3FrameType;
+    }>;
+    /**
+     * Reference image inputs for reference-to-video generation.
+     */
+    inputReferences?: Array<DataContent>;
     /**
      * Whether the model should generate audio alongside the video.
      */

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV3Embedding, EmbeddingModelV3Middleware, ImageModelV3, ImageModelV2, ImageModelV3ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV3Middleware, JSONValue as JSONValue$1, LanguageModelV3, LanguageModelV2, SharedV3Warning, LanguageModelV3Source, LanguageModelV3Middleware, RerankingModelV3, SharedV3ProviderMetadata, SpeechModelV3, SpeechModelV2, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV3Usage, AISDKError, LanguageModelV3ToolCall, JSONSchema7, LanguageModelV3CallOptions, LanguageModelV3ToolChoice, ProviderV3, JSONParseError, TypeValidationError, Experimental_VideoModelV3, EmbeddingModelV3CallOptions, ProviderV2, NoSuchModelError } from '@ai-sdk/provider';
+import { EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV3Embedding, EmbeddingModelV3Middleware, ImageModelV3, ImageModelV2, ImageModelV3ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV3Middleware, JSONValue as JSONValue$1, LanguageModelV3, LanguageModelV2, SharedV3Warning, LanguageModelV3Source, LanguageModelV3Middleware, RerankingModelV3, SharedV3ProviderMetadata, SpeechModelV3, SpeechModelV2, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV3Usage, AISDKError, LanguageModelV3ToolCall, JSONSchema7, LanguageModelV3CallOptions, LanguageModelV3ToolChoice, ProviderV3, JSONParseError, TypeValidationError, Experimental_VideoModelV3, Experimental_VideoModelV3FrameType, EmbeddingModelV3CallOptions, ProviderV2, NoSuchModelError } from '@ai-sdk/provider';
 export { AISDKError, APICallError, EmptyResponseBodyError, InvalidPromptError, InvalidResponseDataError, JSONParseError, JSONSchema7, LoadAPIKeyError, LoadSettingError, NoContentGeneratedError, NoSuchModelError, TooManyEmbeddingValuesForCallError, TypeValidationError, UnsupportedFunctionalityError } from '@ai-sdk/provider';
 import { GatewayModelId } from '@ai-sdk/gateway';
 export { GatewayModelId, createGateway, gateway } from '@ai-sdk/gateway';
@@ -5863,7 +5863,7 @@ type GenerateVideoPrompt = string | {
     image: DataContent;
     text?: string;
 };
-declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
+declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, frameImages, inputReferences, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
     /**
      * The video model to use.
      */
@@ -5900,6 +5900,23 @@ declare function experimental_generateVideo({ model: modelArg, prompt: promptArg
      * Seed for the video generation.
      */
     seed?: number;
+    /**
+     * Role-tagged image inputs for image-to-video and first-last-frame generation.
+     */
+    frameImages?: Array<{
+        /**
+         * The image for this frame.
+         */
+        image: DataContent;
+        /**
+         * Which frame this image represents.
+         */
+        frameType: Experimental_VideoModelV3FrameType;
+    }>;
+    /**
+     * Reference image inputs for reference-to-video generation.
+     */
+    inputReferences?: Array<DataContent>;
     /**
      * Whether the model should generate audio alongside the video.
      */

package/dist/index.js

@@ -1281,7 +1281,7 @@ function detectMediaType({
 var import_provider_utils3 = require("@ai-sdk/provider-utils");
 
 // src/version.ts
-var VERSION = true ? "6.0.216" : "0.0.0-test";
+var VERSION = true ? "6.0.217" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({
@@ -11892,6 +11892,8 @@ async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -11899,7 +11901,7 @@ async function experimental_generateVideo({
   headers,
   download: downloadFn = defaultDownload
 }) {
-  var _a22;
+  var _a22, _b;
   const model = resolveVideoModel(modelArg);
   const headersWithUserAgent = (0, import_provider_utils35.withUserAgentSuffix)(
     headers != null ? headers : {},
@@ -11910,7 +11912,32 @@ async function experimental_generateVideo({
     abortSignal
   });
   const { prompt, image } = normalizePrompt2(promptArg);
-  const maxVideosPerCallWithDefault = (_a22 = maxVideosPerCall != null ? maxVideosPerCall : await invokeModelMaxVideosPerCall(model)) != null ? _a22 : 1;
+  const normalizedFrameImages = frameImages == null ? void 0 : frameImages.map((frame) => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType
+  }));
+  const normalizedInputReferences = inputReferences == null ? void 0 : inputReferences.map(
+    (reference) => normalizeImageData(reference)
+  );
+  const effectiveInputReferences = normalizedFrameImages != null && normalizedFrameImages.length > 0 ? void 0 : normalizedInputReferences;
+  const warnings = [];
+  if (normalizedFrameImages != null && normalizedFrameImages.length > 0 && normalizedInputReferences != null && normalizedInputReferences.length > 0) {
+    warnings.push({
+      type: "other",
+      message: "inputReferences were ignored because frameImages were provided; frameImages and inputReferences cannot be combined."
+    });
+  }
+  const firstFrameImage = (_a22 = normalizedFrameImages == null ? void 0 : normalizedFrameImages.find(
+    (frame) => frame.frameType === "first_frame"
+  )) == null ? void 0 : _a22.image;
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: "other",
+      message: "prompt.image was ignored because a first_frame frameImage was provided; the first_frame frameImage takes precedence as the start image."
+    });
+  }
+  const resolvedImage = firstFrameImage != null ? firstFrameImage : image;
+  const maxVideosPerCallWithDefault = (_b = maxVideosPerCall != null ? maxVideosPerCall : await invokeModelMaxVideosPerCall(model)) != null ? _b : 1;
   const callCount = Math.ceil(n / maxVideosPerCallWithDefault);
   const callVideoCounts = Array.from({ length: callCount }, (_, index) => {
     const remaining = n - index * maxVideosPerCallWithDefault;
@@ -11918,7 +11945,7 @@ async function experimental_generateVideo({
   });
   const results = await Promise.all(
     callVideoCounts.map(
-      async (callVideoCount) => retry(
+      async (callVideoCount) => await retry(
         () => model.doGenerate({
           prompt,
           n: callVideoCount,
@@ -11927,8 +11954,10 @@ async function experimental_generateVideo({
           duration,
           fps,
           seed,
+          image: resolvedImage,
+          frameImages: normalizedFrameImages,
+          inputReferences: effectiveInputReferences,
           generateAudio,
-          image,
           providerOptions: providerOptions != null ? providerOptions : {},
           headers: headersWithUserAgent,
           abortSignal
@@ -11937,7 +11966,6 @@ async function experimental_generateVideo({
     )
   );
   const videos = [];
-  const warnings = [];
   const responses = [];
   const providerMetadata = {};
   for (const result of results) {
@@ -12033,56 +12061,49 @@ async function experimental_generateVideo({
   };
 }
 function normalizePrompt2(promptArg) {
-  var _a22, _b;
   if (typeof promptArg === "string") {
     return {
       prompt: promptArg,
       image: void 0
     };
   }
-  let image;
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-    if (typeof dataContent === "string") {
-      if (dataContent.startsWith("http://") || dataContent.startsWith("https://")) {
-        image = {
-          type: "url",
-          url: dataContent
-        };
-      } else if (dataContent.startsWith("data:")) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: "file",
-          mediaType: mediaType != null ? mediaType : "image/png",
-          data: (0, import_provider_utils35.convertBase64ToUint8Array)(base64Content != null ? base64Content : "")
-        };
-      } else {
-        const bytes = (0, import_provider_utils35.convertBase64ToUint8Array)(dataContent);
-        const mediaType = (_a22 = detectMediaType({
-          data: bytes,
-          signatures: imageMediaTypeSignatures
-        })) != null ? _a22 : "image/png";
-        image = {
-          type: "file",
-          mediaType,
-          data: bytes
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType = (_b = detectMediaType({
-        data: dataContent,
-        signatures: imageMediaTypeSignatures
-      })) != null ? _b : "image/png";
-      image = {
+  return {
+    prompt: promptArg.text,
+    image: promptArg.image != null ? normalizeImageData(promptArg.image) : void 0
+  };
+}
+function normalizeImageData(dataContent) {
+  var _a22, _b;
+  if (typeof dataContent === "string") {
+    if (dataContent.startsWith("http://") || dataContent.startsWith("https://")) {
+      return {
+        type: "url",
+        url: dataContent
+      };
+    }
+    if (dataContent.startsWith("data:")) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: "file",
-        mediaType,
-        data: dataContent
+        mediaType: mediaType != null ? mediaType : "image/png",
+        data: (0, import_provider_utils35.convertBase64ToUint8Array)(base64Content != null ? base64Content : "")
       };
     }
+    const bytes2 = (0, import_provider_utils35.convertBase64ToUint8Array)(dataContent);
+    return {
+      type: "file",
+      mediaType: (_a22 = detectMediaType({
+        data: bytes2,
+        signatures: imageMediaTypeSignatures
+      })) != null ? _a22 : "image/png",
+      data: bytes2
+    };
   }
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image
+    type: "file",
+    mediaType: (_b = detectMediaType({ data: bytes, signatures: imageMediaTypeSignatures })) != null ? _b : "image/png",
+    data: bytes
   };
 }
 async function invokeModelMaxVideosPerCall(model) {