ai 7.0.7 → 7.0.8

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # ai
 
+## 7.0.8
+
+### Patch Changes
+
+- 0274f34: feat (video): add first-class `frameImages` and `inputReferences` call options for video generation
+- Updated dependencies [0274f34]
+  - @ai-sdk/provider@4.0.1
+  - @ai-sdk/gateway@4.0.6
+  - @ai-sdk/provider-utils@5.0.2
+
 ## 7.0.7
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { EmbeddingModelV4, EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV4Embedding, EmbeddingModelV4Middleware, ImageModelV4, ImageModelV3, ImageModelV2, ImageModelV4ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV4Middleware, JSONValue as JSONValue$1, LanguageModelV4, LanguageModelV3, LanguageModelV2, SharedV4Warning, LanguageModelV4Source, LanguageModelV4Middleware, RerankingModelV4, RerankingModelV3, SharedV4ProviderMetadata, SharedV4ProviderReference, SpeechModelV4, SpeechModelV3, SpeechModelV2, TranscriptionModelV4, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV4Usage, LanguageModelV4CallOptions, LanguageModelV4Prompt, AISDKError, LanguageModelV4ToolCall, JSONSchema7, ProviderV4, ProviderV3, ProviderV2, SharedV4Headers, JSONParseError, TypeValidationError, Experimental_VideoModelV4, Experimental_VideoModelV3, EmbeddingModelV4CallOptions, Experimental_RealtimeFactoryV4, Experimental_RealtimeFactoryV4GetTokenOptions, Experimental_RealtimeFactoryV4GetTokenResult, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4SessionConfig, Experimental_RealtimeModelV4ToolDefinition, FilesV4, SkillsV4, NoSuchModelError, FilesV4UploadFileCallOptions, SkillsV4UploadSkillResult, SkillsV4UploadSkillCallOptions, SkillsV4File } from '@ai-sdk/provider';
+import { EmbeddingModelV4, EmbeddingModelV3, EmbeddingModelV2, EmbeddingModelV4Embedding, EmbeddingModelV4Middleware, ImageModelV4, ImageModelV3, ImageModelV2, ImageModelV4ProviderMetadata, ImageModelV2ProviderMetadata, ImageModelV4Middleware, JSONValue as JSONValue$1, LanguageModelV4, LanguageModelV3, LanguageModelV2, SharedV4Warning, LanguageModelV4Source, LanguageModelV4Middleware, RerankingModelV4, RerankingModelV3, SharedV4ProviderMetadata, SharedV4ProviderReference, SpeechModelV4, SpeechModelV3, SpeechModelV2, TranscriptionModelV4, TranscriptionModelV3, TranscriptionModelV2, JSONObject, ImageModelV4Usage, LanguageModelV4CallOptions, LanguageModelV4Prompt, AISDKError, LanguageModelV4ToolCall, JSONSchema7, ProviderV4, ProviderV3, ProviderV2, SharedV4Headers, JSONParseError, TypeValidationError, Experimental_VideoModelV4, Experimental_VideoModelV3, Experimental_VideoModelV4FrameType, EmbeddingModelV4CallOptions, Experimental_RealtimeFactoryV4, Experimental_RealtimeFactoryV4GetTokenOptions, Experimental_RealtimeFactoryV4GetTokenResult, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4SessionConfig, Experimental_RealtimeModelV4ToolDefinition, FilesV4, SkillsV4, NoSuchModelError, FilesV4UploadFileCallOptions, SkillsV4UploadSkillResult, SkillsV4UploadSkillCallOptions, SkillsV4File } from '@ai-sdk/provider';
 export { AISDKError, APICallError, EmptyResponseBodyError, InvalidPromptError, InvalidResponseDataError, JSONParseError, JSONSchema7, LoadAPIKeyError, LoadSettingError, NoContentGeneratedError, NoSuchModelError, NoSuchProviderReferenceError, TooManyEmbeddingValuesForCallError, TypeValidationError, UnsupportedFunctionalityError } from '@ai-sdk/provider';
 import { GatewayModelId } from '@ai-sdk/gateway';
 export { GatewayModelId, createGateway, gateway } from '@ai-sdk/gateway';
@@ -7879,7 +7879,7 @@ type GenerateVideoPrompt = string | {
     image: DataContent;
     text?: string;
 };
-declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
+declare function experimental_generateVideo({ model: modelArg, prompt: promptArg, n, maxVideosPerCall, aspectRatio, resolution, duration, fps, seed, frameImages, inputReferences, generateAudio, providerOptions, maxRetries: maxRetriesArg, abortSignal, headers, download: downloadFn, }: {
     /**
      * The video model to use.
      */
@@ -7916,6 +7916,23 @@ declare function experimental_generateVideo({ model: modelArg, prompt: promptArg
      * Seed for the video generation.
      */
     seed?: number;
+    /**
+     * Role-tagged image inputs for image-to-video and first-last-frame generation.
+     */
+    frameImages?: Array<{
+        /**
+         * The image for this frame.
+         */
+        image: DataContent;
+        /**
+         * Which frame this image represents.
+         */
+        frameType: Experimental_VideoModelV4FrameType;
+    }>;
+    /**
+     * Reference image inputs for reference-to-video generation.
+     */
+    inputReferences?: Array<DataContent>;
     /**
      * Whether the model should generate audio alongside the video.
      */

package/dist/index.js

@@ -1089,7 +1089,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "7.0.7" : "0.0.0-test";
+var VERSION = true ? "7.0.8" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({
@@ -13429,6 +13429,8 @@ async function experimental_generateVideo({
   duration,
   fps,
   seed,
+  frameImages,
+  inputReferences,
   generateAudio,
   providerOptions,
   maxRetries: maxRetriesArg,
@@ -13436,7 +13438,7 @@ async function experimental_generateVideo({
   headers,
   download: downloadFn = defaultDownload
 }) {
-  var _a22;
+  var _a22, _b;
   const model = resolveVideoModel(modelArg);
   const headersWithUserAgent = withUserAgentSuffix9(
     headers != null ? headers : {},
@@ -13447,7 +13449,32 @@ async function experimental_generateVideo({
     abortSignal
   });
   const { prompt, image } = normalizePrompt2(promptArg);
-  const maxVideosPerCallWithDefault = (_a22 = maxVideosPerCall != null ? maxVideosPerCall : await invokeModelMaxVideosPerCall(model)) != null ? _a22 : 1;
+  const normalizedFrameImages = frameImages == null ? void 0 : frameImages.map((frame) => ({
+    image: normalizeImageData(frame.image),
+    frameType: frame.frameType
+  }));
+  const normalizedInputReferences = inputReferences == null ? void 0 : inputReferences.map(
+    (reference) => normalizeImageData(reference)
+  );
+  const effectiveInputReferences = normalizedFrameImages != null && normalizedFrameImages.length > 0 ? void 0 : normalizedInputReferences;
+  const warnings = [];
+  if (normalizedFrameImages != null && normalizedFrameImages.length > 0 && normalizedInputReferences != null && normalizedInputReferences.length > 0) {
+    warnings.push({
+      type: "other",
+      message: "inputReferences were ignored because frameImages were provided; frameImages and inputReferences cannot be combined."
+    });
+  }
+  const firstFrameImage = (_a22 = normalizedFrameImages == null ? void 0 : normalizedFrameImages.find(
+    (frame) => frame.frameType === "first_frame"
+  )) == null ? void 0 : _a22.image;
+  if (image != null && firstFrameImage != null) {
+    warnings.push({
+      type: "other",
+      message: "prompt.image was ignored because a first_frame frameImage was provided; the first_frame frameImage takes precedence as the start image."
+    });
+  }
+  const resolvedImage = firstFrameImage != null ? firstFrameImage : image;
+  const maxVideosPerCallWithDefault = (_b = maxVideosPerCall != null ? maxVideosPerCall : await invokeModelMaxVideosPerCall(model)) != null ? _b : 1;
   const callCount = Math.ceil(n / maxVideosPerCallWithDefault);
   const callVideoCounts = Array.from({ length: callCount }, (_, index) => {
     const remaining = n - index * maxVideosPerCallWithDefault;
@@ -13464,8 +13491,10 @@ async function experimental_generateVideo({
           duration,
           fps,
           seed,
+          image: resolvedImage,
+          frameImages: normalizedFrameImages,
+          inputReferences: effectiveInputReferences,
           generateAudio,
-          image,
           providerOptions: providerOptions != null ? providerOptions : {},
           headers: headersWithUserAgent,
           abortSignal
@@ -13474,7 +13503,6 @@ async function experimental_generateVideo({
     )
   );
   const videos = [];
-  const warnings = [];
   const responses = [];
   const providerMetadata = {};
   for (const result of results) {
@@ -13570,56 +13598,46 @@ async function experimental_generateVideo({
   };
 }
 function normalizePrompt2(promptArg) {
-  var _a22, _b;
   if (typeof promptArg === "string") {
     return {
       prompt: promptArg,
       image: void 0
     };
   }
-  let image;
-  if (promptArg.image != null) {
-    const dataContent = promptArg.image;
-    if (typeof dataContent === "string") {
-      if (dataContent.startsWith("http://") || dataContent.startsWith("https://")) {
-        image = {
-          type: "url",
-          url: dataContent
-        };
-      } else if (dataContent.startsWith("data:")) {
-        const { mediaType, base64Content } = splitDataUrl(dataContent);
-        image = {
-          type: "file",
-          mediaType: mediaType != null ? mediaType : "image/png",
-          data: convertBase64ToUint8Array5(base64Content != null ? base64Content : "")
-        };
-      } else {
-        const bytes = convertBase64ToUint8Array5(dataContent);
-        const mediaType = (_a22 = detectMediaType4({
-          data: bytes,
-          topLevelType: "image"
-        })) != null ? _a22 : "image/png";
-        image = {
-          type: "file",
-          mediaType,
-          data: bytes
-        };
-      }
-    } else if (dataContent instanceof Uint8Array) {
-      const mediaType = (_b = detectMediaType4({
-        data: dataContent,
-        topLevelType: "image"
-      })) != null ? _b : "image/png";
-      image = {
+  return {
+    prompt: promptArg.text,
+    image: promptArg.image != null ? normalizeImageData(promptArg.image) : void 0
+  };
+}
+function normalizeImageData(dataContent) {
+  var _a22, _b;
+  if (typeof dataContent === "string") {
+    if (dataContent.startsWith("http://") || dataContent.startsWith("https://")) {
+      return {
+        type: "url",
+        url: dataContent
+      };
+    }
+    if (dataContent.startsWith("data:")) {
+      const { mediaType, base64Content } = splitDataUrl(dataContent);
+      return {
         type: "file",
-        mediaType,
-        data: dataContent
+        mediaType: mediaType != null ? mediaType : "image/png",
+        data: convertBase64ToUint8Array5(base64Content != null ? base64Content : "")
       };
     }
+    const bytes2 = convertBase64ToUint8Array5(dataContent);
+    return {
+      type: "file",
+      mediaType: (_a22 = detectMediaType4({ data: bytes2, topLevelType: "image" })) != null ? _a22 : "image/png",
+      data: bytes2
+    };
   }
+  const bytes = convertDataContentToUint8Array(dataContent);
   return {
-    prompt: promptArg.text,
-    image
+    type: "file",
+    mediaType: (_b = detectMediaType4({ data: bytes, topLevelType: "image" })) != null ? _b : "image/png",
+    data: bytes
   };
 }
 async function invokeModelMaxVideosPerCall(model) {