@fondation-io/ai 7.0.0-beta.45

package/src/generate-video/generate-video.ts

@@ -0,0 +1,402 @@
+import type {
+  Experimental_VideoModelV4,
+  Experimental_VideoModelV4CallOptions,
+  Experimental_VideoModelV4File,
+  SharedV4ProviderMetadata,
+} from '@ai-sdk/provider';
+import {
+  convertBase64ToUint8Array,
+  type DataContent,
+  type ProviderOptions,
+  withUserAgentSuffix,
+} from '@ai-sdk/provider-utils';
+import { NoVideoGeneratedError } from '../error/no-video-generated-error';
+import {
+  DefaultGeneratedFile,
+  type GeneratedFile,
+} from '../generate-text/generated-file';
+import { logWarnings } from '../logger/log-warnings';
+import { resolveVideoModel } from '../model/resolve-model';
+import type { VideoModel } from '../types/video-model';
+import type { VideoModelResponseMetadata } from '../types/video-model-response-metadata';
+import type { Warning } from '../types/warning';
+import {
+  detectMediaType,
+  imageMediaTypeSignatures,
+  videoMediaTypeSignatures,
+} from '../util/detect-media-type';
+import { createDownload } from '../util/download/create-download';
+import { prepareRetries } from '../util/prepare-retries';
+import { VERSION } from '../version';
+import type { GenerateVideoResult } from './generate-video-result';
+import { splitDataUrl } from '../prompt/split-data-url';
+
+export type GenerateVideoPrompt =
+  | string
+  | {
+      image: DataContent;
+      text?: string;
+    };
+
+/**
+ * Generates videos using a video model.
+ *
+ * @param model - The video model to use.
+ * @param prompt - The prompt that should be used to generate the video.
+ * @param n - Number of videos to generate. Default: 1.
+ * @param aspectRatio - Aspect ratio of the videos to generate. Must have the format `{width}:{height}`.
+ * @param resolution - Resolution of the videos to generate. Must have the format `{width}x{height}`.
+ * @param duration - Duration of the video in seconds.
+ * @param fps - Frames per second for the video.
+ * @param seed - Seed for the video generation.
+ * @param providerOptions - Additional provider-specific options that are passed through to the provider
+ * as body parameters.
+ * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
+ * @param abortSignal - An optional abort signal that can be used to cancel the call.
+ * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ *
+ * @returns A result object that contains the generated videos.
+ */
+const defaultDownload = createDownload();
+
+export async function experimental_generateVideo({
+  model: modelArg,
+  prompt: promptArg,
+  n = 1,
+  maxVideosPerCall,
+  aspectRatio,
+  resolution,
+  duration,
+  fps,
+  seed,
+  providerOptions,
+  maxRetries: maxRetriesArg,
+  abortSignal,
+  headers,
+  download: downloadFn = defaultDownload,
+}: {
+  /**
+   * The video model to use.
+   */
+  model: VideoModel;
+
+  /**
+   * The prompt that should be used to generate the video.
+   */
+  prompt: GenerateVideoPrompt;
+
+  /**
+   * Number of videos to generate.
+   */
+  n?: number;
+
+  /**
+   * Maximum number of videos per API call. If not provided, the model's default will be used.
+   */
+  maxVideosPerCall?: number;
+
+  /**
+   * Aspect ratio of the videos to generate. Must have the format `{width}:{height}`.
+   */
+  aspectRatio?: `${number}:${number}`;
+
+  /**
+   * Resolution of the videos to generate. Must have the format `{width}x{height}`.
+   */
+  resolution?: `${number}x${number}`;
+
+  /**
+   * Duration of the video in seconds.
+   */
+  duration?: number;
+
+  /**
+   * Frames per second for the video.
+   */
+  fps?: number;
+
+  /**
+   * Seed for the video generation.
+   */
+  seed?: number;
+
+  /**
+   * Additional provider-specific options that are passed through to the provider
+   * as body parameters.
+   */
+  providerOptions?: ProviderOptions;
+
+  /**
+   * Maximum number of retries per video model call. Set to 0 to disable retries.
+   *
+   * @default 2
+   */
+  maxRetries?: number;
+
+  /**
+   * Abort signal.
+   */
+  abortSignal?: AbortSignal;
+
+  /**
+   * Additional headers to include in the request.
+   * Only applicable for HTTP-based providers.
+   */
+  headers?: Record<string, string>;
+
+  /**
+   * Custom download function for fetching videos from URLs.
+   * Use `createDownload()` from `ai` to create a download function with custom size limits.
+   *
+   * @default createDownload() (2 GiB limit)
+   */
+  download?: (options: {
+    url: URL;
+    abortSignal?: AbortSignal;
+  }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>;
+}): Promise<GenerateVideoResult> {
+  const model = resolveVideoModel(modelArg);
+
+  const headersWithUserAgent = withUserAgentSuffix(
+    headers ?? {},
+    `ai/${VERSION}`,
+  );
+
+  const { retry } = prepareRetries({
+    maxRetries: maxRetriesArg,
+    abortSignal,
+  });
+
+  const { prompt, image } = normalizePrompt(promptArg);
+
+  const maxVideosPerCallWithDefault =
+    maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;
+
+  // parallelize calls to the model:
+  const callCount = Math.ceil(n / maxVideosPerCallWithDefault);
+  const callVideoCounts = Array.from({ length: callCount }, (_, index) => {
+    const remaining = n - index * maxVideosPerCallWithDefault;
+    return Math.min(remaining, maxVideosPerCallWithDefault);
+  });
+
+  const results = await Promise.all(
+    callVideoCounts.map(async callVideoCount =>
+      retry(() =>
+        model.doGenerate({
+          prompt,
+          n: callVideoCount,
+          aspectRatio,
+          resolution,
+          duration,
+          fps,
+          seed,
+          image,
+          providerOptions: providerOptions ?? {},
+          headers: headersWithUserAgent,
+          abortSignal,
+        } satisfies Experimental_VideoModelV4CallOptions),
+      ),
+    ),
+  );
+
+  // collect result videos, warnings, and response metadata
+  const videos: Array<GeneratedFile> = [];
+  const warnings: Array<Warning> = [];
+  const responses: Array<VideoModelResponseMetadata> = [];
+  const providerMetadata: SharedV4ProviderMetadata = {};
+
+  for (const result of results) {
+    for (const videoData of result.videos) {
+      switch (videoData.type) {
+        case 'url': {
+          const { data, mediaType: downloadedMediaType } = await downloadFn({
+            url: new URL(videoData.url),
+            abortSignal,
+          });
+
+          // Filter out generic/unknown media types that should fall through to detection
+          const isUsableMediaType = (type: string | undefined): boolean =>
+            !!type && type !== 'application/octet-stream';
+
+          const mediaType =
+            (isUsableMediaType(videoData.mediaType) && videoData.mediaType) ||
+            (isUsableMediaType(downloadedMediaType) && downloadedMediaType) ||
+            detectMediaType({
+              data,
+              signatures: videoMediaTypeSignatures,
+            }) ||
+            'video/mp4';
+
+          videos.push(
+            new DefaultGeneratedFile({
+              data,
+              mediaType,
+            }),
+          );
+          break;
+        }
+
+        case 'base64': {
+          videos.push(
+            new DefaultGeneratedFile({
+              data: videoData.data,
+              mediaType: videoData.mediaType || 'video/mp4',
+            }),
+          );
+          break;
+        }
+
+        case 'binary': {
+          const mediaType =
+            videoData.mediaType ||
+            detectMediaType({
+              data: videoData.data,
+              signatures: videoMediaTypeSignatures,
+            }) ||
+            'video/mp4';
+
+          videos.push(
+            new DefaultGeneratedFile({
+              data: videoData.data,
+              mediaType,
+            }),
+          );
+          break;
+        }
+      }
+    }
+
+    warnings.push(...result.warnings);
+
+    responses.push({
+      timestamp: result.response.timestamp,
+      modelId: result.response.modelId,
+      headers: result.response.headers,
+      providerMetadata: result.providerMetadata,
+    });
+
+    if (result.providerMetadata != null) {
+      for (const [providerName, metadata] of Object.entries(
+        result.providerMetadata,
+      )) {
+        const existingMetadata = providerMetadata[providerName];
+        if (existingMetadata != null && typeof existingMetadata === 'object') {
+          providerMetadata[providerName] = {
+            ...existingMetadata,
+            ...metadata,
+          };
+
+          // Merge videos arrays if both exist
+          if (
+            'videos' in existingMetadata &&
+            Array.isArray(existingMetadata.videos) &&
+            'videos' in metadata &&
+            Array.isArray(metadata.videos)
+          ) {
+            (providerMetadata[providerName] as { videos: unknown[] }).videos = [
+              ...existingMetadata.videos,
+              ...metadata.videos,
+            ];
+          }
+        } else {
+          providerMetadata[providerName] = metadata;
+        }
+      }
+    }
+  }
+
+  if (videos.length === 0) {
+    throw new NoVideoGeneratedError({ responses });
+  }
+
+  if (warnings.length > 0) {
+    logWarnings({
+      warnings,
+      provider: model.provider,
+      model: model.modelId,
+    });
+  }
+
+  return {
+    video: videos[0],
+    videos,
+    warnings,
+    responses,
+    providerMetadata,
+  };
+}
+
+function normalizePrompt(promptArg: GenerateVideoPrompt): {
+  prompt: string | undefined;
+  image: Experimental_VideoModelV4File | undefined;
+} {
+  if (typeof promptArg === 'string') {
+    return {
+      prompt: promptArg,
+      image: undefined,
+    };
+  }
+
+  let image: Experimental_VideoModelV4File | undefined;
+
+  if (promptArg.image != null) {
+    const dataContent = promptArg.image;
+
+    if (typeof dataContent === 'string') {
+      if (
+        dataContent.startsWith('http://') ||
+        dataContent.startsWith('https://')
+      ) {
+        image = {
+          type: 'url',
+          url: dataContent,
+        };
+      } else if (dataContent.startsWith('data:')) {
+        const { mediaType, base64Content } = splitDataUrl(dataContent);
+        image = {
+          type: 'file',
+          mediaType: mediaType ?? 'image/png',
+          data: convertBase64ToUint8Array(base64Content ?? ''),
+        };
+      } else {
+        const bytes = convertBase64ToUint8Array(dataContent);
+        const mediaType =
+          detectMediaType({
+            data: bytes,
+            signatures: imageMediaTypeSignatures,
+          }) ?? 'image/png';
+
+        image = {
+          type: 'file',
+          mediaType,
+          data: bytes,
+        };
+      }
+    } else if (dataContent instanceof Uint8Array) {
+      const mediaType =
+        detectMediaType({
+          data: dataContent,
+          signatures: imageMediaTypeSignatures,
+        }) ?? 'image/png';
+
+      image = {
+        type: 'file',
+        mediaType,
+        data: dataContent,
+      };
+    }
+  }
+
+  return {
+    prompt: promptArg.text,
+    image,
+  };
+}
+
+async function invokeModelMaxVideosPerCall(model: Experimental_VideoModelV4) {
+  if (typeof model.maxVideosPerCall === 'function') {
+    return await model.maxVideosPerCall({ modelId: model.modelId });
+  }
+
+  return model.maxVideosPerCall;
+}

package/src/generate-video/index.ts

@@ -0,0 +1,3 @@
+export type { GenerateVideoPrompt } from './generate-video';
+export { experimental_generateVideo } from './generate-video';
+export type { GenerateVideoResult } from './generate-video-result';

package/src/global.ts

@@ -0,0 +1,36 @@
+import { ProviderV2, ProviderV3, ProviderV4 } from '@ai-sdk/provider';
+import { LogWarningsFunction } from './logger/log-warnings';
+import type { TelemetryIntegration } from './telemetry/telemetry-integration';
+
+// add AI SDK default provider to the globalThis object
+declare global {
+  /**
+   * The default provider to use for the AI SDK.
+   * String model ids are resolved to the default provider and model id.
+   *
+   * If not set, the default provider is the Vercel AI gateway provider.
+   *
+   * @see https://ai-sdk.dev/docs/ai-sdk-core/provider-management#global-provider-configuration
+   */
+  var AI_SDK_DEFAULT_PROVIDER: ProviderV4 | ProviderV3 | ProviderV2 | undefined;
+
+  /**
+   * The warning logger to use for the AI SDK.
+   *
+   * If not set, the default logger is the console.warn function.
+   *
+   * If set to false, no warnings are logged.
+   */
+  var AI_SDK_LOG_WARNINGS: LogWarningsFunction | undefined | false;
+
+  /**
+   * Globally registered telemetry integrations for the AI SDK.
+   *
+   * Integrations registered here receive lifecycle events (onStart, onStepStart,
+   * etc.) from every `generateText`, `streamText`, and similar call.
+   *
+   * Prefer using `registerTelemetryIntegration()` from `'ai'` instead of
+   * assigning this directly.
+   */
+  var AI_SDK_TELEMETRY_INTEGRATIONS: TelemetryIntegration[] | undefined;
+}

package/src/index.ts

@@ -0,0 +1,49 @@
+// re-exports:
+export { createGateway, gateway, type GatewayModelId } from '@ai-sdk/gateway';
+export {
+  asSchema,
+  createIdGenerator,
+  dynamicTool,
+  generateId,
+  jsonSchema,
+  parseJsonEventStream,
+  tool,
+  zodSchema,
+  type FlexibleSchema,
+  type IdGenerator,
+  type InferSchema,
+  type InferToolInput,
+  type InferToolOutput,
+  type Schema,
+  type Tool,
+  type ToolApprovalRequest,
+  type ToolApprovalResponse,
+  type ToolCallOptions,
+  type ToolExecutionOptions,
+  type ToolExecuteFunction,
+} from '@ai-sdk/provider-utils';
+
+// directory exports
+export * from './agent';
+export * from './embed';
+export * from './error';
+export * from './generate-image';
+export * from './generate-object';
+export * from './generate-speech';
+export * from './generate-text';
+export * from './generate-video';
+export * from './logger';
+export * from './middleware';
+export * from './prompt';
+export * from './registry';
+export * from './rerank';
+export * from './text-stream';
+export * from './transcribe';
+export * from './types';
+export * from './ui';
+export * from './ui-message-stream';
+export * from './util';
+export * from './telemetry';
+
+// import globals
+import './global';

package/src/logger/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @deprecated  Use `LogWarningsFunction` instead.
+ */
+export type { LogWarningsFunction as Experimental_LogWarningsFunction } from './log-warnings';
+
+export { type LogWarningsFunction } from './log-warnings';

package/src/logger/log-warnings.ts

@@ -0,0 +1,140 @@
+import { Warning } from '../types';
+
+/**
+ * A function for logging warnings.
+ *
+ * You can assign it to the `AI_SDK_LOG_WARNINGS` global variable to use it as the default warning logger.
+ *
+ * @example
+ * ```ts
+ * globalThis.AI_SDK_LOG_WARNINGS = (options) => {
+ *   console.log('WARNINGS:', options.warnings, options.provider, options.model);
+ * };
+ * ```
+ */
+export type LogWarningsFunction = (options: {
+  /**
+   * The warnings returned by the model provider.
+   */
+  warnings: Warning[];
+
+  /**
+   * The provider id used for the call.
+   */
+  provider: string;
+
+  /**
+   * The model id used for the call.
+   */
+  model: string;
+}) => void;
+
+/**
+ * Formats a warning object into a human-readable string with clear AI SDK branding.
+ *
+ * @param options - The options for formatting the warning.
+ * @param options.warning - The warning to format.
+ * @param options.provider - The provider id used for the call.
+ * @param options.model - The model id used for the call.
+ * @returns A formatted warning message string.
+ */
+function formatWarning({
+  warning,
+  provider,
+  model,
+}: {
+  warning: Warning;
+  provider: string;
+  model: string;
+}): string {
+  const prefix = `AI SDK Warning (${provider} / ${model}):`;
+
+  switch (warning.type) {
+    case 'unsupported': {
+      let message = `${prefix} The feature "${warning.feature}" is not supported.`;
+      if (warning.details) {
+        message += ` ${warning.details}`;
+      }
+      return message;
+    }
+
+    case 'compatibility': {
+      let message = `${prefix} The feature "${warning.feature}" is used in a compatibility mode.`;
+      if (warning.details) {
+        message += ` ${warning.details}`;
+      }
+      return message;
+    }
+
+    case 'other': {
+      return `${prefix} ${warning.message}`;
+    }
+
+    default: {
+      // Fallback for any unknown warning types
+      return `${prefix} ${JSON.stringify(warning, null, 2)}`;
+    }
+  }
+}
+
+export const FIRST_WARNING_INFO_MESSAGE =
+  'AI SDK Warning System: To turn off warning logging, set the AI_SDK_LOG_WARNINGS global to false.';
+
+let hasLoggedBefore = false;
+
+/**
+ * Logs warnings to the console or uses a custom logger if configured.
+ *
+ * The behavior can be customized via the `AI_SDK_LOG_WARNINGS` global variable:
+ * - If set to `false`, warnings are suppressed.
+ * - If set to a function, that function is called with the warnings.
+ * - Otherwise, warnings are logged to the console using `console.warn`.
+ *
+ * @param options - The options containing warnings and context.
+ * @param options.warnings - The warnings to log.
+ * @param options.provider - The provider id used for the call.
+ * @param options.model - The model id used for the call.
+ */
+export const logWarnings: LogWarningsFunction = options => {
+  // if the warnings array is empty, do nothing
+  if (options.warnings.length === 0) {
+    return;
+  }
+
+  const logger = globalThis.AI_SDK_LOG_WARNINGS;
+
+  // if the logger is set to false, do nothing
+  if (logger === false) {
+    return;
+  }
+
+  // use the provided logger if it is a function
+  if (typeof logger === 'function') {
+    logger(options);
+    return;
+  }
+
+  // display information note on first call
+  if (!hasLoggedBefore) {
+    hasLoggedBefore = true;
+    console.info(FIRST_WARNING_INFO_MESSAGE);
+  }
+
+  // default behavior: log warnings to the console
+  for (const warning of options.warnings) {
+    console.warn(
+      formatWarning({
+        warning,
+        provider: options.provider,
+        model: options.model,
+      }),
+    );
+  }
+};
+
+/**
+ * Resets the internal logging state. Used for testing purposes.
+ */
+export const resetLogWarningsState = () => {
+  hasLoggedBefore = false;
+};