@tanstack/ai 0.31.0 → 0.32.0

package/src/activities/generateImage/adapter.ts

@@ -1,4 +1,8 @@
-import type { ImageGenerationOptions, ImageGenerationResult } from '../../types'
+import type {
+  ImageGenerationOptions,
+  ImageGenerationResult,
+  ModelInputModalitiesByName,
+} from '../../types'
 
 /**
  * Resolve the size type for a model from the model-size map.
@@ -29,6 +33,8 @@ export interface ImageAdapterConfig {
  * - TProviderOptions: Base provider-specific options (already resolved)
  * - TModelProviderOptionsByName: Map from model name to its specific provider options
  * - TModelSizeByName: Map from model name to its supported sizes
+ * - TModelInputModalitiesByName: Map from model name to the non-text prompt
+ *   modalities it accepts (constrains the `prompt` part types at compile time)
  */
 export interface ImageAdapter<
   TModel extends string = string,
@@ -38,6 +44,8 @@ export interface ImageAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
 > {
   /** Discriminator for adapter kind - used by generate() to determine API shape */
   readonly kind: 'image'
@@ -53,6 +61,7 @@ export interface ImageAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
   }
 
   /**
@@ -67,7 +76,7 @@ export interface ImageAdapter<
  * An ImageAdapter with any/unknown type parameters.
  * Useful as a constraint in generic functions and interfaces.
  */
-export type AnyImageAdapter = ImageAdapter<any, any, any, any>
+export type AnyImageAdapter = ImageAdapter<any, any, any, any, any>
 
 /**
  * Abstract base class for image generation adapters.
@@ -83,11 +92,14 @@ export abstract class BaseImageAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
 > implements ImageAdapter<
   TModel,
   TProviderOptions,
   TModelProviderOptionsByName,
-  TModelSizeByName
+  TModelSizeByName,
+  TModelInputModalitiesByName
 > {
   readonly kind = 'image' as const
   abstract readonly name: string
@@ -98,6 +110,7 @@ export abstract class BaseImageAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
   }
 
   protected config: ImageAdapterConfig

package/src/activities/generateImage/index.ts

@@ -8,10 +8,16 @@
 import { aiEventClient } from '@tanstack/ai-event-client'
 import { streamGenerationResult } from '../stream-generation-result.js'
 import { resolveDebugOption } from '../../logger/resolve'
+import { resolveMediaPrompt } from '../../utilities/media-prompt'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
 import type { ImageAdapter } from './adapter'
-import type { ImageGenerationResult, StreamChunk } from '../../types'
+import type {
+  ImageGenerationResult,
+  MediaPrompt,
+  MediaPromptFor,
+  StreamChunk,
+} from '../../types'
 
 // ===========================
 // Activity Kind
@@ -55,6 +61,23 @@ export type ImageSizeForModel<TAdapter, TModel extends string> =
         : string
     : string
 
+/**
+ * Extract the prompt type a model accepts from an ImageAdapter via ~types.
+ * Adapters declare a per-model input-modality map; models in the map get a
+ * `prompt` narrowed to text + their supported part types (text-only models
+ * accept `string | Array<TextPart>`), so unsupported media parts fail at
+ * compile time. Adapters without a map fall back to the full MediaPrompt.
+ */
+export type ImagePromptForModel<TAdapter, TModel extends string> =
+  TAdapter extends ImageAdapter<any, any, any, any, infer ModsByName>
+    ? string extends keyof ModsByName
+      ? // No explicit map - accept the full union
+        MediaPrompt
+      : TModel extends keyof ModsByName
+        ? MediaPromptFor<ModsByName[TModel][number]>
+        : MediaPrompt
+    : MediaPrompt
+
 // ===========================
 // Activity Options Type
 // ===========================
@@ -72,8 +95,16 @@ export type ImageActivityOptions<
 > = {
   /** The image adapter to use (must be created with a model) */
   adapter: TAdapter & { kind: typeof kind }
-  /** Text description of the desired image(s) */
-  prompt: string
+  /**
+   * Description of the desired image(s). Either a plain string, or — for
+   * models that support image-conditioned generation — an ordered array of
+   * content parts interleaving text with image inputs (image-to-image,
+   * reference-guided, edit, multi-reference). Media parts may carry
+   * `metadata.role` (`'reference' | 'mask' | 'control' | 'character'`) to
+   * disambiguate intent. The accepted part types are narrowed per model via
+   * the adapter's input-modality map.
+   */
+  prompt: ImagePromptForModel<TAdapter, TAdapter['model']>
   /** Number of images to generate (default: 1) */
   numberOfImages?: number
   /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
@@ -203,13 +234,26 @@ async function runGenerateImage<
   const startTime = Date.now()
   const logger: InternalLogger = resolveDebugOption(options.debug)
 
+  // Devtools events carry the flattened prompt text plus media-part counts —
+  // the wire payload stays `prompt: string` regardless of the prompt shape.
+  const resolved = resolveMediaPrompt(rest.prompt)
+
   aiEventClient.emit('image:request:started', {
     requestId,
     provider: adapter.name,
     model,
-    prompt: rest.prompt,
+    prompt: resolved.text,
     numberOfImages: rest.numberOfImages,
     size: rest.size,
+    ...(resolved.images.length > 0 && {
+      imageInputCount: resolved.images.length,
+    }),
+    ...(resolved.videos.length > 0 && {
+      videoInputCount: resolved.videos.length,
+    }),
+    ...(resolved.audios.length > 0 && {
+      audioInputCount: resolved.audios.length,
+    }),
     modelOptions: rest.modelOptions,
     timestamp: startTime,
   })

package/src/activities/generateVideo/adapter.ts

@@ -1,10 +1,30 @@
 import type {
+  ModelInputModalitiesByName,
   VideoGenerationOptions,
   VideoJobResult,
   VideoStatusResult,
   VideoUrlResult,
 } from '../../types'
 
+/**
+ * Structured description of the durations a video model accepts.
+ *
+ * Tagged union so the same shape can express discrete enums (OpenAI Sora,
+ * Veo), continuous ranges, mixed shapes, and models with no duration field.
+ * Consumed by `VideoAdapter.availableDurations()`.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type DurationOptions<T extends string | number | undefined> =
+  | { kind: 'discrete'; values: ReadonlyArray<NonNullable<T>> }
+  | { kind: 'range'; min: number; max: number; step?: number; unit: 'seconds' }
+  | {
+      kind: 'mixed'
+      values: ReadonlyArray<NonNullable<T>>
+      range?: { min: number; max: number; step?: number }
+    }
+  | { kind: 'none' }
+
 /**
  * Configuration for video adapter instances
  *
@@ -31,6 +51,11 @@ export interface VideoAdapterConfig {
  * - TProviderOptions: Provider-specific options (already resolved)
  * - TModelProviderOptionsByName: Map from model name to its specific provider options
  * - TModelSizeByName: Map from model name to its supported sizes
+ * - TModelInputModalitiesByName: Map from model name to the non-text prompt
+ *   modalities it accepts (constrains the `prompt` part types at compile time)
+ * - TModelDurationByName: Map from model name to its supported duration
+ *   union. Defaults to `Record<string, number>` so adapters that haven't
+ *   declared a map keep today's `duration?: number` typing.
  */
 export interface VideoAdapter<
   TModel extends string = string,
@@ -40,6 +65,10 @@ export interface VideoAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
+  TModelDurationByName extends Record<string, string | number | undefined> =
+    Record<string, number>,
 > {
   /** Discriminator for adapter kind - used to determine API shape */
   readonly kind: 'video'
@@ -55,6 +84,8 @@ export interface VideoAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
+    modelDurationByName: TModelDurationByName
   }
 
   /**
@@ -62,7 +93,11 @@ export interface VideoAdapter<
    * Returns a job ID that can be used to poll for status and retrieve the video.
    */
   createVideoJob: (
-    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
+    options: VideoGenerationOptions<
+      TProviderOptions,
+      TModelSizeByName[TModel],
+      TModelDurationByName[TModel]
+    >,
   ) => Promise<VideoJobResult>
 
   /**
@@ -75,13 +110,26 @@ export interface VideoAdapter<
    * Should only be called after status is 'completed'.
    */
   getVideoUrl: (jobId: string) => Promise<VideoUrlResult>
+
+  /**
+   * Describe the durations this adapter's model accepts. Returns a tagged
+   * union so consumers can render UI / coerce input without provider-specific
+   * knowledge.
+   */
+  availableDurations: () => DurationOptions<TModelDurationByName[TModel]>
+
+  /**
+   * Coerce a raw seconds value to the closest valid duration for this model.
+   * Returns `undefined` for models with no duration field.
+   */
+  snapDuration: (seconds: number) => TModelDurationByName[TModel] | undefined
 }
 
 /**
  * A VideoAdapter with any/unknown type parameters.
  * Useful as a constraint in generic functions and interfaces.
  */
-export type AnyVideoAdapter = VideoAdapter<any, any, any, any>
+export type AnyVideoAdapter = VideoAdapter<any, any, any, any, any, any>
 
 /**
  * Abstract base class for video generation adapters.
@@ -99,11 +147,17 @@ export abstract class BaseVideoAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
+  TModelDurationByName extends Record<string, string | number | undefined> =
+    Record<string, number>,
 > implements VideoAdapter<
   TModel,
   TProviderOptions,
   TModelProviderOptionsByName,
-  TModelSizeByName
+  TModelSizeByName,
+  TModelInputModalitiesByName,
+  TModelDurationByName
 > {
   readonly kind = 'video' as const
   abstract readonly name: string
@@ -114,6 +168,8 @@ export abstract class BaseVideoAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
+    modelDurationByName: TModelDurationByName
   }
 
   protected config: VideoAdapterConfig
@@ -124,13 +180,33 @@ export abstract class BaseVideoAdapter<
   }
 
   abstract createVideoJob(
-    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
+    options: VideoGenerationOptions<
+      TProviderOptions,
+      TModelSizeByName[TModel],
+      TModelDurationByName[TModel]
+    >,
   ): Promise<VideoJobResult>
 
   abstract getVideoStatus(jobId: string): Promise<VideoStatusResult>
 
   abstract getVideoUrl(jobId: string): Promise<VideoUrlResult>
 
+  /**
+   * Default implementation returns `{ kind: 'none' }`. Adapters that have
+   * declared their per-model duration map should override this.
+   */
+  availableDurations(): DurationOptions<TModelDurationByName[TModel]> {
+    return { kind: 'none' }
+  }
+
+  /**
+   * Default implementation returns `undefined`. Adapters that have declared
+   * their per-model duration map should override.
+   */
+  snapDuration(_seconds: number): TModelDurationByName[TModel] | undefined {
+    return undefined
+  }
+
   protected generateId(): string {
     return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
   }

package/src/activities/generateVideo/index.ts

@@ -14,6 +14,8 @@ import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
 import type { VideoAdapter } from './adapter'
 import type {
+  MediaPrompt,
+  MediaPromptFor,
   StreamChunk,
   TokenUsage,
   VideoJobResult,
@@ -50,6 +52,40 @@ export type VideoSizeForAdapter<TAdapter> =
       : string
     : string
 
+/**
+ * Extract the prompt type a model accepts from a VideoAdapter via ~types.
+ * Mirrors `ImagePromptForModel`: models in the adapter's input-modality map
+ * get a `prompt` narrowed to text + their supported part types; adapters
+ * without a map fall back to the full MediaPrompt.
+ */
+export type VideoPromptForAdapter<TAdapter> =
+  TAdapter extends VideoAdapter<infer TModel, any, any, any, infer ModsByName>
+    ? string extends keyof ModsByName
+      ? MediaPrompt
+      : TModel extends keyof ModsByName
+        ? MediaPromptFor<ModsByName[TModel][number]>
+        : MediaPrompt
+    : MediaPrompt
+
+/**
+ * Extract the duration type for a VideoAdapter's model via ~types.
+ * Mirrors `VideoSizeForAdapter`. Falls back to `number` for adapters that
+ * haven't declared per-model duration constraints.
+ */
+export type VideoDurationForAdapter<TAdapter> =
+  TAdapter extends VideoAdapter<
+    infer TModel,
+    any,
+    any,
+    any,
+    any,
+    infer TDurationMap
+  >
+    ? TModel extends keyof TDurationMap
+      ? TDurationMap[TModel]
+      : number
+    : number
+
 // ===========================
 // Activity Options Types
 
@@ -84,12 +120,25 @@ export type VideoCreateOptions<
 > = VideoActivityBaseOptions<TAdapter> & {
   /** Request type - create a new job (default if not specified) */
   request?: 'create'
-  /** Text description of the desired video */
-  prompt: string
+  /**
+   * Description of the desired video. Either a plain string, or — for models
+   * that support image-conditioned generation — an ordered array of content
+   * parts interleaving text with image inputs. Image parts may carry
+   * `metadata.role` (`'start_frame' | 'end_frame' | 'reference' |
+   * 'character'`) to disambiguate intent; positional fallback otherwise. The
+   * accepted part types are narrowed per model via the adapter's
+   * input-modality map.
+   */
+  prompt: VideoPromptForAdapter<TAdapter>
   /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
   size?: VideoSizeForAdapter<TAdapter>
-  /** Video duration in seconds */
-  duration?: number
+  /**
+   * Video duration in seconds. Adapters that declare a per-model duration
+   * map narrow this to the model's valid union (e.g. `4 | 6 | 8` for Veo 3).
+   * Pass `adapter.snapDuration(seconds)` to coerce raw seconds to a valid
+   * value.
+   */
+  duration?: VideoDurationForAdapter<TAdapter>
   /**
    * Whether to stream the video generation lifecycle.
    * When true, returns an AsyncIterable<StreamChunk> that handles the full

package/src/activities/generateVideo/snap.ts

@@ -0,0 +1,100 @@
+import type { DurationOptions } from './adapter'
+
+/**
+ * Extract a numeric seconds value from a `DurationOptions` entry. Returns
+ * `null` for entries that don't parse as a number — e.g. `'auto'`.
+ *
+ * Handles the keyword-with-unit form FAL uses for Luma/Veo (`'8s'`, `'9s'`)
+ * by stripping a trailing `s`. Pure-numeric strings (`'5'`, `'10'`) parse via
+ * Number(). Numbers pass through.
+ */
+function entryToSeconds(entry: string | number): number | null {
+  if (typeof entry === 'number') {
+    return Number.isFinite(entry) ? entry : null
+  }
+  const stripped = entry.endsWith('s') ? entry.slice(0, -1) : entry
+  const parsed = Number(stripped)
+  return Number.isFinite(parsed) ? parsed : null
+}
+
+/**
+ * Snap a raw seconds value to the closest valid duration for a model's
+ * `DurationOptions`.
+ *
+ * - `none`            → `undefined`
+ * - `discrete`        → closest numeric-parseable entry; if none parse,
+ *                       returns `values[0]` (keyword-only models like 'auto')
+ * - `range`           → clamped to [min, max] and rounded to `step` (default 1)
+ * - `mixed`           → closest of (discrete numerics ∪ range values)
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export function snapToDurationOption<T extends string | number | undefined>(
+  seconds: number,
+  options: DurationOptions<T>,
+): T | undefined {
+  switch (options.kind) {
+    case 'none':
+      return undefined
+
+    case 'discrete': {
+      return pickClosestDiscrete(seconds, options.values)
+    }
+
+    case 'range': {
+      const step = options.step ?? 1
+      const clamped = Math.min(options.max, Math.max(options.min, seconds))
+      const snapped =
+        Math.round((clamped - options.min) / step) * step + options.min
+      return Math.min(options.max, Math.max(options.min, snapped)) as T
+    }
+
+    case 'mixed': {
+      const discreteCandidate = pickClosestDiscrete(seconds, options.values)
+      if (!options.range) return discreteCandidate
+
+      const { min, max, step = 1 } = options.range
+      const clamped = Math.min(max, Math.max(min, seconds))
+      const rangeValue = Math.min(
+        max,
+        Math.max(min, Math.round((clamped - min) / step) * step + min),
+      )
+
+      // Compare distance; range value is numeric, discrete may have non-numeric
+      // first-entry fallback (return distance Infinity for non-numerics).
+      const discreteSeconds =
+        typeof discreteCandidate === 'number'
+          ? discreteCandidate
+          : discreteCandidate !== undefined
+            ? (entryToSeconds(discreteCandidate) ?? Infinity)
+            : Infinity
+
+      return Math.abs(discreteSeconds - seconds) <=
+        Math.abs(rangeValue - seconds)
+        ? discreteCandidate
+        : (rangeValue as T)
+    }
+  }
+}
+
+function pickClosestDiscrete<T extends string | number>(
+  seconds: number,
+  values: ReadonlyArray<T>,
+): T | undefined {
+  if (values.length === 0) return undefined
+
+  let best: T | undefined
+  let bestDistance = Infinity
+  for (const value of values) {
+    const v = entryToSeconds(value)
+    if (v === null) continue
+    const distance = Math.abs(v - seconds)
+    if (distance < bestDistance) {
+      bestDistance = distance
+      best = value
+    }
+  }
+
+  // Keyword-only set (no numeric-parseable entries) — fall back to first entry.
+  return best ?? values[0]
+}

package/src/activities/index.ts

@@ -119,6 +119,7 @@ export {
   type VideoCreateOptions,
   type VideoStatusOptions,
   type VideoUrlOptions,
+  type VideoDurationForAdapter,
 } from './generateVideo/index'
 
 export {
@@ -126,8 +127,11 @@ export {
   type VideoAdapter,
   type VideoAdapterConfig,
   type AnyVideoAdapter,
+  type DurationOptions,
 } from './generateVideo/adapter'
 
+export { snapToDurationOption } from './generateVideo/snap'
+
 // ===========================
 // TTS Activity
 // ===========================

package/src/client.ts

@@ -97,6 +97,10 @@ export type {
   CustomEvent,
   DocumentPart,
   ImagePart,
+  MediaInputMetadata,
+  MediaInputRole,
+  MediaPrompt,
+  MediaPromptPart,
   MessagePart,
   ModelMessage,
   RunErrorEvent,

package/src/index.ts

@@ -138,6 +138,10 @@ export * from './types'
 // Usage utilities
 export { buildBaseUsage, type BaseUsageInput } from './utilities/usage'
 
+// Media-generation prompt resolution (used by image / video adapters)
+export { resolveMediaPrompt } from './utilities/media-prompt'
+export type { ResolvedMediaPrompt } from './utilities/media-prompt'
+
 // System prompts (type + normaliser used by adapters)
 export type { SystemPrompt, NormalizedSystemPrompt } from './system-prompts'
 export { normalizeSystemPrompts } from './system-prompts'