@tanstack/ai 0.31.0 → 0.32.0

package/dist/esm/activities/generateVideo/snap.js

@@ -0,0 +1,54 @@
+function entryToSeconds(entry) {
+  if (typeof entry === "number") {
+    return Number.isFinite(entry) ? entry : null;
+  }
+  const stripped = entry.endsWith("s") ? entry.slice(0, -1) : entry;
+  const parsed = Number(stripped);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+function snapToDurationOption(seconds, options) {
+  switch (options.kind) {
+    case "none":
+      return void 0;
+    case "discrete": {
+      return pickClosestDiscrete(seconds, options.values);
+    }
+    case "range": {
+      const step = options.step ?? 1;
+      const clamped = Math.min(options.max, Math.max(options.min, seconds));
+      const snapped = Math.round((clamped - options.min) / step) * step + options.min;
+      return Math.min(options.max, Math.max(options.min, snapped));
+    }
+    case "mixed": {
+      const discreteCandidate = pickClosestDiscrete(seconds, options.values);
+      if (!options.range) return discreteCandidate;
+      const { min, max, step = 1 } = options.range;
+      const clamped = Math.min(max, Math.max(min, seconds));
+      const rangeValue = Math.min(
+        max,
+        Math.max(min, Math.round((clamped - min) / step) * step + min)
+      );
+      const discreteSeconds = typeof discreteCandidate === "number" ? discreteCandidate : discreteCandidate !== void 0 ? entryToSeconds(discreteCandidate) ?? Infinity : Infinity;
+      return Math.abs(discreteSeconds - seconds) <= Math.abs(rangeValue - seconds) ? discreteCandidate : rangeValue;
+    }
+  }
+}
+function pickClosestDiscrete(seconds, values) {
+  if (values.length === 0) return void 0;
+  let best;
+  let bestDistance = Infinity;
+  for (const value of values) {
+    const v = entryToSeconds(value);
+    if (v === null) continue;
+    const distance = Math.abs(v - seconds);
+    if (distance < bestDistance) {
+      bestDistance = distance;
+      best = value;
+    }
+  }
+  return best ?? values[0];
+}
+export {
+  snapToDurationOption
+};
+//# sourceMappingURL=snap.js.map

package/dist/esm/activities/generateVideo/snap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snap.js","sources":["../../../../src/activities/generateVideo/snap.ts"],"sourcesContent":["import type { DurationOptions } from './adapter'\n\n/**\n * Extract a numeric seconds value from a `DurationOptions` entry. Returns\n * `null` for entries that don't parse as a number — e.g. `'auto'`.\n *\n * Handles the keyword-with-unit form FAL uses for Luma/Veo (`'8s'`, `'9s'`)\n * by stripping a trailing `s`. Pure-numeric strings (`'5'`, `'10'`) parse via\n * Number(). Numbers pass through.\n */\nfunction entryToSeconds(entry: string | number): number | null {\n  if (typeof entry === 'number') {\n    return Number.isFinite(entry) ? entry : null\n  }\n  const stripped = entry.endsWith('s') ? entry.slice(0, -1) : entry\n  const parsed = Number(stripped)\n  return Number.isFinite(parsed) ? parsed : null\n}\n\n/**\n * Snap a raw seconds value to the closest valid duration for a model's\n * `DurationOptions`.\n *\n * - `none`            → `undefined`\n * - `discrete`        → closest numeric-parseable entry; if none parse,\n *                       returns `values[0]` (keyword-only models like 'auto')\n * - `range`           → clamped to [min, max] and rounded to `step` (default 1)\n * - `mixed`           → closest of (discrete numerics ∪ range values)\n *\n * @experimental Video generation is an experimental feature and may change.\n */\nexport function snapToDurationOption<T extends string | number | undefined>(\n  seconds: number,\n  options: DurationOptions<T>,\n): T | undefined {\n  switch (options.kind) {\n    case 'none':\n      return undefined\n\n    case 'discrete': {\n      return pickClosestDiscrete(seconds, options.values)\n    }\n\n    case 'range': {\n      const step = options.step ?? 1\n      const clamped = Math.min(options.max, Math.max(options.min, seconds))\n      const snapped =\n        Math.round((clamped - options.min) / step) * step + options.min\n      return Math.min(options.max, Math.max(options.min, snapped)) as T\n    }\n\n    case 'mixed': {\n      const discreteCandidate = pickClosestDiscrete(seconds, options.values)\n      if (!options.range) return discreteCandidate\n\n      const { min, max, step = 1 } = options.range\n      const clamped = Math.min(max, Math.max(min, seconds))\n      const rangeValue = Math.min(\n        max,\n        Math.max(min, Math.round((clamped - min) / step) * step + min),\n      )\n\n      // Compare distance; range value is numeric, discrete may have non-numeric\n      // first-entry fallback (return distance Infinity for non-numerics).\n      const discreteSeconds =\n        typeof discreteCandidate === 'number'\n          ? discreteCandidate\n          : discreteCandidate !== undefined\n            ? (entryToSeconds(discreteCandidate) ?? Infinity)\n            : Infinity\n\n      return Math.abs(discreteSeconds - seconds) <=\n        Math.abs(rangeValue - seconds)\n        ? discreteCandidate\n        : (rangeValue as T)\n    }\n  }\n}\n\nfunction pickClosestDiscrete<T extends string | number>(\n  seconds: number,\n  values: ReadonlyArray<T>,\n): T | undefined {\n  if (values.length === 0) return undefined\n\n  let best: T | undefined\n  let bestDistance = Infinity\n  for (const value of values) {\n    const v = entryToSeconds(value)\n    if (v === null) continue\n    const distance = Math.abs(v - seconds)\n    if (distance < bestDistance) {\n      bestDistance = distance\n      best = value\n    }\n  }\n\n  // Keyword-only set (no numeric-parseable entries) — fall back to first entry.\n  return best ?? values[0]\n}\n"],"names":[],"mappings":"AAUA,SAAS,eAAe,OAAuC;AAC7D,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AAAA,EAC1C;AACA,QAAM,WAAW,MAAM,SAAS,GAAG,IAAI,MAAM,MAAM,GAAG,EAAE,IAAI;AAC5D,QAAM,SAAS,OAAO,QAAQ;AAC9B,SAAO,OAAO,SAAS,MAAM,IAAI,SAAS;AAC5C;AAcO,SAAS,qBACd,SACA,SACe;AACf,UAAQ,QAAQ,MAAA;AAAA,IACd,KAAK;AACH,aAAO;AAAA,IAET,KAAK,YAAY;AACf,aAAO,oBAAoB,SAAS,QAAQ,MAAM;AAAA,IACpD;AAAA,IAEA,KAAK,SAAS;AACZ,YAAM,OAAO,QAAQ,QAAQ;AAC7B,YAAM,UAAU,KAAK,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,OAAO,CAAC;AACpE,YAAM,UACJ,KAAK,OAAO,UAAU,QAAQ,OAAO,IAAI,IAAI,OAAO,QAAQ;AAC9D,aAAO,KAAK,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,OAAO,CAAC;AAAA,IAC7D;AAAA,IAEA,KAAK,SAAS;AACZ,YAAM,oBAAoB,oBAAoB,SAAS,QAAQ,MAAM;AACrE,UAAI,CAAC,QAAQ,MAAO,QAAO;AAE3B,YAAM,EAAE,KAAK,KAAK,OAAO,EAAA,IAAM,QAAQ;AACvC,YAAM,UAAU,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,OAAO,CAAC;AACpD,YAAM,aAAa,KAAK;AAAA,QACtB;AAAA,QACA,KAAK,IAAI,KAAK,KAAK,OAAO,UAAU,OAAO,IAAI,IAAI,OAAO,GAAG;AAAA,MAAA;AAK/D,YAAM,kBACJ,OAAO,sBAAsB,WACzB,oBACA,sBAAsB,SACnB,eAAe,iBAAiB,KAAK,WACtC;AAER,aAAO,KAAK,IAAI,kBAAkB,OAAO,KACvC,KAAK,IAAI,aAAa,OAAO,IAC3B,oBACC;AAAA,IACP;AAAA,EAAA;AAEJ;AAEA,SAAS,oBACP,SACA,QACe;AACf,MAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,MAAI;AACJ,MAAI,eAAe;AACnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,IAAI,eAAe,KAAK;AAC9B,QAAI,MAAM,KAAM;AAChB,UAAM,WAAW,KAAK,IAAI,IAAI,OAAO;AACrC,QAAI,WAAW,cAAc;AAC3B,qBAAe;AACf,aAAO;AAAA,IACT;AAAA,EACF;AAGA,SAAO,QAAQ,OAAO,CAAC;AACzB;"}

package/dist/esm/activities/index.d.ts

@@ -14,8 +14,9 @@ export { kind as imageKind, generateImage, type ImageActivityOptions, type Image
 export { BaseImageAdapter, type ImageAdapter, type ImageAdapterConfig, type AnyImageAdapter, } from './generateImage/adapter.js';
 export { kind as audioKind, generateAudio, type AudioActivityOptions, type AudioActivityResult, type AudioProviderOptions, } from './generateAudio/index.js';
 export { BaseAudioAdapter, type AudioAdapter, type AudioAdapterConfig, type AnyAudioAdapter, } from './generateAudio/adapter.js';
-export { kind as videoKind, generateVideo, getVideoJobStatus, type VideoActivityOptions, type VideoActivityResult, type VideoProviderOptions, type VideoCreateOptions, type VideoStatusOptions, type VideoUrlOptions, } from './generateVideo/index.js';
-export { BaseVideoAdapter, type VideoAdapter, type VideoAdapterConfig, type AnyVideoAdapter, } from './generateVideo/adapter.js';
+export { kind as videoKind, generateVideo, getVideoJobStatus, type VideoActivityOptions, type VideoActivityResult, type VideoProviderOptions, type VideoCreateOptions, type VideoStatusOptions, type VideoUrlOptions, type VideoDurationForAdapter, } from './generateVideo/index.js';
+export { BaseVideoAdapter, type VideoAdapter, type VideoAdapterConfig, type AnyVideoAdapter, type DurationOptions, } from './generateVideo/adapter.js';
+export { snapToDurationOption } from './generateVideo/snap.js';
 export { kind as ttsKind, generateSpeech, type TTSActivityOptions, type TTSActivityResult, type TTSProviderOptions, } from './generateSpeech/index.js';
 export { BaseTTSAdapter, type TTSAdapter, type TTSAdapterConfig, type AnyTTSAdapter, } from './generateSpeech/adapter.js';
 export { kind as transcriptionKind, generateTranscription, type TranscriptionActivityOptions, type TranscriptionActivityResult, type TranscriptionProviderOptions, } from './generateTranscription/index.js';

package/dist/esm/activities/index.js

@@ -9,6 +9,7 @@ import { kind as kind4, generateAudio } from "./generateAudio/index.js";
 import { BaseAudioAdapter } from "./generateAudio/adapter.js";
 import { generateVideo, getVideoJobStatus, kind as kind5 } from "./generateVideo/index.js";
 import { BaseVideoAdapter } from "./generateVideo/adapter.js";
+import { snapToDurationOption } from "./generateVideo/snap.js";
 import { generateSpeech, kind as kind6 } from "./generateSpeech/index.js";
 import { BaseTTSAdapter } from "./generateSpeech/adapter.js";
 import { generateTranscription, kind as kind7 } from "./generateTranscription/index.js";
@@ -31,6 +32,7 @@ export {
   generateVideo,
   getVideoJobStatus,
   kind3 as imageKind,
+  snapToDurationOption,
   summarize,
   kind2 as summarizeKind,
   kind as textKind,

package/dist/esm/activities/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;"}
+{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;"}

package/dist/esm/client.d.ts

@@ -40,5 +40,5 @@ export { BatchStrategy, CompositeStrategy, defaultJSONParser, ImmediateStrategy,
 export type { ChunkRecording, ChunkStrategy, InternalToolCallState, JSONParser, ProcessorResult, ProcessorState, StreamProcessorEvents, StreamProcessorOptions, ToolCallState, ToolResultState, } from './activities/chat/stream/index.js';
 export { uiMessagesToWire } from './utilities/ag-ui-wire.js';
 export type { WireMessage } from './utilities/ag-ui-wire.js';
-export type { AudioPart, ContentPart, ContentPartDataSource, ContentPartSource, ContentPartUrlSource, CustomEvent, DocumentPart, ImagePart, MessagePart, ModelMessage, RunErrorEvent, RunFinishedEvent, SchemaInput, StreamChunk, StructuredOutputPart, TextPart, ThinkingPart, ToolCall, ToolCallPart, ToolResultPart, UIMessage, VideoPart, InferSchemaType, } from './types.js';
+export type { AudioPart, ContentPart, ContentPartDataSource, ContentPartSource, ContentPartUrlSource, CustomEvent, DocumentPart, ImagePart, MediaInputMetadata, MediaInputRole, MediaPrompt, MediaPromptPart, MessagePart, ModelMessage, RunErrorEvent, RunFinishedEvent, SchemaInput, StreamChunk, StructuredOutputPart, TextPart, ThinkingPart, ToolCall, ToolCallPart, ToolResultPart, UIMessage, VideoPart, InferSchemaType, } from './types.js';
 export type { AudioVisualization, RealtimeError, RealtimeErrorCode, RealtimeEvent, RealtimeEventHandler, RealtimeEventPayloads, RealtimeMessage, RealtimeMessagePart, RealtimeMode, RealtimeSessionConfig, RealtimeStatus, RealtimeToken, RealtimeAudioPart, RealtimeImagePart, RealtimeTextPart, RealtimeToolCallPart, RealtimeToolResultPart, VADConfig, } from './realtime/types.js';

package/dist/esm/client.js.map

@@ -1 +1 @@
-{"version":3,"file":"client.js","sources":["../../src/client.ts"],"sourcesContent":["export enum EventType {\n  TEXT_MESSAGE_START = 'TEXT_MESSAGE_START',\n  TEXT_MESSAGE_CONTENT = 'TEXT_MESSAGE_CONTENT',\n  TEXT_MESSAGE_END = 'TEXT_MESSAGE_END',\n  TEXT_MESSAGE_CHUNK = 'TEXT_MESSAGE_CHUNK',\n  TOOL_CALL_START = 'TOOL_CALL_START',\n  TOOL_CALL_ARGS = 'TOOL_CALL_ARGS',\n  TOOL_CALL_END = 'TOOL_CALL_END',\n  TOOL_CALL_CHUNK = 'TOOL_CALL_CHUNK',\n  TOOL_CALL_RESULT = 'TOOL_CALL_RESULT',\n  THINKING_START = 'THINKING_START',\n  THINKING_END = 'THINKING_END',\n  THINKING_TEXT_MESSAGE_START = 'THINKING_TEXT_MESSAGE_START',\n  THINKING_TEXT_MESSAGE_CONTENT = 'THINKING_TEXT_MESSAGE_CONTENT',\n  THINKING_TEXT_MESSAGE_END = 'THINKING_TEXT_MESSAGE_END',\n  STATE_SNAPSHOT = 'STATE_SNAPSHOT',\n  STATE_DELTA = 'STATE_DELTA',\n  MESSAGES_SNAPSHOT = 'MESSAGES_SNAPSHOT',\n  ACTIVITY_SNAPSHOT = 'ACTIVITY_SNAPSHOT',\n  ACTIVITY_DELTA = 'ACTIVITY_DELTA',\n  RAW = 'RAW',\n  CUSTOM = 'CUSTOM',\n  RUN_STARTED = 'RUN_STARTED',\n  RUN_FINISHED = 'RUN_FINISHED',\n  RUN_ERROR = 'RUN_ERROR',\n  STEP_STARTED = 'STEP_STARTED',\n  STEP_FINISHED = 'STEP_FINISHED',\n  REASONING_START = 'REASONING_START',\n  REASONING_MESSAGE_START = 'REASONING_MESSAGE_START',\n  REASONING_MESSAGE_CONTENT = 'REASONING_MESSAGE_CONTENT',\n  REASONING_MESSAGE_END = 'REASONING_MESSAGE_END',\n  REASONING_MESSAGE_CHUNK = 'REASONING_MESSAGE_CHUNK',\n  REASONING_END = 'REASONING_END',\n  REASONING_ENCRYPTED_VALUE = 'REASONING_ENCRYPTED_VALUE',\n}\n\nexport {\n  toolDefinition,\n  type AnyClientTool,\n  type ClientTool,\n  type InferToolInput,\n  type InferToolName,\n  type InferToolOutput,\n  type ToolDefinition,\n  type ToolDefinitionConfig,\n  type ToolDefinitionInstance,\n} from './activities/chat/tools/tool-definition'\n\nexport {\n  convertSchemaToJsonSchema,\n  isStandardSchema,\n  parseWithStandardSchema,\n} from './activities/chat/tools/schema-converter'\n\nexport {\n  convertMessagesToModelMessages,\n  generateMessageId,\n  modelMessageToUIMessage,\n  modelMessagesToUIMessages,\n  normalizeToUIMessage,\n  uiMessageToModelMessages,\n} from './activities/chat/messages'\n\nexport {\n  BatchStrategy,\n  CompositeStrategy,\n  defaultJSONParser,\n  ImmediateStrategy,\n  parsePartialJSON,\n  PartialJSONParser,\n  PunctuationStrategy,\n  StreamProcessor,\n  WordBoundaryStrategy,\n} from './activities/chat/stream/index'\nexport type {\n  ChunkRecording,\n  ChunkStrategy,\n  InternalToolCallState,\n  JSONParser,\n  ProcessorResult,\n  ProcessorState,\n  StreamProcessorEvents,\n  StreamProcessorOptions,\n  ToolCallState,\n  ToolResultState,\n} from './activities/chat/stream/index'\n\nexport { uiMessagesToWire } from './utilities/ag-ui-wire'\nexport type { WireMessage } from './utilities/ag-ui-wire'\n\nexport type {\n  AudioPart,\n  ContentPart,\n  ContentPartDataSource,\n  ContentPartSource,\n  ContentPartUrlSource,\n  CustomEvent,\n  DocumentPart,\n  ImagePart,\n  MessagePart,\n  ModelMessage,\n  RunErrorEvent,\n  RunFinishedEvent,\n  SchemaInput,\n  StreamChunk,\n  StructuredOutputPart,\n  TextPart,\n  ThinkingPart,\n  ToolCall,\n  ToolCallPart,\n  ToolResultPart,\n  UIMessage,\n  VideoPart,\n  InferSchemaType,\n} from './types'\n\nexport type {\n  AudioVisualization,\n  RealtimeError,\n  RealtimeErrorCode,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeEventPayloads,\n  RealtimeMessage,\n  RealtimeMessagePart,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeStatus,\n  RealtimeToken,\n  RealtimeAudioPart,\n  RealtimeImagePart,\n  RealtimeTextPart,\n  RealtimeToolCallPart,\n  RealtimeToolResultPart,\n  VADConfig,\n} from './realtime/types'\n"],"names":["EventType"],"mappings":";;;;;;;AAAO,IAAK,8BAAAA,eAAL;AACLA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,sBAAA,IAAuB;AACvBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,cAAA,IAAe;AACfA,aAAA,6BAAA,IAA8B;AAC9BA,aAAA,+BAAA,IAAgC;AAChCA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,aAAA,IAAc;AACdA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,KAAA,IAAM;AACNA,aAAA,QAAA,IAAS;AACTA,aAAA,aAAA,IAAc;AACdA,aAAA,cAAA,IAAe;AACfA,aAAA,WAAA,IAAY;AACZA,aAAA,cAAA,IAAe;AACfA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,uBAAA,IAAwB;AACxBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,eAAA,IAAgB;AAChBA,aAAA,2BAAA,IAA4B;AAjClB,SAAAA;AAAA,GAAA,aAAA,CAAA,CAAA;"}
+{"version":3,"file":"client.js","sources":["../../src/client.ts"],"sourcesContent":["export enum EventType {\n  TEXT_MESSAGE_START = 'TEXT_MESSAGE_START',\n  TEXT_MESSAGE_CONTENT = 'TEXT_MESSAGE_CONTENT',\n  TEXT_MESSAGE_END = 'TEXT_MESSAGE_END',\n  TEXT_MESSAGE_CHUNK = 'TEXT_MESSAGE_CHUNK',\n  TOOL_CALL_START = 'TOOL_CALL_START',\n  TOOL_CALL_ARGS = 'TOOL_CALL_ARGS',\n  TOOL_CALL_END = 'TOOL_CALL_END',\n  TOOL_CALL_CHUNK = 'TOOL_CALL_CHUNK',\n  TOOL_CALL_RESULT = 'TOOL_CALL_RESULT',\n  THINKING_START = 'THINKING_START',\n  THINKING_END = 'THINKING_END',\n  THINKING_TEXT_MESSAGE_START = 'THINKING_TEXT_MESSAGE_START',\n  THINKING_TEXT_MESSAGE_CONTENT = 'THINKING_TEXT_MESSAGE_CONTENT',\n  THINKING_TEXT_MESSAGE_END = 'THINKING_TEXT_MESSAGE_END',\n  STATE_SNAPSHOT = 'STATE_SNAPSHOT',\n  STATE_DELTA = 'STATE_DELTA',\n  MESSAGES_SNAPSHOT = 'MESSAGES_SNAPSHOT',\n  ACTIVITY_SNAPSHOT = 'ACTIVITY_SNAPSHOT',\n  ACTIVITY_DELTA = 'ACTIVITY_DELTA',\n  RAW = 'RAW',\n  CUSTOM = 'CUSTOM',\n  RUN_STARTED = 'RUN_STARTED',\n  RUN_FINISHED = 'RUN_FINISHED',\n  RUN_ERROR = 'RUN_ERROR',\n  STEP_STARTED = 'STEP_STARTED',\n  STEP_FINISHED = 'STEP_FINISHED',\n  REASONING_START = 'REASONING_START',\n  REASONING_MESSAGE_START = 'REASONING_MESSAGE_START',\n  REASONING_MESSAGE_CONTENT = 'REASONING_MESSAGE_CONTENT',\n  REASONING_MESSAGE_END = 'REASONING_MESSAGE_END',\n  REASONING_MESSAGE_CHUNK = 'REASONING_MESSAGE_CHUNK',\n  REASONING_END = 'REASONING_END',\n  REASONING_ENCRYPTED_VALUE = 'REASONING_ENCRYPTED_VALUE',\n}\n\nexport {\n  toolDefinition,\n  type AnyClientTool,\n  type ClientTool,\n  type InferToolInput,\n  type InferToolName,\n  type InferToolOutput,\n  type ToolDefinition,\n  type ToolDefinitionConfig,\n  type ToolDefinitionInstance,\n} from './activities/chat/tools/tool-definition'\n\nexport {\n  convertSchemaToJsonSchema,\n  isStandardSchema,\n  parseWithStandardSchema,\n} from './activities/chat/tools/schema-converter'\n\nexport {\n  convertMessagesToModelMessages,\n  generateMessageId,\n  modelMessageToUIMessage,\n  modelMessagesToUIMessages,\n  normalizeToUIMessage,\n  uiMessageToModelMessages,\n} from './activities/chat/messages'\n\nexport {\n  BatchStrategy,\n  CompositeStrategy,\n  defaultJSONParser,\n  ImmediateStrategy,\n  parsePartialJSON,\n  PartialJSONParser,\n  PunctuationStrategy,\n  StreamProcessor,\n  WordBoundaryStrategy,\n} from './activities/chat/stream/index'\nexport type {\n  ChunkRecording,\n  ChunkStrategy,\n  InternalToolCallState,\n  JSONParser,\n  ProcessorResult,\n  ProcessorState,\n  StreamProcessorEvents,\n  StreamProcessorOptions,\n  ToolCallState,\n  ToolResultState,\n} from './activities/chat/stream/index'\n\nexport { uiMessagesToWire } from './utilities/ag-ui-wire'\nexport type { WireMessage } from './utilities/ag-ui-wire'\n\nexport type {\n  AudioPart,\n  ContentPart,\n  ContentPartDataSource,\n  ContentPartSource,\n  ContentPartUrlSource,\n  CustomEvent,\n  DocumentPart,\n  ImagePart,\n  MediaInputMetadata,\n  MediaInputRole,\n  MediaPrompt,\n  MediaPromptPart,\n  MessagePart,\n  ModelMessage,\n  RunErrorEvent,\n  RunFinishedEvent,\n  SchemaInput,\n  StreamChunk,\n  StructuredOutputPart,\n  TextPart,\n  ThinkingPart,\n  ToolCall,\n  ToolCallPart,\n  ToolResultPart,\n  UIMessage,\n  VideoPart,\n  InferSchemaType,\n} from './types'\n\nexport type {\n  AudioVisualization,\n  RealtimeError,\n  RealtimeErrorCode,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeEventPayloads,\n  RealtimeMessage,\n  RealtimeMessagePart,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeStatus,\n  RealtimeToken,\n  RealtimeAudioPart,\n  RealtimeImagePart,\n  RealtimeTextPart,\n  RealtimeToolCallPart,\n  RealtimeToolResultPart,\n  VADConfig,\n} from './realtime/types'\n"],"names":["EventType"],"mappings":";;;;;;;AAAO,IAAK,8BAAAA,eAAL;AACLA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,sBAAA,IAAuB;AACvBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,cAAA,IAAe;AACfA,aAAA,6BAAA,IAA8B;AAC9BA,aAAA,+BAAA,IAAgC;AAChCA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,aAAA,IAAc;AACdA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,KAAA,IAAM;AACNA,aAAA,QAAA,IAAS;AACTA,aAAA,aAAA,IAAc;AACdA,aAAA,cAAA,IAAe;AACfA,aAAA,WAAA,IAAY;AACZA,aAAA,cAAA,IAAe;AACfA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,uBAAA,IAAwB;AACxBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,eAAA,IAAgB;AAChBA,aAAA,2BAAA,IAA4B;AAjClB,SAAAA;AAAA,GAAA,aAAA,CAAA,CAAA;"}

package/dist/esm/index.d.ts

@@ -22,6 +22,8 @@ export { createCapability, defineChatMiddleware, createChatMiddleware, } from '.
 export type { Capability, CapabilityHandle, CapabilityContext, CapabilityGetter, CapabilityProvider, } from './activities/chat/middleware/index.js';
 export * from './types.js';
 export { buildBaseUsage, type BaseUsageInput } from './utilities/usage.js';
+export { resolveMediaPrompt } from './utilities/media-prompt.js';
+export type { ResolvedMediaPrompt } from './utilities/media-prompt.js';
 export type { SystemPrompt, NormalizedSystemPrompt } from './system-prompts.js';
 export { normalizeSystemPrompts } from './system-prompts.js';
 export { detectImageMimeType } from './utils.js';

package/dist/esm/index.js

@@ -14,6 +14,7 @@ import { brandProviderTool } from "./tools/provider-tool.js";
 import { combineStrategies, maxIterations, untilFinishReason } from "./activities/chat/agent-loop-strategies.js";
 import { createFrozenRegistry, createToolRegistry } from "./tool-registry.js";
 import { buildBaseUsage } from "./utilities/usage.js";
+import { resolveMediaPrompt } from "./utilities/media-prompt.js";
 import { normalizeSystemPrompts } from "./system-prompts.js";
 import { detectImageMimeType } from "./utils.js";
 import { realtimeToken } from "./realtime/index.js";
@@ -88,6 +89,7 @@ export {
   parsePartialJSON,
   parseWithStandardSchema,
   realtimeToken,
+  resolveMediaPrompt,
   streamToText,
   summarize,
   toHttpResponse,

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}
+{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/esm/types.d.ts

@@ -1189,6 +1189,76 @@ export interface SummarizationResult {
     summary: string;
     usage: TokenUsage;
 }
+/**
+ * Optional role hint on a media input part (image / video / audio). Adapters
+ * read `metadata.role` to route the part to the provider-specific request
+ * field — e.g. `'mask'` → OpenAI `mask` / fal `mask_url`, `'end_frame'` → fal
+ * `end_image_url`, `'reference'` → fal `reference_image_urls`. When omitted
+ * the adapter falls back to positional routing.
+ */
+export type MediaInputRole = 'reference' | 'mask' | 'control' | 'start_frame' | 'end_frame' | 'character';
+/**
+ * Metadata convention for image / video / audio inputs to media generation.
+ * Carried on `ImagePart.metadata` / `VideoPart.metadata` / `AudioPart.metadata`
+ * when used as conditioning inputs to `generateImage()` or `generateVideo()`.
+ */
+export interface MediaInputMetadata {
+    /** Optional role hint disambiguating the part's intent for the adapter */
+    role?: MediaInputRole;
+    /**
+     * Optional user-defined label for this input (e.g. `'woman-in-red-dress'`).
+     * **Informational only** — adapters never read it and the SDK never
+     * rewrites prompt text based on it. Use it to correlate parts with the
+     * references you write in your prompt using the provider's own syntax
+     * (fal's `@Image1`, OpenAI's "image 1", etc.), or for your own
+     * bookkeeping/logging.
+     */
+    tag?: string;
+}
+/**
+ * A single part of a multimodal media-generation prompt. Reuses the chat
+ * content-part shapes: text parts carry the instruction, image / video /
+ * audio parts carry conditioning inputs (with an optional
+ * `metadata.role` hint — see {@link MediaInputRole}).
+ */
+export type MediaPromptPart = TextPart | ImagePart<MediaInputMetadata> | VideoPart<MediaInputMetadata> | AudioPart<MediaInputMetadata>;
+/**
+ * Prompt accepted by `generateImage()` / `generateVideo()`: a plain string,
+ * or an ordered array of content parts for image-conditioned generation
+ * ("not like this *(image)*, more like this *(image)*"). Part order is
+ * meaningful — adapters with native multimodal prompts (Gemini, OpenRouter)
+ * preserve the interleaving; named-field providers (fal, OpenAI, xAI)
+ * extract the media parts and flatten the text. Text is always sent
+ * verbatim: to reference inputs from the prompt, write the provider's own
+ * syntax yourself (e.g. fal's `@Image1`, OpenAI's "image 1"). An array may
+ * be media-only (e.g. upscalers or pure img2img endpoints that take no
+ * instruction text).
+ */
+export type MediaPrompt = string | Array<MediaPromptPart>;
+/**
+ * Non-text modalities a media-generation model can accept in its prompt.
+ */
+export type MediaPromptModality = 'image' | 'video' | 'audio';
+/** Maps a prompt modality to its content-part type. @internal */
+interface MediaPartByModality {
+    image: ImagePart<MediaInputMetadata>;
+    video: VideoPart<MediaInputMetadata>;
+    audio: AudioPart<MediaInputMetadata>;
+}
+/**
+ * Prompt type narrowed to the modalities a specific model supports.
+ * `MediaPromptFor<never>` (a text-only model) is `string | Array<TextPart>`;
+ * `MediaPromptFor<'image'>` additionally admits image parts, etc. Used by
+ * the activity option types together with the adapter's per-model input
+ * modality map so unsupported parts fail at compile time.
+ */
+export type MediaPromptFor<TModalities extends MediaPromptModality = never> = string | Array<TextPart | MediaPartByModality[TModalities]>;
+/**
+ * Per-model map from model name to the prompt modalities it accepts, used as
+ * an adapter type parameter (`TModelInputModalitiesByName`). Models absent
+ * from the map fall back to the unconstrained {@link MediaPrompt}.
+ */
+export type ModelInputModalitiesByName = Record<string, ReadonlyArray<MediaPromptModality>>;
 /**
  * Options for image generation.
  * These are the common options supported across providers.
@@ -1196,8 +1266,16 @@ export interface SummarizationResult {
 export interface ImageGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string> {
     /** The model to use for image generation */
     model: string;
-    /** Text description of the desired image(s) */
-    prompt: string;
+    /**
+     * Description of the desired image(s): a plain string, or an ordered array
+     * of content parts for image-conditioned generation (image-to-image,
+     * reference-guided, edit, multi-reference). Media parts may carry
+     * `metadata.role` to disambiguate intent (mask, control, reference, …).
+     * Adapters map parts onto the provider-native request — e.g. Gemini
+     * multimodal `contents`, OpenAI `images.edit()`, fal `image_url` /
+     * `mask_url` — and throw a clear runtime error for unsupported modalities.
+     */
+    prompt: MediaPrompt;
     /** Number of images to generate (default: 1) */
     numberOfImages?: number;
     /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
@@ -1293,15 +1371,26 @@ export interface AudioGenerationResult {
  *
  * @experimental Video generation is an experimental feature and may change.
  */
-export interface VideoGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string> {
+export interface VideoGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string, TDuration extends string | number | undefined = number> {
     /** The model to use for video generation */
     model: string;
-    /** Text description of the desired video */
-    prompt: string;
+    /**
+     * Description of the desired video: a plain string, or an ordered array of
+     * content parts for image-conditioned generation. Image parts may carry
+     * `metadata.role` (`'start_frame' | 'end_frame' | 'reference' |
+     * 'character'`) to disambiguate intent; adapters route them onto the
+     * provider-native request (e.g. OpenAI Sora `input_reference`, fal
+     * `image_url` / `end_image_url`) and throw at runtime if unsupported.
+     */
+    prompt: MediaPrompt;
     /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
     size?: TSize;
-    /** Video duration in seconds */
-    duration?: number;
+    /**
+     * Video duration in seconds. Adapters that declare a per-model duration
+     * map narrow this to the model's valid union; use
+     * `adapter.snapDuration(seconds)` to coerce raw seconds to a valid value.
+     */
+    duration?: TDuration;
     /** Model-specific options for video generation */
     modelOptions?: TProviderOptions;
     /**

package/dist/esm/utilities/media-prompt.d.ts

@@ -0,0 +1,35 @@
+import { AudioPart, ImagePart, MediaInputMetadata, MediaPrompt, MediaPromptPart, VideoPart } from '../types.js';
+/**
+ * A {@link MediaPrompt} decomposed into the views adapters consume.
+ *
+ * Adapters with native multimodal prompts (Gemini `contents`, OpenRouter
+ * chat content parts) consume `parts` to preserve interleaving; named-field
+ * providers (fal, OpenAI) consume `text` plus the typed media buckets.
+ *
+ * Prompt text is **never rewritten**: text parts are concatenated verbatim.
+ * Providers that support referencing inputs from the prompt (e.g. fal's
+ * `@Image1`, OpenAI's "image 1" prose) expect the user to write that syntax
+ * themselves — the SDK does not inject or substitute markers.
+ */
+export interface ResolvedMediaPrompt {
+    /**
+     * Text parts concatenated verbatim (paragraph-separated). Empty string
+     * for media-only prompts.
+     */
+    text: string;
+    /** The prompt as ordered parts; a string prompt becomes one text part. */
+    parts: Array<MediaPromptPart>;
+    /** Image parts in prompt order. */
+    images: Array<ImagePart<MediaInputMetadata>>;
+    /** Video parts in prompt order. */
+    videos: Array<VideoPart<MediaInputMetadata>>;
+    /** Audio parts in prompt order. */
+    audios: Array<AudioPart<MediaInputMetadata>>;
+}
+/**
+ * Decompose a {@link MediaPrompt} into flattened text and per-modality part
+ * buckets, preserving prompt order everywhere. This is the single downrev
+ * point from the canonical interleaved prompt shape to the named-field
+ * request shapes most providers expose.
+ */
+export declare function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt;

package/dist/esm/utilities/media-prompt.js

@@ -0,0 +1,43 @@
+function resolveMediaPrompt(prompt) {
+  if (typeof prompt === "string") {
+    const textPart = { type: "text", content: prompt };
+    return {
+      text: prompt,
+      parts: [textPart],
+      images: [],
+      videos: [],
+      audios: []
+    };
+  }
+  const images = [];
+  const videos = [];
+  const audios = [];
+  const textSegments = [];
+  for (const part of prompt) {
+    switch (part.type) {
+      case "text":
+        if (part.content) textSegments.push(part.content);
+        break;
+      case "image":
+        images.push(part);
+        break;
+      case "video":
+        videos.push(part);
+        break;
+      case "audio":
+        audios.push(part);
+        break;
+    }
+  }
+  return {
+    text: textSegments.join("\n\n"),
+    parts: prompt,
+    images,
+    videos,
+    audios
+  };
+}
+export {
+  resolveMediaPrompt
+};
+//# sourceMappingURL=media-prompt.js.map

package/dist/esm/utilities/media-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"media-prompt.js","sources":["../../../src/utilities/media-prompt.ts"],"sourcesContent":["import type {\n  AudioPart,\n  ImagePart,\n  MediaInputMetadata,\n  MediaPrompt,\n  MediaPromptPart,\n  TextPart,\n  VideoPart,\n} from '../types'\n\n/**\n * A {@link MediaPrompt} decomposed into the views adapters consume.\n *\n * Adapters with native multimodal prompts (Gemini `contents`, OpenRouter\n * chat content parts) consume `parts` to preserve interleaving; named-field\n * providers (fal, OpenAI) consume `text` plus the typed media buckets.\n *\n * Prompt text is **never rewritten**: text parts are concatenated verbatim.\n * Providers that support referencing inputs from the prompt (e.g. fal's\n * `@Image1`, OpenAI's \"image 1\" prose) expect the user to write that syntax\n * themselves — the SDK does not inject or substitute markers.\n */\nexport interface ResolvedMediaPrompt {\n  /**\n   * Text parts concatenated verbatim (paragraph-separated). Empty string\n   * for media-only prompts.\n   */\n  text: string\n  /** The prompt as ordered parts; a string prompt becomes one text part. */\n  parts: Array<MediaPromptPart>\n  /** Image parts in prompt order. */\n  images: Array<ImagePart<MediaInputMetadata>>\n  /** Video parts in prompt order. */\n  videos: Array<VideoPart<MediaInputMetadata>>\n  /** Audio parts in prompt order. */\n  audios: Array<AudioPart<MediaInputMetadata>>\n}\n\n/**\n * Decompose a {@link MediaPrompt} into flattened text and per-modality part\n * buckets, preserving prompt order everywhere. This is the single downrev\n * point from the canonical interleaved prompt shape to the named-field\n * request shapes most providers expose.\n */\nexport function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt {\n  if (typeof prompt === 'string') {\n    const textPart: TextPart = { type: 'text', content: prompt }\n    return {\n      text: prompt,\n      parts: [textPart],\n      images: [],\n      videos: [],\n      audios: [],\n    }\n  }\n\n  const images: Array<ImagePart<MediaInputMetadata>> = []\n  const videos: Array<VideoPart<MediaInputMetadata>> = []\n  const audios: Array<AudioPart<MediaInputMetadata>> = []\n  const textSegments: Array<string> = []\n\n  for (const part of prompt) {\n    switch (part.type) {\n      case 'text':\n        if (part.content) textSegments.push(part.content)\n        break\n      case 'image':\n        images.push(part)\n        break\n      case 'video':\n        videos.push(part)\n        break\n      case 'audio':\n        audios.push(part)\n        break\n    }\n  }\n\n  return {\n    text: textSegments.join('\\n\\n'),\n    parts: prompt,\n    images,\n    videos,\n    audios,\n  }\n}\n"],"names":[],"mappings":"AA4CO,SAAS,mBAAmB,QAA0C;AAC3E,MAAI,OAAO,WAAW,UAAU;AAC9B,UAAM,WAAqB,EAAE,MAAM,QAAQ,SAAS,OAAA;AACpD,WAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO,CAAC,QAAQ;AAAA,MAChB,QAAQ,CAAA;AAAA,MACR,QAAQ,CAAA;AAAA,MACR,QAAQ,CAAA;AAAA,IAAC;AAAA,EAEb;AAEA,QAAM,SAA+C,CAAA;AACrD,QAAM,SAA+C,CAAA;AACrD,QAAM,SAA+C,CAAA;AACrD,QAAM,eAA8B,CAAA;AAEpC,aAAW,QAAQ,QAAQ;AACzB,YAAQ,KAAK,MAAA;AAAA,MACX,KAAK;AACH,YAAI,KAAK,QAAS,cAAa,KAAK,KAAK,OAAO;AAChD;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,IAAA;AAAA,EAEN;AAEA,SAAO;AAAA,IACL,MAAM,aAAa,KAAK,MAAM;AAAA,IAC9B,OAAO;AAAA,IACP;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "description": "Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.",
   "author": "Tanner Linsley",
   "license": "MIT",
@@ -76,7 +76,7 @@
     "@ag-ui/core": "^0.0.52",
     "@standard-schema/spec": "^1.1.0",
     "partial-json": "^0.1.7",
-    "@tanstack/ai-event-client": "0.6.2"
+    "@tanstack/ai-event-client": "0.6.3"
   },
   "peerDependencies": {
     "@opentelemetry/api": ">=1.9.0"

package/skills/ai-core/media-generation/SKILL.md

@@ -3,8 +3,9 @@ name: ai-core/media-generation
 description: >
   Image, audio, video, speech (TTS), and transcription generation using
   activity-specific adapters: generateImage() with openaiImage/geminiImage,
-  generateAudio() with geminiAudio/falAudio, generateVideo() with async
-  polling, generateSpeech() with openaiSpeech, generateTranscription() with
+  generateAudio() with geminiAudio/falAudio, generateVideo() with
+  openaiVideo/geminiVideo (async polling, per-model typed durations),
+  generateSpeech() with openaiSpeech, generateTranscription() with
   openaiTranscription. React hooks: useGenerateImage, useGenerateAudio,
   useGenerateSpeech, useTranscription, useGenerateVideo.
   TanStack Start server function integration with toServerSentEventsResponse.
@@ -189,6 +190,103 @@ Result shape: `ImageGenerationResult` with `images` array where each entry
 has `b64Json?`, `url?`, and `revisedPrompt?`. OpenAI image URLs expire
 after 1 hour -- download or display immediately.
 
+#### Image-conditioned generation: multimodal `prompt` parts
+
+Both `generateImage()` and `generateVideo()` accept the `prompt` either as
+a plain string or as an ordered array of content parts (`TextPart` /
+`ImagePart` / `VideoPart` / `AudioPart` — the same shapes used elsewhere in
+TanStack AI). Part order is meaningful: natively multimodal providers
+(Gemini, OpenRouter) receive parts in order; named-field providers (OpenAI,
+fal, xAI) extract media parts and flatten the text. Prompt text is always
+sent verbatim — to reference inputs from the prompt, write the provider's
+own syntax (fal `@Image1`, OpenAI "image 1" prose); the SDK never injects
+or rewrites markers. Each media part may carry an optional
+`metadata.role` hint that adapters use to route the part to the
+provider-specific field. The accepted part types are narrowed per model at
+compile time via the adapter's input-modality map.
+
+```typescript
+import { generateImage } from '@tanstack/ai'
+import { openaiImage } from '@tanstack/ai-openai'
+
+// Image-to-image (OpenAI gpt-image-2 / gpt-image-1, dall-e-2)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Turn this into a cinematic product photo' },
+    { type: 'image', source: { type: 'url', value: 'https://…/product.png' } },
+  ],
+})
+
+// Multi-reference (up to 16 for gpt-image models; up to ~14 for Gemini native
+// — a provider limit, not enforced by the SDK)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Apply the second image as style to the first' },
+    { type: 'image', source: { type: 'url', value: 'https://…/product.png' } },
+    { type: 'image', source: { type: 'url', value: 'https://…/style.png' } },
+  ],
+})
+
+// Inpaint via metadata.role === 'mask' (OpenAI gpt-image models, dall-e-2; fal mask_url)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Replace the masked region with a tree' },
+    { type: 'image', source: { type: 'url', value: photoUrl } },
+    {
+      type: 'image',
+      source: { type: 'url', value: maskUrl },
+      metadata: { role: 'mask' },
+    },
+  ],
+})
+
+// Image-to-video (OpenAI Sora: single input_reference; fal: image_url + optional end_image_url)
+import { generateVideo } from '@tanstack/ai'
+import { falVideo } from '@tanstack/ai-fal'
+
+await generateVideo({
+  adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'),
+  prompt: [
+    { type: 'image', source: { type: 'url', value: firstFrameUrl } },
+    { type: 'text', content: 'Slow cinematic push-in' },
+    {
+      type: 'image',
+      source: { type: 'url', value: lastFrameUrl },
+      metadata: { role: 'end_frame' },
+    },
+  ],
+})
+```
+
+**Role hints** (`metadata.role`):
+
+| Role            | Maps to                                                                                               |
+| --------------- | ----------------------------------------------------------------------------------------------------- |
+| `'reference'`   | fal `reference_image_urls`; Gemini multimodal part; positional otherwise                              |
+| `'character'`   | Same as `'reference'`; Veo `referenceImages` slot (planned — no Veo adapter yet)                      |
+| `'mask'`        | OpenAI `mask` (gpt-image-2, gpt-image-1, dall-e-2); fal `mask_url`                                    |
+| `'control'`     | fal `control_image_url` (ControlNet / depth / pose)                                                   |
+| `'start_frame'` | fal `start_image_url` (or the endpoint's field, e.g. `image_url` on Kling i2v); Veo `image` (planned) |
+| `'end_frame'`   | fal `end_image_url` (or e.g. `tail_image_url` / `last_frame_url`); Veo `lastFrame` (planned)          |
+
+**Provider support matrix:**
+
+| Provider   | `generateImage` image parts                                                                                                                                                                              | `generateVideo` image parts                                                                                                                                                                        |
+| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| OpenAI     | gpt-image-2 / gpt-image-1 / -mini → `images.edit()` (up to 16). dall-e-2 → edit (1). dall-e-3 throws.                                                                                                    | Sora-2 / -pro → `input_reference` (single). Throws if >1.                                                                                                                                          |
+| Gemini     | Native (gemini-\*-flash-image, "nano-banana") → multimodal `contents`. Imagen throws.                                                                                                                    | No native Veo adapter yet — deferred to a follow-up.                                                                                                                                               |
+| fal        | Per-endpoint field names from a generated map (`pnpm generate:fal-image-fields`). Defaults: 1 input → `image_url`; >1 → `image_urls`; roles → `mask_url` / `control_image_url` / `reference_image_urls`. | Per-endpoint map (e.g. Kling i2v start frame → `image_url`). Defaults: 1 input → `image_url`; `start_frame`/`end_frame` → `start_image_url`/`end_image_url`; `reference` → `reference_image_urls`. |
+| Grok       | grok-imagine models → `/v1/images/edits` JSON endpoint (≤3 sources, addressed by xAI in request order; prompt sent verbatim; mask/control throw). grok-2-image-1212 throws.                              | n/a                                                                                                                                                                                                |
+| OpenRouter | Prompt parts map 1:1 onto multimodal `text` / `image_url` content parts, preserving interleaved order.                                                                                                   | n/a                                                                                                                                                                                                |
+| Anthropic  | n/a (no image generation API).                                                                                                                                                                           | n/a                                                                                                                                                                                                |
+
+Video and audio prompt parts follow the same `metadata.role` convention
+for video-to-video and lipsync flows on fal; other providers throw when
+they're passed.
+
 ### 2. Audio Generation (Music, Sound Effects)
 
 Distinct from TTS — `generateAudio()` produces non-speech audio content.
@@ -331,6 +429,31 @@ const stream = generateVideo({
 return toServerSentEventsResponse(stream)
 ```
 
+Google Veo (`@tanstack/ai-gemini`) uses the same jobs/polling flow. Its
+`duration` option is typed per model (e.g. `4 | 6 | 8` for Veo 3.x,
+`5 | 6 | 8` for Veo 2); use `adapter.snapDuration(seconds)` to coerce raw
+seconds and `adapter.availableDurations()` to enumerate the valid set.
+Image prompt parts route by `metadata.role`: first un-roled /
+`'start_frame'` image → input image, `'end_frame'` → `lastFrame`,
+`'reference'` / `'character'` → `referenceImages`:
+
+```typescript
+import { geminiVideo } from '@tanstack/ai-gemini'
+
+const adapter = geminiVideo('veo-3.1-generate-preview')
+adapter.availableDurations() // { kind: 'discrete', values: [4, 6, 8] }
+
+const { jobId } = await generateVideo({
+  adapter,
+  prompt: 'A golden retriever playing in sunflowers',
+  size: '16:9', // Veo sizes are aspect ratios: '16:9' | '9:16'
+  duration: adapter.snapDuration(7), // 6
+  modelOptions: { resolution: '1080p', generateAudio: true },
+})
+// Note: Veo result URLs require the Google API key to download
+// (x-goog-api-key header or ?key= query parameter).
+```
+
 Client hook with job tracking:
 
 ```tsx
@@ -607,7 +730,54 @@ generateSpeech({
 
 > Source: Gemini TTS adapter validation; CodeRabbit review of PR #463.
 
-### h. LOW: Writing a logging middleware to see media chunks flow through
+### h. HIGH: Passing image prompt parts to a model that doesn't support image-conditioned generation
+
+Not every model accepts image-conditioned prompts. The `prompt` type is
+narrowed per model, so passing an image part to a text-only model
+(dall-e-3, Imagen, grok-2-image) is a **compile-time error**; adapters
+also throw a clear runtime error as a backstop, so users learn at call
+time rather than getting silently wrong output.
+
+```typescript
+// WRONG — dall-e-3 has no edit/inputs API; image parts are a type error
+generateImage({
+  adapter: openaiImage('dall-e-3'),
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } }, // ❌ type error
+  ],
+})
+
+// WRONG — Imagen is text-to-image only; same compile-time rejection
+generateImage({
+  adapter: geminiImage('imagen-4.0-generate-001'),
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } }, // ❌ type error
+  ],
+})
+
+// CORRECT — use a model that supports image-conditioned generation
+generateImage({
+  adapter: openaiImage('gpt-image-2'), // edits up to 16 images
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } },
+  ],
+})
+
+generateImage({
+  adapter: geminiImage('gemini-3.1-flash-image-preview'), // native multimodal
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } },
+  ],
+})
+```
+
+> Source: docs/media/image-generation.md, docs/media/video-generation.md.
+
+### i. LOW: Writing a logging middleware to see media chunks flow through
 
 Every media activity — `generateAudio`, `generateSpeech`,
 `generateTranscription`, `generateImage`, `generateVideo` — accepts the