@tanstack/ai 0.31.0 → 0.33.0

package/src/types.ts

@@ -820,14 +820,14 @@ export interface TextOptions<
   systemPrompts?: Array<SystemPrompt>
   agentLoopStrategy?: AgentLoopStrategy
   /**
-   * Additional metadata to attach to the request.
-   * Can be used for tracking, debugging, or passing custom information.
-   * Structure and constraints vary by provider.
+   * Observability metadata attached to this call. Surfaced to middleware,
+   * devtools, and the event client; values may be arbitrarily structured
+   * (objects, arrays). Adapters never forward this field onto the provider
+   * wire request.
    *
-   * Provider usage:
-   * - OpenAI: `metadata` (Record<string, string>) - max 16 key-value pairs, keys max 64 chars, values max 512 chars
-   * - Anthropic: `metadata` (Record<string, any>) - includes optional user_id (max 256 chars)
-   * - Gemini: Not directly available in TextProviderOptions
+   * To send provider-side request metadata, use the provider's
+   * `modelOptions` field instead, where the provider supports one (e.g.
+   * OpenAI's and OpenRouter's `metadata` are both Record<string, string>).
    */
   metadata?: Record<string, any> | undefined
   modelOptions?: TProviderOptionsForModel
@@ -1470,6 +1470,99 @@ export interface SummarizationResult {
 // Image Generation Types
 // ============================================================================
 
+/**
+ * Optional role hint on a media input part (image / video / audio). Adapters
+ * read `metadata.role` to route the part to the provider-specific request
+ * field — e.g. `'mask'` → OpenAI `mask` / fal `mask_url`, `'end_frame'` → fal
+ * `end_image_url`, `'reference'` → fal `reference_image_urls`. When omitted
+ * the adapter falls back to positional routing.
+ */
+export type MediaInputRole =
+  | 'reference'
+  | 'mask'
+  | 'control'
+  | 'start_frame'
+  | 'end_frame'
+  | 'character'
+
+/**
+ * Metadata convention for image / video / audio inputs to media generation.
+ * Carried on `ImagePart.metadata` / `VideoPart.metadata` / `AudioPart.metadata`
+ * when used as conditioning inputs to `generateImage()` or `generateVideo()`.
+ */
+export interface MediaInputMetadata {
+  /** Optional role hint disambiguating the part's intent for the adapter */
+  role?: MediaInputRole
+  /**
+   * Optional user-defined label for this input (e.g. `'woman-in-red-dress'`).
+   * **Informational only** — adapters never read it and the SDK never
+   * rewrites prompt text based on it. Use it to correlate parts with the
+   * references you write in your prompt using the provider's own syntax
+   * (fal's `@Image1`, OpenAI's "image 1", etc.), or for your own
+   * bookkeeping/logging.
+   */
+  tag?: string
+}
+
+/**
+ * A single part of a multimodal media-generation prompt. Reuses the chat
+ * content-part shapes: text parts carry the instruction, image / video /
+ * audio parts carry conditioning inputs (with an optional
+ * `metadata.role` hint — see {@link MediaInputRole}).
+ */
+export type MediaPromptPart =
+  | TextPart
+  | ImagePart<MediaInputMetadata>
+  | VideoPart<MediaInputMetadata>
+  | AudioPart<MediaInputMetadata>
+
+/**
+ * Prompt accepted by `generateImage()` / `generateVideo()`: a plain string,
+ * or an ordered array of content parts for image-conditioned generation
+ * ("not like this *(image)*, more like this *(image)*"). Part order is
+ * meaningful — adapters with native multimodal prompts (Gemini, OpenRouter)
+ * preserve the interleaving; named-field providers (fal, OpenAI, xAI)
+ * extract the media parts and flatten the text. Text is always sent
+ * verbatim: to reference inputs from the prompt, write the provider's own
+ * syntax yourself (e.g. fal's `@Image1`, OpenAI's "image 1"). An array may
+ * be media-only (e.g. upscalers or pure img2img endpoints that take no
+ * instruction text).
+ */
+export type MediaPrompt = string | Array<MediaPromptPart>
+
+/**
+ * Non-text modalities a media-generation model can accept in its prompt.
+ */
+export type MediaPromptModality = 'image' | 'video' | 'audio'
+
+/** Maps a prompt modality to its content-part type. @internal */
+interface MediaPartByModality {
+  image: ImagePart<MediaInputMetadata>
+  video: VideoPart<MediaInputMetadata>
+  audio: AudioPart<MediaInputMetadata>
+}
+
+/**
+ * Prompt type narrowed to the modalities a specific model supports.
+ * `MediaPromptFor<never>` (a text-only model) is `string | Array<TextPart>`;
+ * `MediaPromptFor<'image'>` additionally admits image parts, etc. Used by
+ * the activity option types together with the adapter's per-model input
+ * modality map so unsupported parts fail at compile time.
+ */
+export type MediaPromptFor<TModalities extends MediaPromptModality = never> =
+  | string
+  | Array<TextPart | MediaPartByModality[TModalities]>
+
+/**
+ * Per-model map from model name to the prompt modalities it accepts, used as
+ * an adapter type parameter (`TModelInputModalitiesByName`). Models absent
+ * from the map fall back to the unconstrained {@link MediaPrompt}.
+ */
+export type ModelInputModalitiesByName = Record<
+  string,
+  ReadonlyArray<MediaPromptModality>
+>
+
 /**
  * Options for image generation.
  * These are the common options supported across providers.
@@ -1480,8 +1573,16 @@ export interface ImageGenerationOptions<
 > {
   /** The model to use for image generation */
   model: string
-  /** Text description of the desired image(s) */
-  prompt: string
+  /**
+   * Description of the desired image(s): a plain string, or an ordered array
+   * of content parts for image-conditioned generation (image-to-image,
+   * reference-guided, edit, multi-reference). Media parts may carry
+   * `metadata.role` to disambiguate intent (mask, control, reference, …).
+   * Adapters map parts onto the provider-native request — e.g. Gemini
+   * multimodal `contents`, OpenAI `images.edit()`, fal `image_url` /
+   * `mask_url` — and throw a clear runtime error for unsupported modalities.
+   */
+  prompt: MediaPrompt
   /** Number of images to generate (default: 1) */
   numberOfImages?: number
   /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
@@ -1599,15 +1700,27 @@ export interface AudioGenerationResult {
 export interface VideoGenerationOptions<
   TProviderOptions extends object = object,
   TSize extends string | undefined = string,
+  TDuration extends string | number | undefined = number,
 > {
   /** The model to use for video generation */
   model: string
-  /** Text description of the desired video */
-  prompt: string
+  /**
+   * Description of the desired video: a plain string, or an ordered array of
+   * content parts for image-conditioned generation. Image parts may carry
+   * `metadata.role` (`'start_frame' | 'end_frame' | 'reference' |
+   * 'character'`) to disambiguate intent; adapters route them onto the
+   * provider-native request (e.g. OpenAI Sora `input_reference`, fal
+   * `image_url` / `end_image_url`) and throw at runtime if unsupported.
+   */
+  prompt: MediaPrompt
   /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
   size?: TSize
-  /** Video duration in seconds */
-  duration?: number
+  /**
+   * Video duration in seconds. Adapters that declare a per-model duration
+   * map narrow this to the model's valid union; use
+   * `adapter.snapDuration(seconds)` to coerce raw seconds to a valid value.
+   */
+  duration?: TDuration
   /** Model-specific options for video generation */
   modelOptions?: TProviderOptions
   /**

package/src/utilities/errors.ts

@@ -0,0 +1,29 @@
+/**
+ * Best-effort extraction of a human-readable message from an unknown thrown
+ * value, returning `undefined` when none can be found.
+ *
+ * Used by `otelMiddleware` so error reporting stays identical across chat and
+ * media spans.
+ */
+export function errorMessage(err: unknown): string | undefined {
+  if (err instanceof Error) return err.message
+  if (typeof err === 'string') return err
+  if (err && typeof err === 'object' && 'message' in err) {
+    const m = (err as { message?: unknown }).message
+    if (typeof m === 'string') return m
+  }
+  return undefined
+}
+
+/**
+ * Best-effort extraction of an error's type name (used for the `error.type`
+ * metric attribute), falling back to `'Error'` when no name is available.
+ */
+export function errorTypeName(err: unknown): string {
+  if (err instanceof Error) return err.name || 'Error'
+  if (err && typeof err === 'object' && 'name' in err) {
+    const n = (err as { name?: unknown }).name
+    if (typeof n === 'string') return n
+  }
+  return 'Error'
+}

package/src/utilities/media-prompt.ts

@@ -0,0 +1,86 @@
+import type {
+  AudioPart,
+  ImagePart,
+  MediaInputMetadata,
+  MediaPrompt,
+  MediaPromptPart,
+  TextPart,
+  VideoPart,
+} from '../types'
+
+/**
+ * A {@link MediaPrompt} decomposed into the views adapters consume.
+ *
+ * Adapters with native multimodal prompts (Gemini `contents`, OpenRouter
+ * chat content parts) consume `parts` to preserve interleaving; named-field
+ * providers (fal, OpenAI) consume `text` plus the typed media buckets.
+ *
+ * Prompt text is **never rewritten**: text parts are concatenated verbatim.
+ * Providers that support referencing inputs from the prompt (e.g. fal's
+ * `@Image1`, OpenAI's "image 1" prose) expect the user to write that syntax
+ * themselves — the SDK does not inject or substitute markers.
+ */
+export interface ResolvedMediaPrompt {
+  /**
+   * Text parts concatenated verbatim (paragraph-separated). Empty string
+   * for media-only prompts.
+   */
+  text: string
+  /** The prompt as ordered parts; a string prompt becomes one text part. */
+  parts: Array<MediaPromptPart>
+  /** Image parts in prompt order. */
+  images: Array<ImagePart<MediaInputMetadata>>
+  /** Video parts in prompt order. */
+  videos: Array<VideoPart<MediaInputMetadata>>
+  /** Audio parts in prompt order. */
+  audios: Array<AudioPart<MediaInputMetadata>>
+}
+
+/**
+ * Decompose a {@link MediaPrompt} into flattened text and per-modality part
+ * buckets, preserving prompt order everywhere. This is the single downrev
+ * point from the canonical interleaved prompt shape to the named-field
+ * request shapes most providers expose.
+ */
+export function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt {
+  if (typeof prompt === 'string') {
+    const textPart: TextPart = { type: 'text', content: prompt }
+    return {
+      text: prompt,
+      parts: [textPart],
+      images: [],
+      videos: [],
+      audios: [],
+    }
+  }
+
+  const images: Array<ImagePart<MediaInputMetadata>> = []
+  const videos: Array<VideoPart<MediaInputMetadata>> = []
+  const audios: Array<AudioPart<MediaInputMetadata>> = []
+  const textSegments: Array<string> = []
+
+  for (const part of prompt) {
+    switch (part.type) {
+      case 'text':
+        if (part.content) textSegments.push(part.content)
+        break
+      case 'image':
+        images.push(part)
+        break
+      case 'video':
+        videos.push(part)
+        break
+      case 'audio':
+        audios.push(part)
+        break
+    }
+  }
+
+  return {
+    text: textSegments.join('\n\n'),
+    parts: prompt,
+    images,
+    videos,
+    audios,
+  }
+}

package/src/utilities/numbers.ts

@@ -0,0 +1,15 @@
+/**
+ * Return the first candidate that is a finite `number`, or `undefined`.
+ *
+ * Handy for picking a value from among several possible spellings/sources where
+ * only some are populated — e.g. the provider-native sampling option names read
+ * by the OTel middleware, or the optional numeric fields on `TokenUsage`.
+ */
+export function firstNumber(...candidates: Array<unknown>): number | undefined {
+  for (const candidate of candidates) {
+    if (typeof candidate === 'number' && Number.isFinite(candidate)) {
+      return candidate
+    }
+  }
+  return undefined
+}