@tanstack/ai 0.31.0 → 0.33.0

package/src/activities/generateImage/index.ts

@@ -8,10 +8,24 @@
 import { aiEventClient } from '@tanstack/ai-event-client'
 import { streamGenerationResult } from '../stream-generation-result.js'
 import { resolveDebugOption } from '../../logger/resolve'
+import {
+  createGenerationContext,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware'
+import { resolveMediaPrompt } from '../../utilities/media-prompt'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware'
 import type { ImageAdapter } from './adapter'
-import type { ImageGenerationResult, StreamChunk } from '../../types'
+import type {
+  ImageGenerationResult,
+  MediaPrompt,
+  MediaPromptFor,
+  StreamChunk,
+} from '../../types'
 
 // ===========================
 // Activity Kind
@@ -55,6 +69,23 @@ export type ImageSizeForModel<TAdapter, TModel extends string> =
         : string
     : string
 
+/**
+ * Extract the prompt type a model accepts from an ImageAdapter via ~types.
+ * Adapters declare a per-model input-modality map; models in the map get a
+ * `prompt` narrowed to text + their supported part types (text-only models
+ * accept `string | Array<TextPart>`), so unsupported media parts fail at
+ * compile time. Adapters without a map fall back to the full MediaPrompt.
+ */
+export type ImagePromptForModel<TAdapter, TModel extends string> =
+  TAdapter extends ImageAdapter<any, any, any, any, infer ModsByName>
+    ? string extends keyof ModsByName
+      ? // No explicit map - accept the full union
+        MediaPrompt
+      : TModel extends keyof ModsByName
+        ? MediaPromptFor<ModsByName[TModel][number]>
+        : MediaPrompt
+    : MediaPrompt
+
 // ===========================
 // Activity Options Type
 // ===========================
@@ -72,8 +103,16 @@ export type ImageActivityOptions<
 > = {
   /** The image adapter to use (must be created with a model) */
   adapter: TAdapter & { kind: typeof kind }
-  /** Text description of the desired image(s) */
-  prompt: string
+  /**
+   * Description of the desired image(s). Either a plain string, or — for
+   * models that support image-conditioned generation — an ordered array of
+   * content parts interleaving text with image inputs (image-to-image,
+   * reference-guided, edit, multi-reference). Media parts may carry
+   * `metadata.role` (`'reference' | 'mask' | 'control' | 'character'`) to
+   * disambiguate intent. The accepted part types are narrowed per model via
+   * the adapter's input-modality map.
+   */
+  prompt: ImagePromptForModel<TAdapter, TAdapter['model']>
   /** Number of images to generate (default: 1) */
   numberOfImages?: number
   /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
@@ -92,6 +131,12 @@ export type ImageActivityOptions<
    * control and/or a custom `Logger`.
    */
   debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
 } & ({} extends ImageProviderOptionsForModel<TAdapter, TAdapter['model']>
   ? {
       /** Provider-specific options for image generation */ modelOptions?: ImageProviderOptionsForModel<
@@ -197,19 +242,49 @@ async function runGenerateImage<
 >(
   options: ImageActivityOptions<TAdapter, boolean>,
 ): Promise<ImageGenerationResult> {
-  const { adapter, stream: _stream, debug: _debug, ...rest } = options
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    ...rest
+  } = options
   const model = adapter.model
   const requestId = createId('image')
   const startTime = Date.now()
   const logger: InternalLogger = resolveDebugOption(options.debug)
 
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'image',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
+  // Devtools events carry the flattened prompt text plus media-part counts —
+  // the wire payload stays `prompt: string` regardless of the prompt shape.
+  const resolved = resolveMediaPrompt(rest.prompt)
+
   aiEventClient.emit('image:request:started', {
     requestId,
     provider: adapter.name,
     model,
-    prompt: rest.prompt,
+    prompt: resolved.text,
     numberOfImages: rest.numberOfImages,
     size: rest.size,
+    ...(resolved.images.length > 0 && {
+      imageInputCount: resolved.images.length,
+    }),
+    ...(resolved.videos.length > 0 && {
+      videoInputCount: resolved.videos.length,
+    }),
+    ...(resolved.audios.length > 0 && {
+      audioInputCount: resolved.audios.length,
+    }),
     modelOptions: rest.modelOptions,
     timestamp: startTime,
   })
@@ -255,8 +330,18 @@ async function runGenerateImage<
       count: result.images.length,
     })
 
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration,
+      usage: result.usage,
+    })
+
     return result
   } catch (error) {
+    await runGenerationError(middleware, mwCtx, {
+      error,
+      duration: Date.now() - startTime,
+    })
     logger.errors('generateImage activity failed', {
       error,
       source: 'generateImage',

package/src/activities/generateSpeech/index.ts

@@ -8,8 +8,16 @@
 import { aiEventClient } from '@tanstack/ai-event-client'
 import { streamGenerationResult } from '../stream-generation-result.js'
 import { resolveDebugOption } from '../../logger/resolve'
+import {
+  createGenerationContext,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware'
 import type { TTSAdapter } from './adapter'
 import type { StreamChunk, TTSResult } from '../../types'
 
@@ -73,6 +81,12 @@ export interface TTSActivityOptions<
    * control and/or a custom `Logger`.
    */
   debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
 }
 
 // ===========================
@@ -143,7 +157,13 @@ export function generateSpeech<
 async function runGenerateSpeech<
   TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
 >(options: TTSActivityOptions<TAdapter, boolean>): Promise<TTSResult> {
-  const { adapter, stream: _stream, debug: _debug, ...rest } = options
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    ...rest
+  } = options
   const model = adapter.model
   const requestId = createId('speech')
   const startTime = Date.now()
@@ -153,6 +173,17 @@ async function runGenerateSpeech<
     (adapter as { name?: string }).name ??
     'unknown'
 
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'tts',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
   aiEventClient.emit('speech:request:started', {
     requestId,
     provider: adapter.name,
@@ -202,6 +233,12 @@ async function runGenerateSpeech<
       contentType: result.contentType,
     })
 
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration,
+      usage: result.usage,
+    })
+
     return result
   } catch (error) {
     const duration = Date.now() - startTime
@@ -215,6 +252,10 @@ async function runGenerateSpeech<
       modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
       timestamp: Date.now(),
     })
+    await runGenerationError(middleware, mwCtx, {
+      error,
+      duration,
+    })
     logger.errors('generateSpeech activity failed', {
       error,
       source: 'generateSpeech',

package/src/activities/generateTranscription/index.ts

@@ -8,8 +8,16 @@
 import { aiEventClient } from '@tanstack/ai-event-client'
 import { streamGenerationResult } from '../stream-generation-result.js'
 import { resolveDebugOption } from '../../logger/resolve'
+import {
+  createGenerationContext,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware'
 import type { TranscriptionAdapter } from './adapter'
 import type { StreamChunk, TranscriptionResult } from '../../types'
 
@@ -76,6 +84,12 @@ export interface TranscriptionActivityOptions<
    * control and/or a custom `Logger`.
    */
   debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
 }
 
 // ===========================
@@ -174,7 +188,13 @@ async function runGenerateTranscription<
 >(
   options: TranscriptionActivityOptions<TAdapter, boolean>,
 ): Promise<TranscriptionResult> {
-  const { adapter, stream: _stream, debug: _debug, ...rest } = options
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    ...rest
+  } = options
   const model = adapter.model
   const requestId = createId('transcription')
   const startTime = Date.now()
@@ -184,6 +204,17 @@ async function runGenerateTranscription<
     (adapter as { name?: string }).name ??
     'unknown'
 
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'transcription',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
   aiEventClient.emit('transcription:request:started', {
     requestId,
     provider: adapter.name,
@@ -220,6 +251,12 @@ async function runGenerateTranscription<
       { hasText: !!result.text },
     )
 
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration,
+      usage: result.usage,
+    })
+
     return result
   } catch (error) {
     const duration = Date.now() - startTime
@@ -233,6 +270,10 @@ async function runGenerateTranscription<
       modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
       timestamp: Date.now(),
     })
+    await runGenerationError(middleware, mwCtx, {
+      error,
+      duration,
+    })
     logger.errors('generateTranscription activity failed', {
       error,
       source: 'generateTranscription',

package/src/activities/generateVideo/adapter.ts

@@ -1,10 +1,30 @@
 import type {
+  ModelInputModalitiesByName,
   VideoGenerationOptions,
   VideoJobResult,
   VideoStatusResult,
   VideoUrlResult,
 } from '../../types'
 
+/**
+ * Structured description of the durations a video model accepts.
+ *
+ * Tagged union so the same shape can express discrete enums (OpenAI Sora,
+ * Veo), continuous ranges, mixed shapes, and models with no duration field.
+ * Consumed by `VideoAdapter.availableDurations()`.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type DurationOptions<T extends string | number | undefined> =
+  | { kind: 'discrete'; values: ReadonlyArray<NonNullable<T>> }
+  | { kind: 'range'; min: number; max: number; step?: number; unit: 'seconds' }
+  | {
+      kind: 'mixed'
+      values: ReadonlyArray<NonNullable<T>>
+      range?: { min: number; max: number; step?: number }
+    }
+  | { kind: 'none' }
+
 /**
  * Configuration for video adapter instances
  *
@@ -31,6 +51,11 @@ export interface VideoAdapterConfig {
  * - TProviderOptions: Provider-specific options (already resolved)
  * - TModelProviderOptionsByName: Map from model name to its specific provider options
  * - TModelSizeByName: Map from model name to its supported sizes
+ * - TModelInputModalitiesByName: Map from model name to the non-text prompt
+ *   modalities it accepts (constrains the `prompt` part types at compile time)
+ * - TModelDurationByName: Map from model name to its supported duration
+ *   union. Defaults to `Record<string, number>` so adapters that haven't
+ *   declared a map keep today's `duration?: number` typing.
  */
 export interface VideoAdapter<
   TModel extends string = string,
@@ -40,6 +65,10 @@ export interface VideoAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
+  TModelDurationByName extends Record<string, string | number | undefined> =
+    Record<string, number>,
 > {
   /** Discriminator for adapter kind - used to determine API shape */
   readonly kind: 'video'
@@ -55,6 +84,8 @@ export interface VideoAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
+    modelDurationByName: TModelDurationByName
   }
 
   /**
@@ -62,7 +93,11 @@ export interface VideoAdapter<
    * Returns a job ID that can be used to poll for status and retrieve the video.
    */
   createVideoJob: (
-    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
+    options: VideoGenerationOptions<
+      TProviderOptions,
+      TModelSizeByName[TModel],
+      TModelDurationByName[TModel]
+    >,
   ) => Promise<VideoJobResult>
 
   /**
@@ -75,13 +110,26 @@ export interface VideoAdapter<
    * Should only be called after status is 'completed'.
    */
   getVideoUrl: (jobId: string) => Promise<VideoUrlResult>
+
+  /**
+   * Describe the durations this adapter's model accepts. Returns a tagged
+   * union so consumers can render UI / coerce input without provider-specific
+   * knowledge.
+   */
+  availableDurations: () => DurationOptions<TModelDurationByName[TModel]>
+
+  /**
+   * Coerce a raw seconds value to the closest valid duration for this model.
+   * Returns `undefined` for models with no duration field.
+   */
+  snapDuration: (seconds: number) => TModelDurationByName[TModel] | undefined
 }
 
 /**
  * A VideoAdapter with any/unknown type parameters.
  * Useful as a constraint in generic functions and interfaces.
  */
-export type AnyVideoAdapter = VideoAdapter<any, any, any, any>
+export type AnyVideoAdapter = VideoAdapter<any, any, any, any, any, any>
 
 /**
  * Abstract base class for video generation adapters.
@@ -99,11 +147,17 @@ export abstract class BaseVideoAdapter<
     string,
     string
   >,
+  TModelInputModalitiesByName extends ModelInputModalitiesByName =
+    ModelInputModalitiesByName,
+  TModelDurationByName extends Record<string, string | number | undefined> =
+    Record<string, number>,
 > implements VideoAdapter<
   TModel,
   TProviderOptions,
   TModelProviderOptionsByName,
-  TModelSizeByName
+  TModelSizeByName,
+  TModelInputModalitiesByName,
+  TModelDurationByName
 > {
   readonly kind = 'video' as const
   abstract readonly name: string
@@ -114,6 +168,8 @@ export abstract class BaseVideoAdapter<
     providerOptions: TProviderOptions
     modelProviderOptionsByName: TModelProviderOptionsByName
     modelSizeByName: TModelSizeByName
+    modelInputModalitiesByName: TModelInputModalitiesByName
+    modelDurationByName: TModelDurationByName
   }
 
   protected config: VideoAdapterConfig
@@ -124,13 +180,33 @@ export abstract class BaseVideoAdapter<
   }
 
   abstract createVideoJob(
-    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
+    options: VideoGenerationOptions<
+      TProviderOptions,
+      TModelSizeByName[TModel],
+      TModelDurationByName[TModel]
+    >,
   ): Promise<VideoJobResult>
 
   abstract getVideoStatus(jobId: string): Promise<VideoStatusResult>
 
   abstract getVideoUrl(jobId: string): Promise<VideoUrlResult>
 
+  /**
+   * Default implementation returns `{ kind: 'none' }`. Adapters that have
+   * declared their per-model duration map should override this.
+   */
+  availableDurations(): DurationOptions<TModelDurationByName[TModel]> {
+    return { kind: 'none' }
+  }
+
+  /**
+   * Default implementation returns `undefined`. Adapters that have declared
+   * their per-model duration map should override.
+   */
+  snapDuration(_seconds: number): TModelDurationByName[TModel] | undefined {
+    return undefined
+  }
+
   protected generateId(): string {
     return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
   }