@tanstack/ai 0.13.0 → 0.15.0

package/src/activities/generateAudio/index.ts

@@ -0,0 +1,224 @@
+/**
+ * Audio Generation Activity
+ *
+ * Generates audio (music, sound effects, etc.) from text prompts.
+ * This is a self-contained module with implementation, types, and JSDoc.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { AudioAdapter } from './adapter'
+import type { AudioGenerationResult, StreamChunk } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'audio' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from an AudioAdapter via ~types.
+ */
+export type AudioProviderOptions<TAdapter> =
+  TAdapter extends AudioAdapter<any, any>
+    ? TAdapter['~types']['providerOptions']
+    : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the audio generation activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The audio adapter type
+ * @template TStream - Whether to stream the output
+ */
+export interface AudioActivityOptions<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The audio adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /** Text description of the desired audio */
+  prompt: string
+  /** Desired duration in seconds */
+  duration?: number
+  /** Provider-specific options for audio generation */
+  modelOptions?: AudioProviderOptions<TAdapter>
+  /**
+   * Whether to stream the generation result.
+   * When true, returns an AsyncIterable<StreamChunk> for streaming transport.
+   * When false or not provided, returns a Promise<AudioGenerationResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the audio generation activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<AudioGenerationResult>
+ */
+export type AudioActivityResult<TStream extends boolean = false> =
+  TStream extends true
+    ? AsyncIterable<StreamChunk>
+    : Promise<AudioGenerationResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * Audio generation activity - generates audio from text prompts.
+ *
+ * Uses AI models to create music, sound effects, and other audio content.
+ *
+ * @example Generate music from a prompt
+ * ```ts
+ * import { generateAudio } from '@tanstack/ai'
+ * import { falAudio } from '@tanstack/ai-fal'
+ *
+ * const result = await generateAudio({
+ *   adapter: falAudio('fal-ai/diffrhythm'),
+ *   prompt: 'An upbeat electronic track with synths',
+ *   duration: 10
+ * })
+ *
+ * console.log(result.audio.url) // URL to generated audio
+ * ```
+ */
+export function generateAudio<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: AudioActivityOptions<TAdapter, TStream>,
+): AudioActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(() =>
+      runGenerateAudio(options),
+    ) as AudioActivityResult<TStream>
+  }
+  return runGenerateAudio(options) as AudioActivityResult<TStream>
+}
+
+/**
+ * Run the core audio generation logic (non-streaming).
+ */
+async function runGenerateAudio<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+>(
+  options: AudioActivityOptions<TAdapter, boolean>,
+): Promise<AudioGenerationResult> {
+  const { adapter, stream: _stream, debug: _debug, ...rest } = options
+  const model = adapter.model
+  const requestId = createId('audio')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const providerName =
+    (adapter as { name?: string; provider?: string }).provider ??
+    (adapter as { name?: string }).name ??
+    'unknown'
+
+  aiEventClient.emit('audio:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    duration: rest.duration,
+    modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+    timestamp: startTime,
+  })
+
+  logger.request(`activity=generateAudio provider=${providerName}`, {
+    provider: providerName,
+    model,
+  })
+
+  try {
+    const result = await adapter.generateAudio({ ...rest, model, logger })
+    const elapsedMs = Date.now() - startTime
+
+    aiEventClient.emit('audio:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model,
+      audio: result.audio,
+      duration: elapsedMs,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+
+    logger.output(`activity=generateAudio provider=${providerName}`, {
+      contentType: result.audio.contentType,
+      audioDuration: result.audio.duration,
+    })
+
+    return result
+  } catch (error) {
+    const elapsedMs = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('audio:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration: elapsedMs,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+    logger.errors('generateAudio activity failed', {
+      error,
+      source: 'generateAudio',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateAudio() function without executing.
+ */
+export function createAudioOptions<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: AudioActivityOptions<TAdapter, TStream>,
+): AudioActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  AudioAdapter,
+  AudioAdapterConfig,
+  AnyAudioAdapter,
+} from './adapter'
+export { BaseAudioAdapter } from './adapter'

package/src/activities/generateImage/adapter.ts

@@ -96,7 +96,7 @@ export abstract class BaseImageAdapter<
 
   protected config: ImageAdapterConfig
 
-  constructor(config: ImageAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: ImageAdapterConfig = {}) {
     this.config = config
     this.model = model
   }

package/src/activities/generateSpeech/adapter.ts

@@ -72,7 +72,7 @@ export abstract class BaseTTSAdapter<
 
   protected config: TTSAdapterConfig
 
-  constructor(config: TTSAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: TTSAdapterConfig = {}) {
     this.config = config
     this.model = model
   }

package/src/activities/generateSpeech/index.ts

@@ -44,7 +44,7 @@ export type TTSProviderOptions<TAdapter> =
  * @template TStream - Whether to stream the output
  */
 export interface TTSActivityOptions<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 > {
   /** The TTS adapter to use (must be created with a model) */
@@ -126,7 +126,7 @@ function createId(prefix: string): string {
  * ```
  */
 export function generateSpeech<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 >(options: TTSActivityOptions<TAdapter, TStream>): TTSActivityResult<TStream> {
   if (options.stream) {
@@ -140,9 +140,9 @@ export function generateSpeech<
 /**
  * Run the core TTS generation logic (non-streaming).
  */
-async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
-  options: TTSActivityOptions<TAdapter, boolean>,
-): Promise<TTSResult> {
+async function runGenerateSpeech<
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
+>(options: TTSActivityOptions<TAdapter, boolean>): Promise<TTSResult> {
   const { adapter, stream: _stream, debug: _debug, ...rest } = options
   const model = adapter.model
   const requestId = createId('speech')
@@ -172,7 +172,6 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
 
   try {
     const result = await adapter.generateSpeech({ ...rest, model, logger })
-
     const duration = Date.now() - startTime
 
     aiEventClient.emit('speech:request:completed', {
@@ -195,6 +194,17 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
 
     return result
   } catch (error) {
+    const duration = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('speech:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
     logger.errors('generateSpeech activity failed', {
       error,
       source: 'generateSpeech',
@@ -211,7 +221,7 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
  * Create typed options for the generateSpeech() function without executing.
  */
 export function createSpeechOptions<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 >(
   options: TTSActivityOptions<TAdapter, TStream>,

package/src/activities/generateTranscription/adapter.ts

@@ -74,7 +74,7 @@ export abstract class BaseTranscriptionAdapter<
 
   protected config: TranscriptionAdapterConfig
 
-  constructor(config: TranscriptionAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: TranscriptionAdapterConfig = {}) {
     this.config = config
     this.model = model
   }

package/src/activities/generateTranscription/index.ts

@@ -44,7 +44,10 @@ export type TranscriptionProviderOptions<TAdapter> =
  * @template TStream - Whether to stream the output
  */
 export interface TranscriptionActivityOptions<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 > {
   /** The transcription adapter to use (must be created with a model) */
@@ -141,7 +144,10 @@ function createId(prefix: string): string {
  * ```
  */
 export function generateTranscription<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 >(
   options: TranscriptionActivityOptions<TAdapter, TStream>,
@@ -161,7 +167,10 @@ export function generateTranscription<
  * Run non-streaming transcription
  */
 async function runGenerateTranscription<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
 >(
   options: TranscriptionActivityOptions<TAdapter, boolean>,
 ): Promise<TranscriptionResult> {
@@ -213,6 +222,17 @@ async function runGenerateTranscription<
 
     return result
   } catch (error) {
+    const duration = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('transcription:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
     logger.errors('generateTranscription activity failed', {
       error,
       source: 'generateTranscription',
@@ -229,7 +249,10 @@ async function runGenerateTranscription<
  * Create typed options for the generateTranscription() function without executing.
  */
 export function createTranscriptionOptions<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 >(
   options: TranscriptionActivityOptions<TAdapter, TStream>,

package/src/activities/generateVideo/index.ts

@@ -8,6 +8,7 @@
  */
 
 import { aiEventClient } from '@tanstack/ai-event-client'
+import { toRunErrorPayload } from '../error-payload'
 import { resolveDebugOption } from '../../logger/resolve'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
@@ -399,21 +400,20 @@ async function* runStreamingVideoGeneration<
     }
 
     throw new Error('Video generation timed out')
-  } catch (error: any) {
+  } catch (error: unknown) {
+    const payload = toRunErrorPayload(error, 'Video generation failed')
     logger.errors('generateVideo activity failed', {
-      error,
+      message: payload.message,
+      code: payload.code,
       source: 'generateVideo',
     })
     yield {
       type: 'RUN_ERROR',
       runId,
       threadId,
-      message: error.message || 'Video generation failed',
-      code: error.code,
-      error: {
-        message: error.message || 'Video generation failed',
-        code: error.code,
-      },
+      message: payload.message,
+      code: payload.code,
+      error: payload,
       timestamp: Date.now(),
     } as StreamChunk
   }

package/src/activities/index.ts

@@ -17,6 +17,7 @@
 import type { AnyTextAdapter } from './chat/adapter'
 import type { AnySummarizeAdapter } from './summarize/adapter'
 import type { AnyImageAdapter } from './generateImage/adapter'
+import type { AnyAudioAdapter } from './generateAudio/adapter'
 import type { AnyVideoAdapter } from './generateVideo/adapter'
 import type { AnyTTSAdapter } from './generateSpeech/adapter'
 import type { AnyTranscriptionAdapter } from './generateTranscription/adapter'
@@ -80,6 +81,25 @@ export {
   type AnyImageAdapter,
 } from './generateImage/adapter'
 
+// ===========================
+// Audio Activity
+// ===========================
+
+export {
+  kind as audioKind,
+  generateAudio,
+  type AudioActivityOptions,
+  type AudioActivityResult,
+  type AudioProviderOptions,
+} from './generateAudio/index'
+
+export {
+  BaseAudioAdapter,
+  type AudioAdapter,
+  type AudioAdapterConfig,
+  type AnyAudioAdapter,
+} from './generateAudio/adapter'
+
 // ===========================
 // Video Activity (Experimental)
 // ===========================
@@ -150,6 +170,7 @@ export type AIAdapter =
   | AnyTextAdapter
   | AnySummarizeAdapter
   | AnyImageAdapter
+  | AnyAudioAdapter
   | AnyVideoAdapter
   | AnyTTSAdapter
   | AnyTranscriptionAdapter
@@ -159,6 +180,7 @@ export type AdapterKind =
   | 'text'
   | 'summarize'
   | 'image'
+  | 'audio'
   | 'video'
   | 'tts'
   | 'transcription'

package/src/activities/stream-generation-result.ts

@@ -5,6 +5,7 @@
  */
 
 import { EventType } from '@ag-ui/core'
+import { toRunErrorPayload } from './error-payload'
 import type { StreamChunk } from '../types'
 
 function createId(prefix: string): string {
@@ -52,18 +53,16 @@ export async function* streamGenerationResult<TResult>(
       finishReason: 'stop',
       timestamp: Date.now(),
     } as StreamChunk
-  } catch (error: any) {
+  } catch (error: unknown) {
+    const payload = toRunErrorPayload(error, 'Generation failed')
     yield {
       type: EventType.RUN_ERROR,
       runId,
       threadId,
-      message: error.message || 'Generation failed',
-      code: error.code,
+      message: payload.message,
+      code: payload.code,
       // Deprecated nested form for backward compatibility
-      error: {
-        message: error.message || 'Generation failed',
-        code: error.code,
-      },
+      error: payload,
       timestamp: Date.now(),
     } as StreamChunk
   }

package/src/adapter-internals.ts

@@ -5,3 +5,4 @@
 export type { ResolvedCategories } from './logger/internal-logger'
 export { InternalLogger } from './logger/internal-logger'
 export { resolveDebugOption } from './logger/resolve'
+export { toRunErrorPayload } from './activities/error-payload'

package/src/index.ts

@@ -3,6 +3,7 @@ export {
   chat,
   summarize,
   generateImage,
+  generateAudio,
   generateVideo,
   getVideoJobStatus,
   generateSpeech,
@@ -13,6 +14,7 @@ export {
 export { createChatOptions } from './activities/chat/index'
 export { createSummarizeOptions } from './activities/summarize/index'
 export { createImageOptions } from './activities/generateImage/index'
+export { createAudioOptions } from './activities/generateAudio/index'
 export { createVideoOptions } from './activities/generateVideo/index'
 export { createSpeechOptions } from './activities/generateSpeech/index'
 export { createTranscriptionOptions } from './activities/generateTranscription/index'
@@ -26,6 +28,8 @@ export type {
   AnyTextAdapter,
   AnySummarizeAdapter,
   SummarizeAdapter,
+  AnyAudioAdapter,
+  AudioAdapter,
   AnyTTSAdapter,
   TTSAdapter,
   AnyTranscriptionAdapter,

package/src/middlewares/index.ts

@@ -11,3 +11,8 @@ export {
   type ContentGuardRule,
   type ContentFilteredInfo,
 } from './content-guard'
+
+// otelMiddleware is exported from the dedicated subpath
+// `@tanstack/ai/middlewares/otel` so that importing the main middlewares barrel
+// does not eagerly require `@opentelemetry/api` (which is an optional peer
+// dependency).