@tanstack/ai 0.13.0 → 0.14.0

package/skills/ai-core/media-generation/SKILL.md

@@ -1,11 +1,12 @@
 ---
 name: ai-core/media-generation
 description: >
-  Image, video, speech (TTS), and transcription generation using
+  Image, audio, video, speech (TTS), and transcription generation using
   activity-specific adapters: generateImage() with openaiImage/geminiImage,
-  generateVideo() with async polling, generateSpeech() with openaiSpeech,
-  generateTranscription() with openaiTranscription. React hooks:
-  useGenerateImage, useGenerateSpeech, useTranscription, useGenerateVideo.
+  generateAudio() with geminiAudio/falAudio, generateVideo() with async
+  polling, generateSpeech() with openaiSpeech, generateTranscription() with
+  openaiTranscription. React hooks: useGenerateImage, useGenerateAudio,
+  useGenerateSpeech, useTranscription, useGenerateVideo.
   TanStack Start server function integration with toServerSentEventsResponse.
 type: sub-skill
 library: tanstack-ai
@@ -14,9 +15,11 @@ sources:
   - 'TanStack/ai:docs/media/generations.md'
   - 'TanStack/ai:docs/media/generation-hooks.md'
   - 'TanStack/ai:docs/media/image-generation.md'
+  - 'TanStack/ai:docs/media/audio-generation.md'
   - 'TanStack/ai:docs/media/video-generation.md'
   - 'TanStack/ai:docs/media/text-to-speech.md'
   - 'TanStack/ai:docs/media/transcription.md'
+  - 'TanStack/ai:docs/advanced/debug-logging.md'
 ---
 
 # Media Generation
@@ -186,7 +189,40 @@ Result shape: `ImageGenerationResult` with `images` array where each entry
 has `b64Json?`, `url?`, and `revisedPrompt?`. OpenAI image URLs expire
 after 1 hour -- download or display immediately.
 
-### 2. Text-to-Speech
+### 2. Audio Generation (Music, Sound Effects)
+
+Distinct from TTS — `generateAudio()` produces non-speech audio content.
+Supported adapters: `geminiAudio` (Lyria 3 Pro / Lyria 3 Clip) and
+`falAudio` (MiniMax Music, DiffRhythm, Stable Audio, ElevenLabs SFX, etc.).
+
+```typescript
+import { generateAudio } from '@tanstack/ai'
+import { falAudio } from '@tanstack/ai-fal'
+
+const result = await generateAudio({
+  adapter: falAudio('fal-ai/diffrhythm'),
+  prompt: 'An upbeat electronic track with synths',
+  duration: 10,
+})
+
+// result.audio.url or result.audio.b64Json (provider-dependent)
+// result.audio.contentType e.g. "audio/mpeg"
+```
+
+Client hook:
+
+```tsx
+import { useGenerateAudio, fetchServerSentEvents } from '@tanstack/ai-react'
+
+const { generate, result, isLoading } = useGenerateAudio({
+  connection: fetchServerSentEvents('/api/generate/audio'),
+})
+
+// Trigger: generate({ prompt: 'Upbeat synths', duration: 10 })
+// Play:    <audio src={result.audio.url} controls />
+```
+
+### 3. Text-to-Speech
 
 Adapter: `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-audio-preview).
 
@@ -220,7 +256,7 @@ const { generate, result, isLoading } = useGenerateSpeech({
 // Play:   <audio src={`data:audio/${result.format};base64,${result.audio}`} controls />
 ```
 
-### 3. Audio Transcription
+### 4. Audio Transcription
 
 Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
 gpt-4o-mini-transcribe).
@@ -257,7 +293,7 @@ const { generate, result, isLoading } = useTranscription({
 // Trigger: generate({ audio: dataUrl, language: 'en' })
 ```
 
-### 4. Video Generation (Experimental -- async polling)
+### 5. Video Generation (Experimental -- async polling)
 
 Video generation uses a jobs/polling architecture. The server creates a job,
 polls for status, and streams updates to the client.
@@ -454,12 +490,116 @@ for (const img of result.images) {
 Not all generation activities support streaming. Passing `stream: true` to
 an activity that does not support it may hang or produce unexpected results.
 Check the activity documentation before enabling streaming. All built-in
-activities (`generateImage`, `generateSpeech`, `generateTranscription`,
-`generateVideo`, `summarize`) support `stream: true`, but custom
-`useGeneration` setups may not.
+activities (`generateImage`, `generateAudio`, `generateSpeech`,
+`generateTranscription`, `generateVideo`, `summarize`) support `stream: true`,
+but custom `useGeneration` setups may not.
 
 > Source: docs/media/generations.md.
 
+### e. HIGH: Passing `responseMimeType` or `negativePrompt` to Gemini Lyria
+
+Gemini's `GenerateContentConfig` (used by Lyria 3 Pro / Lyria 3 Clip) does
+**not** support `responseMimeType` or `negativePrompt`. Lyria 3 Clip always
+returns 30-second `audio/mp3`; Lyria 3 Pro returns `audio/mp3`. These fields
+are not in `GeminiAudioProviderOptions` — don't reach for them via `as any`.
+
+```typescript
+// WRONG — both fields are silently ignored or rejected by the SDK
+generateAudio({
+  adapter: geminiAudio('lyria-3-pro-preview'),
+  prompt: 'ambient piano',
+  modelOptions: {
+    responseMimeType: 'audio/wav', // unsupported
+    negativePrompt: 'vocals', // unsupported
+  } as any,
+})
+
+// CORRECT — shape the prompt itself for what you want
+generateAudio({
+  adapter: geminiAudio('lyria-3-pro-preview'),
+  prompt: 'ambient piano, no vocals',
+})
+```
+
+> Source: Gemini API `GenerateContentConfig` type; docs/media/audio-generation.md.
+
+### f. MEDIUM: Passing `duration` to Lyria expecting it to control length
+
+Lyria 3 Clip is fixed at 30 seconds — the `duration` option is ignored on
+that model. Lyria 3 Pro accepts duration via natural-language in the
+**prompt** ("2-minute ambient track with a 30-second build"), not via the
+`duration` field. `duration` works for fal audio models (mapped to each
+model's native field like `music_length_ms` or `seconds_total`), but not
+for Lyria.
+
+```typescript
+// For Lyria: put length guidance in the prompt
+generateAudio({
+  adapter: geminiAudio('lyria-3-pro-preview'),
+  prompt: 'A 2-minute ambient piano piece with gentle strings',
+  // duration: 120  // ← does nothing; rely on the prompt
+})
+
+// For fal: duration works and is translated per-model
+generateAudio({
+  adapter: falAudio('fal-ai/minimax-music/v2'),
+  prompt: 'upbeat synth melody',
+  duration: 60, // → music_length_ms: 60_000
+})
+```
+
+> Source: Google Lyria 3 docs; docs/media/audio-generation.md.
+
+### g. MEDIUM: Gemini TTS multi-speaker with 0 or 3+ speakers
+
+`multiSpeakerVoiceConfig.speakerVoiceConfigs` is validated to be length 1 or 2. Passing an empty array or three+ entries throws at the adapter boundary
+(not at Gemini's API) with a clear error. Don't try to work around it with
+`as any`.
+
+```typescript
+generateSpeech({
+  adapter: geminiSpeech('gemini-2.5-pro-preview-tts'),
+  text: '[Alice] Hi. [Bob] Hello!',
+  modelOptions: {
+    multiSpeakerVoiceConfig: {
+      speakerVoiceConfigs: [
+        {
+          speaker: 'Alice',
+          voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } },
+        },
+        {
+          speaker: 'Bob',
+          voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Puck' } },
+        },
+      ],
+    },
+  },
+})
+```
+
+> Source: Gemini TTS adapter validation; CodeRabbit review of PR #463.
+
+### h. LOW: Writing a logging middleware to see media chunks flow through
+
+Every media activity — `generateAudio`, `generateSpeech`,
+`generateTranscription`, `generateImage`, `generateVideo` — accepts the
+same `debug?: DebugOption` option that `chat()` does. Reach for `debug`
+instead of wiring up logging middleware.
+
+```typescript
+// When a speech generation sounds wrong or a transcription returns garbage
+generateSpeech({
+  adapter: openaiSpeech('tts-1'),
+  text: 'Hello',
+  debug: { provider: true, output: true }, // raw SDK chunks + yielded chunks
+})
+```
+
+See the `ai-core/debug-logging` sub-skill for full details on categories
+and piping into a custom logger.
+
+> Source: docs/advanced/debug-logging.md.
+
 ---
 
 ## Cross-References
@@ -469,3 +609,7 @@ activities (`generateImage`, `generateSpeech`, `generateTranscription`,
   images, `openaiSpeech` for speech, `openaiTranscription` for transcription,
   `openaiVideo` for video). The adapter-configuration skill covers provider
   setup, API keys, and model selection.
+- See also: **ai-core/debug-logging/SKILL.md** -- When a media request
+  returns unexpected output or fails mid-stream, toggle `debug: true` on
+  any `generate*()` call to see request metadata, raw provider chunks, and
+  errors. Covers per-category toggling and piping into pino/winston.

package/src/activities/error-payload.ts

@@ -0,0 +1,35 @@
+/**
+ * Shared error-narrowing helper for activities that convert thrown values
+ * into structured `RUN_ERROR` events.
+ *
+ * Accepts Error instances, objects with string-ish `message`/`code`, or bare
+ * strings; always returns a shape safe to serialize. Never leaks the full
+ * error object (which may carry request/response state from an SDK).
+ */
+export function toRunErrorPayload(
+  error: unknown,
+  fallbackMessage = 'Unknown error occurred',
+): { message: string; code: string | undefined } {
+  if (error instanceof Error) {
+    const codeField = (error as Error & { code?: unknown }).code
+    return {
+      message: error.message || fallbackMessage,
+      code: typeof codeField === 'string' ? codeField : undefined,
+    }
+  }
+  if (typeof error === 'object' && error !== null) {
+    const messageField = (error as { message?: unknown }).message
+    const codeField = (error as { code?: unknown }).code
+    return {
+      message:
+        typeof messageField === 'string' && messageField.length > 0
+          ? messageField
+          : fallbackMessage,
+      code: typeof codeField === 'string' ? codeField : undefined,
+    }
+  }
+  if (typeof error === 'string' && error.length > 0) {
+    return { message: error, code: undefined }
+  }
+  return { message: fallbackMessage, code: undefined }
+}

package/src/activities/generateAudio/adapter.ts

@@ -0,0 +1,89 @@
+import type { AudioGenerationOptions, AudioGenerationResult } from '../../types'
+
+/**
+ * Configuration for audio generation adapter instances
+ */
+export interface AudioAdapterConfig {
+  apiKey?: string
+  baseUrl?: string
+  timeout?: number
+  maxRetries?: number
+  headers?: Record<string, string>
+}
+
+/**
+ * Audio generation adapter interface with pre-resolved generics.
+ *
+ * An adapter is created by a provider function: `provider('model')` → `adapter`
+ * All type resolution happens at the provider call site, not in this interface.
+ *
+ * Generic parameters:
+ * - TModel: The specific model name (e.g., 'fal-ai/diffrhythm')
+ * - TProviderOptions: Provider-specific options (already resolved)
+ */
+export interface AudioAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  /** Discriminator for adapter kind - used to determine API shape */
+  readonly kind: 'audio'
+  /** Adapter name identifier */
+  readonly name: string
+  /** The model this adapter is configured for */
+  readonly model: TModel
+
+  /**
+   * @internal Type-only properties for inference. Not assigned at runtime.
+   */
+  '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  /**
+   * Generate audio from a text prompt
+   */
+  generateAudio: (
+    options: AudioGenerationOptions<TProviderOptions>,
+  ) => Promise<AudioGenerationResult>
+}
+
+/**
+ * An AudioAdapter with any/unknown type parameters.
+ * Useful as a constraint in generic functions and interfaces.
+ */
+export type AnyAudioAdapter = AudioAdapter<any, any>
+
+/**
+ * Abstract base class for audio generation adapters.
+ * Extend this class to implement an audio adapter for a specific provider.
+ *
+ * Generic parameters match AudioAdapter - all pre-resolved by the provider function.
+ */
+export abstract class BaseAudioAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> implements AudioAdapter<TModel, TProviderOptions> {
+  readonly kind = 'audio' as const
+  abstract readonly name: string
+  readonly model: TModel
+
+  // Type-only property - never assigned at runtime
+  declare '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  protected config: AudioAdapterConfig
+
+  constructor(model: TModel, config: AudioAdapterConfig = {}) {
+    this.config = config
+    this.model = model
+  }
+
+  abstract generateAudio(
+    options: AudioGenerationOptions<TProviderOptions>,
+  ): Promise<AudioGenerationResult>
+
+  protected generateId(): string {
+    return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
+  }
+}

package/src/activities/generateAudio/index.ts

@@ -0,0 +1,224 @@
+/**
+ * Audio Generation Activity
+ *
+ * Generates audio (music, sound effects, etc.) from text prompts.
+ * This is a self-contained module with implementation, types, and JSDoc.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { AudioAdapter } from './adapter'
+import type { AudioGenerationResult, StreamChunk } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'audio' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from an AudioAdapter via ~types.
+ */
+export type AudioProviderOptions<TAdapter> =
+  TAdapter extends AudioAdapter<any, any>
+    ? TAdapter['~types']['providerOptions']
+    : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the audio generation activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The audio adapter type
+ * @template TStream - Whether to stream the output
+ */
+export interface AudioActivityOptions<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The audio adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /** Text description of the desired audio */
+  prompt: string
+  /** Desired duration in seconds */
+  duration?: number
+  /** Provider-specific options for audio generation */
+  modelOptions?: AudioProviderOptions<TAdapter>
+  /**
+   * Whether to stream the generation result.
+   * When true, returns an AsyncIterable<StreamChunk> for streaming transport.
+   * When false or not provided, returns a Promise<AudioGenerationResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the audio generation activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<AudioGenerationResult>
+ */
+export type AudioActivityResult<TStream extends boolean = false> =
+  TStream extends true
+    ? AsyncIterable<StreamChunk>
+    : Promise<AudioGenerationResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * Audio generation activity - generates audio from text prompts.
+ *
+ * Uses AI models to create music, sound effects, and other audio content.
+ *
+ * @example Generate music from a prompt
+ * ```ts
+ * import { generateAudio } from '@tanstack/ai'
+ * import { falAudio } from '@tanstack/ai-fal'
+ *
+ * const result = await generateAudio({
+ *   adapter: falAudio('fal-ai/diffrhythm'),
+ *   prompt: 'An upbeat electronic track with synths',
+ *   duration: 10
+ * })
+ *
+ * console.log(result.audio.url) // URL to generated audio
+ * ```
+ */
+export function generateAudio<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: AudioActivityOptions<TAdapter, TStream>,
+): AudioActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(() =>
+      runGenerateAudio(options),
+    ) as AudioActivityResult<TStream>
+  }
+  return runGenerateAudio(options) as AudioActivityResult<TStream>
+}
+
+/**
+ * Run the core audio generation logic (non-streaming).
+ */
+async function runGenerateAudio<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+>(
+  options: AudioActivityOptions<TAdapter, boolean>,
+): Promise<AudioGenerationResult> {
+  const { adapter, stream: _stream, debug: _debug, ...rest } = options
+  const model = adapter.model
+  const requestId = createId('audio')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const providerName =
+    (adapter as { name?: string; provider?: string }).provider ??
+    (adapter as { name?: string }).name ??
+    'unknown'
+
+  aiEventClient.emit('audio:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    duration: rest.duration,
+    modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+    timestamp: startTime,
+  })
+
+  logger.request(`activity=generateAudio provider=${providerName}`, {
+    provider: providerName,
+    model,
+  })
+
+  try {
+    const result = await adapter.generateAudio({ ...rest, model, logger })
+    const elapsedMs = Date.now() - startTime
+
+    aiEventClient.emit('audio:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model,
+      audio: result.audio,
+      duration: elapsedMs,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+
+    logger.output(`activity=generateAudio provider=${providerName}`, {
+      contentType: result.audio.contentType,
+      audioDuration: result.audio.duration,
+    })
+
+    return result
+  } catch (error) {
+    const elapsedMs = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('audio:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration: elapsedMs,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+    logger.errors('generateAudio activity failed', {
+      error,
+      source: 'generateAudio',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateAudio() function without executing.
+ */
+export function createAudioOptions<
+  TAdapter extends AudioAdapter<string, AudioProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: AudioActivityOptions<TAdapter, TStream>,
+): AudioActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  AudioAdapter,
+  AudioAdapterConfig,
+  AnyAudioAdapter,
+} from './adapter'
+export { BaseAudioAdapter } from './adapter'

package/src/activities/generateImage/adapter.ts

@@ -96,7 +96,7 @@ export abstract class BaseImageAdapter<
 
   protected config: ImageAdapterConfig
 
-  constructor(config: ImageAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: ImageAdapterConfig = {}) {
     this.config = config
     this.model = model
   }

package/src/activities/generateSpeech/adapter.ts

@@ -72,7 +72,7 @@ export abstract class BaseTTSAdapter<
 
   protected config: TTSAdapterConfig
 
-  constructor(config: TTSAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: TTSAdapterConfig = {}) {
     this.config = config
     this.model = model
   }

package/src/activities/generateSpeech/index.ts

@@ -44,7 +44,7 @@ export type TTSProviderOptions<TAdapter> =
  * @template TStream - Whether to stream the output
  */
 export interface TTSActivityOptions<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 > {
   /** The TTS adapter to use (must be created with a model) */
@@ -126,7 +126,7 @@ function createId(prefix: string): string {
  * ```
  */
 export function generateSpeech<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 >(options: TTSActivityOptions<TAdapter, TStream>): TTSActivityResult<TStream> {
   if (options.stream) {
@@ -140,9 +140,9 @@ export function generateSpeech<
 /**
  * Run the core TTS generation logic (non-streaming).
  */
-async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
-  options: TTSActivityOptions<TAdapter, boolean>,
-): Promise<TTSResult> {
+async function runGenerateSpeech<
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
+>(options: TTSActivityOptions<TAdapter, boolean>): Promise<TTSResult> {
   const { adapter, stream: _stream, debug: _debug, ...rest } = options
   const model = adapter.model
   const requestId = createId('speech')
@@ -172,7 +172,6 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
 
   try {
     const result = await adapter.generateSpeech({ ...rest, model, logger })
-
     const duration = Date.now() - startTime
 
     aiEventClient.emit('speech:request:completed', {
@@ -195,6 +194,17 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
 
     return result
   } catch (error) {
+    const duration = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('speech:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
     logger.errors('generateSpeech activity failed', {
       error,
       source: 'generateSpeech',
@@ -211,7 +221,7 @@ async function runGenerateSpeech<TAdapter extends TTSAdapter<string, object>>(
  * Create typed options for the generateSpeech() function without executing.
  */
 export function createSpeechOptions<
-  TAdapter extends TTSAdapter<string, object>,
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 >(
   options: TTSActivityOptions<TAdapter, TStream>,

package/src/activities/generateTranscription/adapter.ts

@@ -74,7 +74,7 @@ export abstract class BaseTranscriptionAdapter<
 
   protected config: TranscriptionAdapterConfig
 
-  constructor(config: TranscriptionAdapterConfig = {}, model: TModel) {
+  constructor(model: TModel, config: TranscriptionAdapterConfig = {}) {
     this.config = config
     this.model = model
   }