@tanstack/ai 0.13.0 → 0.14.0

package/src/activities/generateTranscription/index.ts

@@ -44,7 +44,10 @@ export type TranscriptionProviderOptions<TAdapter> =
  * @template TStream - Whether to stream the output
  */
 export interface TranscriptionActivityOptions<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 > {
   /** The transcription adapter to use (must be created with a model) */
@@ -141,7 +144,10 @@ function createId(prefix: string): string {
  * ```
  */
 export function generateTranscription<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 >(
   options: TranscriptionActivityOptions<TAdapter, TStream>,
@@ -161,7 +167,10 @@ export function generateTranscription<
  * Run non-streaming transcription
  */
 async function runGenerateTranscription<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
 >(
   options: TranscriptionActivityOptions<TAdapter, boolean>,
 ): Promise<TranscriptionResult> {
@@ -213,6 +222,17 @@ async function runGenerateTranscription<
 
     return result
   } catch (error) {
+    const duration = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('transcription:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
     logger.errors('generateTranscription activity failed', {
       error,
       source: 'generateTranscription',
@@ -229,7 +249,10 @@ async function runGenerateTranscription<
  * Create typed options for the generateTranscription() function without executing.
  */
 export function createTranscriptionOptions<
-  TAdapter extends TranscriptionAdapter<string, object>,
+  TAdapter extends TranscriptionAdapter<
+    string,
+    TranscriptionProviderOptions<TAdapter>
+  >,
   TStream extends boolean = false,
 >(
   options: TranscriptionActivityOptions<TAdapter, TStream>,

package/src/activities/generateVideo/index.ts

@@ -8,6 +8,7 @@
  */
 
 import { aiEventClient } from '@tanstack/ai-event-client'
+import { toRunErrorPayload } from '../error-payload'
 import { resolveDebugOption } from '../../logger/resolve'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
@@ -399,21 +400,20 @@ async function* runStreamingVideoGeneration<
     }
 
     throw new Error('Video generation timed out')
-  } catch (error: any) {
+  } catch (error: unknown) {
+    const payload = toRunErrorPayload(error, 'Video generation failed')
     logger.errors('generateVideo activity failed', {
-      error,
+      message: payload.message,
+      code: payload.code,
       source: 'generateVideo',
     })
     yield {
       type: 'RUN_ERROR',
       runId,
       threadId,
-      message: error.message || 'Video generation failed',
-      code: error.code,
-      error: {
-        message: error.message || 'Video generation failed',
-        code: error.code,
-      },
+      message: payload.message,
+      code: payload.code,
+      error: payload,
       timestamp: Date.now(),
     } as StreamChunk
   }

package/src/activities/index.ts

@@ -17,6 +17,7 @@
 import type { AnyTextAdapter } from './chat/adapter'
 import type { AnySummarizeAdapter } from './summarize/adapter'
 import type { AnyImageAdapter } from './generateImage/adapter'
+import type { AnyAudioAdapter } from './generateAudio/adapter'
 import type { AnyVideoAdapter } from './generateVideo/adapter'
 import type { AnyTTSAdapter } from './generateSpeech/adapter'
 import type { AnyTranscriptionAdapter } from './generateTranscription/adapter'
@@ -80,6 +81,25 @@ export {
   type AnyImageAdapter,
 } from './generateImage/adapter'
 
+// ===========================
+// Audio Activity
+// ===========================
+
+export {
+  kind as audioKind,
+  generateAudio,
+  type AudioActivityOptions,
+  type AudioActivityResult,
+  type AudioProviderOptions,
+} from './generateAudio/index'
+
+export {
+  BaseAudioAdapter,
+  type AudioAdapter,
+  type AudioAdapterConfig,
+  type AnyAudioAdapter,
+} from './generateAudio/adapter'
+
 // ===========================
 // Video Activity (Experimental)
 // ===========================
@@ -150,6 +170,7 @@ export type AIAdapter =
   | AnyTextAdapter
   | AnySummarizeAdapter
   | AnyImageAdapter
+  | AnyAudioAdapter
   | AnyVideoAdapter
   | AnyTTSAdapter
   | AnyTranscriptionAdapter
@@ -159,6 +180,7 @@ export type AdapterKind =
   | 'text'
   | 'summarize'
   | 'image'
+  | 'audio'
   | 'video'
   | 'tts'
   | 'transcription'

package/src/activities/stream-generation-result.ts

@@ -5,6 +5,7 @@
  */
 
 import { EventType } from '@ag-ui/core'
+import { toRunErrorPayload } from './error-payload'
 import type { StreamChunk } from '../types'
 
 function createId(prefix: string): string {
@@ -52,18 +53,16 @@ export async function* streamGenerationResult<TResult>(
       finishReason: 'stop',
       timestamp: Date.now(),
     } as StreamChunk
-  } catch (error: any) {
+  } catch (error: unknown) {
+    const payload = toRunErrorPayload(error, 'Generation failed')
     yield {
       type: EventType.RUN_ERROR,
       runId,
       threadId,
-      message: error.message || 'Generation failed',
-      code: error.code,
+      message: payload.message,
+      code: payload.code,
       // Deprecated nested form for backward compatibility
-      error: {
-        message: error.message || 'Generation failed',
-        code: error.code,
-      },
+      error: payload,
       timestamp: Date.now(),
     } as StreamChunk
   }

package/src/adapter-internals.ts

@@ -5,3 +5,4 @@
 export type { ResolvedCategories } from './logger/internal-logger'
 export { InternalLogger } from './logger/internal-logger'
 export { resolveDebugOption } from './logger/resolve'
+export { toRunErrorPayload } from './activities/error-payload'

package/src/index.ts

@@ -3,6 +3,7 @@ export {
   chat,
   summarize,
   generateImage,
+  generateAudio,
   generateVideo,
   getVideoJobStatus,
   generateSpeech,
@@ -13,6 +14,7 @@ export {
 export { createChatOptions } from './activities/chat/index'
 export { createSummarizeOptions } from './activities/summarize/index'
 export { createImageOptions } from './activities/generateImage/index'
+export { createAudioOptions } from './activities/generateAudio/index'
 export { createVideoOptions } from './activities/generateVideo/index'
 export { createSpeechOptions } from './activities/generateSpeech/index'
 export { createTranscriptionOptions } from './activities/generateTranscription/index'
@@ -26,6 +28,8 @@ export type {
   AnyTextAdapter,
   AnySummarizeAdapter,
   SummarizeAdapter,
+  AnyAudioAdapter,
+  AudioAdapter,
   AnyTTSAdapter,
   TTSAdapter,
   AnyTranscriptionAdapter,

package/src/stream-to-response.ts

@@ -1,3 +1,4 @@
+import { toRunErrorPayload } from './activities/error-payload'
 import type { StreamChunk } from './types'
 
 /**
@@ -68,7 +69,7 @@ export function toServerSentEventsStream(
         }
 
         controller.close()
-      } catch (error: any) {
+      } catch (error: unknown) {
         // Don't send error if aborted
         if (abortController?.signal.aborted) {
           controller.close()
@@ -81,10 +82,7 @@ export function toServerSentEventsStream(
             `data: ${JSON.stringify({
               type: 'RUN_ERROR',
               timestamp: Date.now(),
-              error: {
-                message: error.message || 'Unknown error occurred',
-                code: error.code,
-              },
+              error: toRunErrorPayload(error),
             })}\n\n`,
           ),
         )
@@ -190,7 +188,7 @@ export function toHttpStream(
         }
 
         controller.close()
-      } catch (error: any) {
+      } catch (error: unknown) {
         // Don't send error if aborted
         if (abortController?.signal.aborted) {
           controller.close()
@@ -203,10 +201,7 @@ export function toHttpStream(
             `${JSON.stringify({
               type: 'RUN_ERROR',
               timestamp: Date.now(),
-              error: {
-                message: error.message || 'Unknown error occurred',
-                code: error.code,
-              },
+              error: toRunErrorPayload(error),
             })}\n`,
           ),
         )

package/src/types.ts

@@ -1219,14 +1219,28 @@ export interface ImageGenerationOptions<
   logger: InternalLogger
 }
 
+/**
+ * Source of a generated media asset. Exactly one of `url` or `b64Json` is
+ * present; the other is absent. Modeled as a mutually-exclusive union so the
+ * type rejects `{}` and `{ url, b64Json }` together at compile time while
+ * preserving the flat `.url` / `.b64Json` access patterns.
+ */
+export type GeneratedMediaSource =
+  | {
+      /** URL to the generated asset (may be temporary) */
+      url: string
+      b64Json?: never
+    }
+  | {
+      /** Base64-encoded asset data */
+      b64Json: string
+      url?: never
+    }
+
 /**
  * A single generated image
  */
-export interface GeneratedImage {
-  /** Base64-encoded image data */
-  b64Json?: string
-  /** URL to the generated image (may be temporary) */
-  url?: string
+export type GeneratedImage = GeneratedMediaSource & {
   /** Revised prompt used by the model (if applicable) */
   revisedPrompt?: string
 }
@@ -1249,6 +1263,61 @@ export interface ImageGenerationResult {
   }
 }
 
+// ============================================================================
+// Audio Generation Types
+// ============================================================================
+
+/**
+ * Options for audio generation (music, sound effects, etc.).
+ * These are the common options supported across providers.
+ */
+export interface AudioGenerationOptions<
+  TProviderOptions extends object = object,
+> {
+  /** The model to use for audio generation */
+  model: string
+  /** Text description of the desired audio */
+  prompt: string
+  /** Desired duration in seconds */
+  duration?: number
+  /** Model-specific options for audio generation */
+  modelOptions?: TProviderOptions
+  /**
+   * Internal logger threaded from the generateAudio() entry point. Adapters
+   * must call logger.request() before the SDK call and logger.errors() in
+   * catch blocks.
+   */
+  logger: InternalLogger
+}
+
+/**
+ * A single generated audio output
+ */
+export type GeneratedAudio = GeneratedMediaSource & {
+  /** Content type of the audio (e.g., 'audio/wav', 'audio/mp3') */
+  contentType?: string
+  /** Duration of the generated audio in seconds */
+  duration?: number
+}
+
+/**
+ * Result of audio generation
+ */
+export interface AudioGenerationResult {
+  /** Unique identifier for the generation */
+  id: string
+  /** Model used for generation */
+  model: string
+  /** The generated audio */
+  audio: GeneratedAudio
+  /** Token usage information (if available) */
+  usage?: {
+    inputTokens?: number
+    outputTokens?: number
+    totalTokens?: number
+  }
+}
+
 // ============================================================================
 // Video Generation Types (Experimental)
 // ============================================================================