@tanstack/ai 0.54.0 → 0.57.0

package/src/activities/generateSpeech/index.ts

@@ -26,8 +26,14 @@ import {
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
 import type { GenerationMiddleware } from '../middleware/types'
-import type { TTSAdapter } from './adapter'
-import type { StreamChunk, TTSResult } from '../../types'
+import type { TTSAdapter, TTSCapabilities } from './adapter'
+import type {
+  ListVoicesOptions,
+  ListVoicesResult,
+  StreamChunk,
+  TTSResult,
+  TTSTurn,
+} from '../../types'
 
 // ===========================
 // Activity Kind
@@ -60,16 +66,46 @@ export type TTSProviderOptions<TAdapter> = TAdapter extends {
  * @template TAdapter - The TTS adapter type
  * @template TStream - Whether to stream the output
  */
-export interface TTSActivityOptions<
+export type TTSActivityOptions<
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> = TTSActivityOptionsBase<TAdapter, TStream> &
+  (
+    | {
+        /** The text to convert to speech */
+        text: string
+        turns?: undefined
+      }
+    | {
+        text?: undefined
+        /**
+         * Multi-voice dialogue turns, one per line of the script. Mutually
+         * exclusive with `text`.
+         *
+         * Only adapters that declare `capabilities.maxSpeakers` accept these
+         * (ElevenLabs 10 voices, Gemini 2); anything else throws before the
+         * request leaves the process.
+         */
+        turns: Array<TTSTurn>
+      }
+  )
+
+/** Shared half of {@link TTSActivityOptions} — everything except text/turns. */
+interface TTSActivityOptionsBase<
   TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
   TStream extends boolean = false,
 > {
   /** The TTS adapter to use (must be created with a model) */
   adapter: TAdapter & { kind: typeof kind }
-  /** The text to convert to speech */
-  text: string
   /** The voice to use for generation */
   voice?: string
+  /**
+   * Ask for `alignment` (character/word timings) and `segments` (per-turn
+   * spans) on the result. Only adapters that declare
+   * `capabilities.timestamps` accept it — on ElevenLabs it is a different
+   * endpoint, on BytePlus a different request flag, so it cannot be inferred.
+   */
+  timestamps?: boolean
   /** The output audio format */
   format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm'
   /** The speed of the generated audio (0.25 to 4.0) */
@@ -130,6 +166,57 @@ function createId(prefix: string): string {
   return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
 }
 
+/**
+ * Validate the text/turns/timestamps trio against what the adapter declares,
+ * and return the `text` every adapter receives.
+ *
+ * For a dialogue request that text is the turn scripts joined by newlines:
+ * dialogue-aware adapters read `turns` and ignore it, but it keeps `text`
+ * non-optional on the adapter contract and gives the devtools event and the
+ * artifact inputs something truthful to show.
+ */
+function resolveSpeechText(
+  adapter: { name: string; capabilities?: TTSCapabilities },
+  input: { text?: string; turns?: Array<TTSTurn>; timestamps?: boolean },
+): string {
+  const { text, turns, timestamps } = input
+
+  if (timestamps && !adapter.capabilities?.timestamps) {
+    throw new Error(
+      `${adapter.name} cannot return timestamps. Drop \`timestamps: true\` — the result would have no alignment to read.`,
+    )
+  }
+
+  if (turns) {
+    if (text !== undefined) {
+      throw new Error(
+        'generateSpeech() takes either `text` or `turns`, not both.',
+      )
+    }
+    if (turns.length === 0) {
+      throw new Error('generateSpeech() `turns` must not be empty.')
+    }
+    const maxSpeakers = adapter.capabilities?.maxSpeakers
+    if (maxSpeakers === undefined) {
+      throw new Error(
+        `${adapter.name} cannot generate dialogue. Pass \`text\` (and \`voice\`) instead of \`turns\`.`,
+      )
+    }
+    const speakers = new Set(turns.map((turn) => turn.voice)).size
+    if (speakers > maxSpeakers) {
+      throw new Error(
+        `${adapter.name} accepts at most ${maxSpeakers} distinct voice${maxSpeakers === 1 ? '' : 's'} per request; received ${speakers}.`,
+      )
+    }
+    return turns.map((turn) => turn.text).join('\n')
+  }
+
+  if (text === undefined) {
+    throw new Error('generateSpeech() requires either `text` or `turns`.')
+  }
+  return text
+}
+
 // ===========================
 // Activity Implementation
 // ===========================
@@ -200,6 +287,7 @@ async function runGenerateSpeech<
     ...rest
   } = options
   const model = adapter.model
+  const text = resolveSpeechText(adapter, rest)
   const requestId = createId('speech')
   const startTime = Date.now()
   const logger: InternalLogger = resolveDebugOption(options.debug)
@@ -219,7 +307,7 @@ async function runGenerateSpeech<
     model,
     modelOptions: rest.modelOptions,
     artifactInputs: {
-      text: rest.text,
+      text,
       voice: rest.voice,
       format: rest.format,
       speed: rest.speed,
@@ -235,7 +323,7 @@ async function runGenerateSpeech<
     requestId,
     provider: adapter.name,
     model,
-    text: rest.text,
+    text,
     voice: rest.voice,
     format: rest.format,
     speed: rest.speed,
@@ -252,6 +340,7 @@ async function runGenerateSpeech<
     const rawResult = await raceWithAbort(
       adapter.generateSpeech({
         ...rest,
+        text,
         model,
         logger,
         ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
@@ -329,6 +418,53 @@ async function runGenerateSpeech<
   }
 }
 
+// ===========================
+// Voice Catalog
+// ===========================
+
+/**
+ * Options for {@link listVoices}.
+ */
+export interface ListVoicesActivityOptions<
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
+> extends ListVoicesOptions {
+  /** The speech adapter whose catalog to read */
+  adapter: TAdapter & { kind: typeof kind }
+}
+
+/**
+ * List the voices an account can pass to `generateSpeech()`.
+ *
+ * Only providers with a per-account catalog implement this. A provider whose
+ * voices are a fixed list publishes that list as a const in its package, so
+ * import it from there rather than calling this.
+ *
+ * @example Find the voices you created
+ * ```ts
+ * import { listVoices } from '@tanstack/ai'
+ * import { elevenlabsSpeech } from '@tanstack/ai-elevenlabs'
+ *
+ * const { voices } = await listVoices({
+ *   adapter: elevenlabsSpeech('eleven_v3'),
+ *   origins: ['generated', 'cloned'],
+ * })
+ * ```
+ */
+export async function listVoices<
+  TAdapter extends TTSAdapter<string, TTSProviderOptions<TAdapter>>,
+>(options: ListVoicesActivityOptions<TAdapter>): Promise<ListVoicesResult> {
+  const { adapter, ...rest } = options
+
+  const list = adapter.listVoices
+  if (!list) {
+    throw new Error(
+      `The ${adapter.name} speech adapter has no per-account voice catalog to list. Its voices are a fixed set — import the voice list or union its package exports instead (for example \`GeminiTTSVoices\` from @tanstack/ai-gemini, or the \`OpenAITTSVoice\` union from @tanstack/ai-openai).`,
+    )
+  }
+
+  return await list.call(adapter, rest)
+}
+
 // ===========================
 // Options Factory
 // ===========================
@@ -346,5 +482,10 @@ export function createSpeechOptions<
 }
 
 // Re-export adapter types
-export type { TTSAdapter, TTSAdapterConfig, AnyTTSAdapter } from './adapter'
+export type {
+  TTSAdapter,
+  TTSAdapterConfig,
+  TTSCapabilities,
+  AnyTTSAdapter,
+} from './adapter'
 export { BaseTTSAdapter } from './adapter'

package/src/activities/generateVoice/adapter.ts

@@ -0,0 +1,89 @@
+import type { VoiceGenerationOptions, VoiceResult } from '../../types'
+
+/**
+ * Configuration for voice adapter instances
+ */
+export interface VoiceAdapterConfig {
+  apiKey?: string
+  baseUrl?: string
+  timeout?: number
+  maxRetries?: number
+  headers?: Record<string, string>
+}
+
+/**
+ * Voice adapter interface with pre-resolved generics.
+ *
+ * An adapter is created by a provider function: `provider('model')` → `adapter`
+ * All type resolution happens at the provider call site, not in this interface.
+ *
+ * Generic parameters:
+ * - TModel: The specific model name (e.g., 'eleven_ttv_v3')
+ * - TProviderOptions: Provider-specific options (already resolved)
+ */
+export interface VoiceAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  /** Discriminator for adapter kind - used to determine API shape */
+  readonly kind: 'voice'
+  /** Adapter name identifier */
+  readonly name: string
+  /** The model this adapter is configured for */
+  readonly model: TModel
+
+  /**
+   * @internal Type-only properties for inference. Not assigned at runtime.
+   */
+  '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  /**
+   * Create a voice from a text description and/or reference audio
+   */
+  generateVoice: (
+    options: VoiceGenerationOptions<TProviderOptions>,
+  ) => Promise<VoiceResult>
+}
+
+/**
+ * A VoiceAdapter with any/unknown type parameters.
+ * Useful as a constraint in generic functions and interfaces.
+ */
+export type AnyVoiceAdapter = VoiceAdapter<any, any>
+
+/**
+ * Abstract base class for voice creation adapters.
+ * Extend this class to implement a voice adapter for a specific provider.
+ *
+ * Generic parameters match VoiceAdapter - all pre-resolved by the provider function.
+ */
+export abstract class BaseVoiceAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> implements VoiceAdapter<TModel, TProviderOptions> {
+  readonly kind = 'voice' as const
+  abstract readonly name: string
+  readonly model: TModel
+
+  // Type-only property - never assigned at runtime
+  declare '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  protected config: VoiceAdapterConfig
+
+  constructor(model: TModel, config: VoiceAdapterConfig = {}) {
+    this.config = config
+    this.model = model
+  }
+
+  abstract generateVoice(
+    options: VoiceGenerationOptions<TProviderOptions>,
+  ): Promise<VoiceResult>
+
+  protected generateId(): string {
+    return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
+  }
+}

package/src/activities/generateVoice/index.ts

@@ -0,0 +1,371 @@
+/**
+ * Voice Activity
+ *
+ * Creates a reusable voice — either designed from a text description or cloned
+ * from reference audio — and returns voice ids that `generateSpeech()` accepts.
+ * This is a self-contained module with implementation, types, and JSDoc.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import {
+  applyGenerationResultTransforms,
+  createGenerationContext,
+  runGenerationAbort,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware/run'
+import {
+  abortReasonMessage,
+  createActivityAbortControls,
+  isActivityAbortError,
+  raceWithAbort,
+} from '../../utilities/activity-abort'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware/types'
+import type { VoiceAdapter } from './adapter'
+import type { StreamChunk, VoiceResult } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'voice' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from a VoiceAdapter via ~types.
+ */
+export type VoiceProviderOptions<TAdapter> = TAdapter extends {
+  '~types': { providerOptions: infer P extends object }
+}
+  ? P
+  : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the voice activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The voice adapter type
+ * @template TStream - Whether to stream the output
+ */
+export interface VoiceActivityOptions<
+  TAdapter extends VoiceAdapter<string, VoiceProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The voice adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /**
+   * Text description of the voice to create, for design-capable models
+   * (e.g. `'A warm, gravelly narrator in his sixties'`).
+   */
+  prompt?: string
+  /**
+   * Reference audio of the speaker to clone, for clone-capable models.
+   * Accepts a base64 string, base64 data URL, File, Blob, or ArrayBuffer.
+   * Remote URLs are not accepted; read the file and pass the bytes.
+   */
+  referenceAudio?: string | File | Blob | ArrayBuffer
+  /**
+   * Name to store the voice under in the provider's voice library. Check
+   * `saved` on each returned voice to see whether it was actually persisted.
+   */
+  name?: string
+  /** Human-readable description stored alongside the voice */
+  description?: string
+  /** Provider-specific options for voice creation */
+  modelOptions?: VoiceProviderOptions<TAdapter>
+  /**
+   * Whether to stream the generation result.
+   * When true, returns an AsyncIterable<StreamChunk> for streaming transport.
+   * When false or not provided, returns a Promise<VoiceResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
+  /** Stable conversation/thread id for correlating this run when persisted. */
+  threadId?: string
+  /** Stable run id for correlating this run when persisted. */
+  runId?: string
+  /**
+   * Maximum duration of this activity invocation in milliseconds.
+   * No SDK-wide default — choose a value suitable for the provider and job.
+   * Composed with {@link abortSignal}; the first abort wins.
+   */
+  timeout?: number
+  /**
+   * Caller cancellation signal (request disconnects, job/runtime cancellation).
+   * Composed with {@link timeout} into an effective signal forwarded to the
+   * adapter. Request-specific — not stored on global provider client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the voice activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<VoiceResult>
+ */
+export type VoiceActivityResult<TStream extends boolean = false> =
+  TStream extends true ? AsyncIterable<StreamChunk> : Promise<VoiceResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * Voice activity - creates a reusable voice.
+ *
+ * Providers create voices in one of two ways, and some support both: design a
+ * new voice from a text description, or clone one from reference audio. Either
+ * way the result carries voice ids you pass back to `generateSpeech()`.
+ *
+ * @example Design a voice from a description
+ * ```ts
+ * import { generateVoice, generateSpeech } from '@tanstack/ai'
+ * import { elevenlabsVoiceDesign, elevenlabsSpeech } from '@tanstack/ai-elevenlabs'
+ *
+ * const designed = await generateVoice({
+ *   adapter: elevenlabsVoiceDesign('eleven_ttv_v3'),
+ *   prompt: 'A warm, gravelly narrator in his sixties with a slight Irish lilt',
+ * })
+ *
+ * const [preview] = designed.voices
+ * if (!preview) throw new Error('No voice candidates returned')
+ *
+ * const speech = await generateSpeech({
+ *   adapter: elevenlabsSpeech('eleven_v3'),
+ *   text: 'Once upon a time...',
+ *   voice: preview.voiceId,
+ * })
+ * ```
+ *
+ * @example Save the voice to the provider's library
+ * ```ts
+ * const saved = await generateVoice({
+ *   adapter: elevenlabsVoiceDesign('eleven_ttv_v3'),
+ *   prompt: 'A bright, upbeat product demo host',
+ *   name: 'Demo Host',
+ *   description: 'Bright, upbeat, mid-30s',
+ * })
+ * ```
+ */
+export function generateVoice<
+  TAdapter extends VoiceAdapter<string, VoiceProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: VoiceActivityOptions<TAdapter, TStream>,
+): VoiceActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(
+      // Only `runId` is taken from the resolved wire identity — see the
+      // matching note in `generateSpeech`.
+      (resolved) => runGenerateVoice({ ...options, runId: resolved.runId }),
+      options,
+    ) as VoiceActivityResult<TStream>
+  }
+  return runGenerateVoice(options) as VoiceActivityResult<TStream>
+}
+
+/**
+ * Run the core voice generation logic (non-streaming).
+ */
+async function runGenerateVoice<
+  TAdapter extends VoiceAdapter<string, VoiceProviderOptions<TAdapter>>,
+>(options: VoiceActivityOptions<TAdapter, boolean>): Promise<VoiceResult> {
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    threadId,
+    runId,
+    timeout,
+    abortSignal: callerAbortSignal,
+    ...rest
+  } = options
+
+  if (rest.prompt == null && rest.referenceAudio == null) {
+    throw new Error(
+      'generateVoice() requires `prompt` (design a new voice) or `referenceAudio` (clone an existing one).',
+    )
+  }
+
+  const model = adapter.model
+  const requestId = createId('voice')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const abortControls = createActivityAbortControls({
+    timeout,
+    abortSignal: callerAbortSignal,
+  })
+
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'voice',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    artifactInputs: {
+      prompt: rest.prompt,
+      name: rest.name,
+      description: rest.description,
+    },
+    threadId,
+    runId,
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
+  aiEventClient.emit('voice:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    name: rest.name,
+    description: rest.description,
+    hasReferenceAudio: rest.referenceAudio != null,
+    modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+    timestamp: startTime,
+  })
+
+  logger.request(`activity=generateVoice provider=${adapter.name}`, {
+    provider: adapter.name,
+    model,
+  })
+
+  try {
+    const rawResult = await raceWithAbort(
+      adapter.generateVoice({
+        ...rest,
+        model,
+        logger,
+        ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
+      }),
+      abortControls.signal,
+    )
+    abortControls.clear()
+    const result = await applyGenerationResultTransforms(mwCtx, rawResult)
+    const duration = Date.now() - startTime
+
+    aiEventClient.emit('voice:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model,
+      voiceIds: result.voices.map((voice) => voice.voiceId),
+      voiceCount: result.voices.length,
+      previewText: result.previewText,
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+
+    if (result.usage) {
+      aiEventClient.emit('voice:usage', {
+        requestId,
+        model,
+        usage: result.usage,
+        modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+        timestamp: Date.now(),
+      })
+    }
+
+    logger.output(`activity=generateVoice voices=${result.voices.length}`, {
+      voices: result.voices.length,
+    })
+
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration,
+      usage: result.usage,
+    })
+
+    return result
+  } catch (error) {
+    abortControls.clear()
+    const duration = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('voice:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration,
+      modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
+      timestamp: Date.now(),
+    })
+    if (isActivityAbortError(error, abortControls.signal)) {
+      await runGenerationAbort(middleware, mwCtx, {
+        reason: abortReasonMessage(error, abortControls.signal),
+        duration,
+      })
+    } else {
+      await runGenerationError(middleware, mwCtx, {
+        error,
+        duration,
+      })
+    }
+    logger.errors('generateVoice activity failed', {
+      error,
+      source: 'generateVoice',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateVoice() function without executing.
+ */
+export function createVoiceOptions<
+  TAdapter extends VoiceAdapter<string, VoiceProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: VoiceActivityOptions<TAdapter, TStream>,
+): VoiceActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  VoiceAdapter,
+  VoiceAdapterConfig,
+  AnyVoiceAdapter,
+} from './adapter'
+export { BaseVoiceAdapter } from './adapter'