@tanstack/ai 0.55.0 → 0.58.0

package/src/types.ts

@@ -629,6 +629,22 @@ type RuntimeContextField<TContext> =
         context: TContext
       }
 
+/**
+ * Options for a single `emitCustomEvent` call, on both the tool-execution and
+ * middleware contexts.
+ */
+export interface EmitCustomEventOptions {
+  /**
+   * Keep this event in the durability batch with later chunks.
+   * CUSTOM events flush as soon as they are emitted, so a progress
+   * indicator can render at emit time. Pass `{ batch: true }` for a
+   * high-volume stream that should share appends with later output.
+   * `process.stdout`, `process.stderr`, `sandbox.file`, and
+   * `sandbox.file.diff` already batch.
+   */
+  batch?: boolean
+}
+
 /**
  * Context passed to tool execute functions, providing capabilities like
  * emitting custom events during execution.
@@ -649,6 +665,8 @@ export type ToolExecutionContext<TContext = unknown> =
      *
      * @param eventName - Name of the custom event
      * @param value - Event payload value
+     * @param options - Pass `{ batch: true }` to keep this event in the
+     *   durability batch instead of flushing it immediately
      *
      * @example
      * ```ts
@@ -661,7 +679,11 @@ export type ToolExecutionContext<TContext = unknown> =
      * })
      * ```
      */
-    emitCustomEvent: (eventName: string, value: Record<string, any>) => void
+    emitCustomEvent: (
+      eventName: string,
+      value: Record<string, any>,
+      options?: EmitCustomEventOptions,
+    ) => void
   }
 
 export type ToolExecuteFunction<
@@ -2350,6 +2372,56 @@ export interface LiveVideoGenerationResult {
 // Text-to-Speech (TTS) Types
 // ============================================================================
 
+/**
+ * One turn of a multi-voice dialogue request.
+ *
+ * Providers that expose a dedicated dialogue endpoint (ElevenLabs
+ * `textToDialogue`, Gemini multi-speaker) take these natively instead of a
+ * single `text` + `voice` pair.
+ */
+export interface TTSTurn {
+  /** The text this voice speaks. */
+  text: string
+  /** Provider voice id (ElevenLabs) or voice name (Gemini) for this turn. */
+  voice: string
+}
+
+/**
+ * Timings for the generated audio, returned when `timestamps: true` was
+ * requested and the adapter declares `capabilities.timestamps`.
+ *
+ * Granularity differs per provider — ElevenLabs reports characters, BytePlus
+ * reports words — so `unit` says which, and the three arrays are parallel.
+ * All times are **seconds**; adapters convert.
+ */
+export interface TTSAlignment {
+  /** Granularity of each entry. */
+  unit: 'character' | 'word'
+  /** Entry text, in audio order. */
+  texts: Array<string>
+  /** Start of each entry in seconds. Same length as `texts`. */
+  startSeconds: Array<number>
+  /** End of each entry in seconds. Same length as `texts`. */
+  endSeconds: Array<number>
+}
+
+/**
+ * A stretch of audio attributable to one turn (multi-voice) or one utterance
+ * (single voice). This is what tells a consumer which turn is where.
+ */
+export interface TTSSegment {
+  /** Start of the segment in seconds. */
+  startSeconds: number
+  /** End of the segment in seconds. */
+  endSeconds: number
+  /** Index into the request's `turns`, when the provider reports it. */
+  turnIndex?: number
+  /** Voice heard in this segment, when the provider reports it. */
+  voice?: string
+  /** Text spoken in this segment, when the provider reports it. */
+  text?: string
+}
+
 /**
  * Options for text-to-speech generation.
  * These are the common options supported across providers.
@@ -2357,8 +2429,24 @@ export interface LiveVideoGenerationResult {
 export interface TTSOptions<TProviderOptions extends object = object> {
   /** The model to use for TTS generation */
   model: string
-  /** The text to convert to speech */
+  /**
+   * The text to convert to speech. When the caller passed `turns`, the
+   * activity fills this with the turn texts joined by newlines so adapters
+   * that only read `text` still receive the full script.
+   */
   text: string
+  /**
+   * Multi-voice dialogue turns, when the caller asked for dialogue. Only
+   * adapters that declare `capabilities.maxSpeakers` ever see this — the
+   * activity rejects `turns` for the rest.
+   */
+  turns?: Array<TTSTurn>
+  /**
+   * Ask for `alignment` / `segments` on the result. Rejected by the activity
+   * unless the adapter declares `capabilities.timestamps`, because on some
+   * providers this is a different endpoint rather than free metadata.
+   */
+  timestamps?: boolean
   /** The voice to use for generation */
   voice?: string
   /** The output audio format */
@@ -2393,8 +2481,18 @@ export interface TTSResult {
   audio: string
   /** Audio format of the generated audio */
   format: string
-  /** Duration of the audio in seconds, if available */
+  /** Duration of the audio file in seconds, if available */
   duration?: number
+  /**
+   * Character- or word-level timings, present when `timestamps: true` was
+   * requested. Use this rather than `duration` to find where *speech* ends.
+   */
+  alignment?: TTSAlignment
+  /**
+   * Per-turn (or per-utterance) spans of the audio, present when
+   * `timestamps: true` was requested and the provider reports segmentation.
+   */
+  segments?: Array<TTSSegment>
   /** Content type of the audio (e.g., 'audio/mp3') */
   contentType?: string
   /** Token usage information (if provided by the adapter) */
@@ -2403,6 +2501,174 @@ export interface TTSResult {
   artifacts?: Array<PersistedArtifactRef>
 }
 
+// ============================================================================
+// Voice Catalog Types
+// ============================================================================
+
+/**
+ * Where a voice in a provider's catalog came from.
+ *
+ * `'premade'` is the provider's own stock catalog. `'generated'` and
+ * `'cloned'` are voices the account made, which is what `generateVoice()`
+ * produces. `'professional'` covers a provider's curated or paid tiers.
+ */
+export type VoiceOrigin = 'premade' | 'generated' | 'cloned' | 'professional'
+
+/** One voice from a provider's catalog. */
+export interface CatalogVoice {
+  /** Pass this to `generateSpeech()` as `voice` */
+  voiceId: string
+  /** Display name, when the provider stores one */
+  name?: string
+  /** Where the voice came from */
+  origin?: VoiceOrigin
+  /** Provider description of the voice */
+  description?: string
+  /** URL of a sample, when the provider hosts one */
+  previewUrl?: string
+  /** Provider labels, such as accent, age, or use case */
+  labels?: Record<string, string>
+}
+
+/** Options for listing a provider's voices. */
+export interface ListVoicesOptions {
+  /**
+   * Restrict the result to voices of these origins. Adapters filter server
+   * side when the provider supports it, and in memory otherwise.
+   */
+  origins?: Array<VoiceOrigin>
+  /**
+   * Effective abort signal. Adapters forward this to the provider SDK when
+   * supported.
+   */
+  abortSignal?: AbortSignal
+}
+
+/** Result of listing a provider's voices. */
+export interface ListVoicesResult {
+  /** The voices available to this account */
+  voices: Array<CatalogVoice>
+}
+
+// ============================================================================
+// Voice Creation Types
+// ============================================================================
+
+/**
+ * Options for creating a voice.
+ *
+ * Providers create voices in one of two ways, and some support both:
+ * - **design** — synthesize a brand new voice from a text {@link prompt}.
+ * - **clone** — derive a voice from {@link referenceAudio} of a real speaker.
+ *
+ * At least one of `prompt` / `referenceAudio` is required; which ones an
+ * adapter accepts depends on the model. An adapter may require both — the
+ * only adapter today, `elevenlabsVoiceDesign`, always needs `prompt` and
+ * takes `referenceAudio` as an additional design reference.
+ */
+export interface VoiceGenerationOptions<
+  TProviderOptions extends object = object,
+> {
+  /** The model to use for voice creation */
+  model: string
+  /** Text description of the voice to create, for design-capable models */
+  prompt?: string
+  /**
+   * Reference audio of the speaker to clone - base64 string, base64 data URL,
+   * File, Blob, or ArrayBuffer. For clone-capable models. Remote URLs are not
+   * accepted; read the file and pass the bytes.
+   */
+  referenceAudio?: string | File | Blob | ArrayBuffer
+  /**
+   * Name to store the voice under in the provider's voice library. Providers
+   * differ on what this implies — ElevenLabs only persists a designed voice
+   * when a name is given. Read {@link GeneratedVoice.saved} to find out what
+   * actually happened.
+   */
+  name?: string
+  /** Human-readable description stored alongside the voice */
+  description?: string
+  /** Model-specific options for voice creation */
+  modelOptions?: TProviderOptions
+  /**
+   * Internal logger threaded from the generateVoice() entry point. Adapters
+   * must call logger.request() before the SDK call and logger.errors() in
+   * catch blocks.
+   */
+  logger: InternalLogger
+  /**
+   * Effective abort signal composed by the activity from caller `abortSignal`
+   * and/or `timeout`. Adapters should forward this to the provider SDK when
+   * supported. Request-specific - never store on a global client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+/**
+ * A single voice produced by {@link VoiceGenerationOptions}.
+ */
+export interface GeneratedVoice {
+  /**
+   * The provider's voice identifier. Pass it straight back as the `voice`
+   * option on `generateSpeech()`.
+   */
+  voiceId: string
+  /** Base64-encoded preview audio, when the provider returns one */
+  audio?: string
+  /** Audio format of the preview (e.g. 'mp3') */
+  format?: string
+  /** Content type of the preview (e.g. 'audio/mpeg') */
+  contentType?: string
+  /** Duration of the preview in seconds, if available */
+  duration?: number
+  /** Language of the preview, if reported */
+  language?: string
+  /**
+   * Whether the voice is persisted in the provider's voice library. Unsaved
+   * voices are previews and generally expire.
+   */
+  saved: boolean
+  /**
+   * Whether the voice can be used in `generateSpeech()` yet. Required so a
+   * caller never has to guess: every adapter states it outright.
+   */
+  status: VoiceTrainingStatus
+}
+
+/**
+ * Whether a created voice is usable.
+ *
+ * - `'ready'` — usable in `generateSpeech()` now. Every adapter today returns
+ *   this, because they all finish the voice inside `generateVoice()`.
+ * - `'training'` — the provider accepted the request but is still building
+ *   the voice, so it is not usable yet. Reserved for providers that train
+ *   asynchronously; no adapter returns it yet, and reading the state back
+ *   will land with the first adapter that needs it.
+ * - `'failed'` — the provider finished without producing a usable voice.
+ */
+export type VoiceTrainingStatus = 'ready' | 'training' | 'failed'
+
+/**
+ * Result of voice creation.
+ *
+ * Design models typically return several candidates to choose between; clone
+ * models return exactly one.
+ */
+export interface VoiceResult {
+  /** Unique identifier for the generation */
+  id: string
+  /** Model used for generation */
+  model: string
+  /** The voices produced, best-first when the provider ranks them */
+  voices: Array<GeneratedVoice>
+  /** The line spoken in the previews, when the provider generated one */
+  previewText?: string
+  /** Token usage information (if provided by the adapter) */
+  usage?: TokenUsage
+  /** Persisted artifact references for generated assets, when available */
+  artifacts?: Array<PersistedArtifactRef>
+}
+
 // ============================================================================
 // Transcription (Speech-to-Text) Types
 // ============================================================================

package/src/utilities/durability-batch.ts

@@ -0,0 +1,48 @@
+import { CUSTOM_EVENT } from '../custom-events'
+import type { CustomEvent, StreamChunk } from '../types'
+import { tanstackMetadata, withTanstackMetadata } from './merge-metadata'
+
+/**
+ * High-volume CUSTOM names that stay in the durability batch.
+ * Everything else flushes as soon as it is emitted.
+ */
+const BATCHED_CUSTOM_EVENT_NAMES = new Set<string>([
+  CUSTOM_EVENT.PROCESS_STDOUT,
+  CUSTOM_EVENT.PROCESS_STDERR,
+  'sandbox.file',
+  'sandbox.file.diff',
+])
+
+function hasBatchHint(chunk: StreamChunk): boolean {
+  const tanstack = tanstackMetadata(chunk)
+  if (tanstack == null) return false
+  return Reflect.get(tanstack, 'batch') === true
+}
+
+/** Mark a CUSTOM chunk so the durability producer keeps it in the batch. */
+export function withDurabilityBatchHint(chunk: CustomEvent): CustomEvent {
+  return withTanstackMetadata(chunk, { batch: true })
+}
+
+export function isDurabilityBatchedCustom(chunk: StreamChunk): boolean {
+  if (chunk.type !== 'CUSTOM') return false
+  if (BATCHED_CUSTOM_EVENT_NAMES.has(chunk.name)) return true
+  return hasBatchHint(chunk)
+}
+
+/**
+ * Drop the in-process batch hint so it does not sit in the log or on the wire.
+ */
+export function stripDurabilityBatchHint(chunk: StreamChunk): StreamChunk {
+  const tanstack = tanstackMetadata(chunk)
+  if (tanstack == null || Reflect.get(tanstack, 'batch') !== true) return chunk
+  Reflect.deleteProperty(tanstack, 'batch')
+  const metadata = chunk.metadata
+  if (metadata != null && Object.keys(tanstack).length === 0) {
+    Reflect.deleteProperty(metadata, 'tanstack')
+    if (Object.keys(metadata).length === 0) {
+      Reflect.deleteProperty(chunk, 'metadata')
+    }
+  }
+  return chunk
+}