@tanstack/ai 0.55.0 → 0.57.0

package/src/activities/index.ts

@@ -20,9 +20,11 @@ import type { AnyImageAdapter } from './generateImage/adapter'
 import type { AnyAudioAdapter } from './generateAudio/adapter'
 import type { AnyVideoAdapter } from './generateVideo/adapter'
 import type { AnyTTSAdapter } from './generateSpeech/adapter'
+import type { AnyVoiceAdapter } from './generateVoice/adapter'
 import type { AnyTranscriptionAdapter } from './generateTranscription/adapter'
 import type { AnyEmbeddingAdapter } from './embed/adapter'
 import type { AnyRerankAdapter } from './rerank/adapter'
+import type { AnyEvaluateAdapter } from './evaluate/adapter'
 import type { AnyWorldAdapter } from './generateWorld/adapter'
 import type { AnyLiveVideoAdapter } from './generateLiveVideo/adapter'
 
@@ -89,6 +91,46 @@ export {
   type AnyRerankAdapter,
 } from './rerank/adapter'
 
+// ===========================
+// Evaluate Activity
+// ===========================
+
+export {
+  kind as evaluateKind,
+  decide,
+  choice,
+  score,
+  boolean,
+  type EvaluateActivityOptions,
+  type EvaluateResult,
+  type EvaluateResultMeta,
+  type EvaluateProviderOptions,
+  type ChoiceAnswer,
+  type ScoreAnswer,
+  type BooleanAnswer,
+  type InferEvaluateAnswer,
+} from './evaluate/index'
+
+export {
+  BaseEvaluateAdapter,
+  type EvaluateAdapter,
+  type EvaluateAdapterConfig,
+  type AnyEvaluateAdapter,
+  type EvaluateOptions,
+  type EvaluateAdapterResult,
+  type EvaluateState,
+  type EvaluateInstructions,
+  type EvaluateJsonValue,
+  type WireQuestion,
+  type WireAnswer,
+  type WireChoiceQuestion,
+  type WireScoreQuestion,
+  type WireNoulQuestion,
+  type WireChoiceAnswer,
+  type WireScoreAnswer,
+  type WireNoulAnswer,
+} from './evaluate/adapter'
+
 // ===========================
 // Image Activity
 // ===========================
@@ -162,6 +204,8 @@ export { snapToDurationOption } from './generateVideo/snap'
 export {
   kind as ttsKind,
   generateSpeech,
+  listVoices,
+  type ListVoicesActivityOptions,
   type TTSActivityOptions,
   type TTSActivityResult,
   type TTSProviderOptions,
@@ -171,9 +215,30 @@ export {
   BaseTTSAdapter,
   type TTSAdapter,
   type TTSAdapterConfig,
+  type TTSCapabilities,
   type AnyTTSAdapter,
 } from './generateSpeech/adapter'
 
+// ===========================
+// Voice Activity
+// ===========================
+
+export {
+  kind as voiceKind,
+  generateVoice,
+  createVoiceOptions,
+  type VoiceActivityOptions,
+  type VoiceActivityResult,
+  type VoiceProviderOptions,
+} from './generateVoice/index'
+
+export {
+  BaseVoiceAdapter,
+  type VoiceAdapter,
+  type VoiceAdapterConfig,
+  type AnyVoiceAdapter,
+} from './generateVoice/adapter'
+
 // ===========================
 // Transcription Activity
 // ===========================
@@ -262,9 +327,11 @@ export type AIAdapter =
   | AnyAudioAdapter
   | AnyVideoAdapter
   | AnyTTSAdapter
+  | AnyVoiceAdapter
   | AnyTranscriptionAdapter
   | AnyEmbeddingAdapter
   | AnyRerankAdapter
+  | AnyEvaluateAdapter
   | AnyWorldAdapter
   | AnyLiveVideoAdapter
 
@@ -276,8 +343,10 @@ export type AdapterKind =
   | 'audio'
   | 'video'
   | 'tts'
+  | 'voice'
   | 'transcription'
   | 'embedding'
   | 'rerank'
+  | 'evaluate'
   | 'world'
   | 'liveVideo'

package/src/activities/middleware/types.ts

@@ -40,9 +40,11 @@ export type GenerationActivity =
   | 'video'
   | 'audio'
   | 'tts'
+  | 'voice'
   | 'transcription'
   | 'embedding'
   | 'rerank'
+  | 'evaluate'
   | 'summarize'
   | 'world'
   | 'liveVideo'

package/src/client.ts

@@ -3,6 +3,7 @@ import type {
   ImageGenerationOptions,
   TTSOptions,
   TranscriptionOptions,
+  VoiceGenerationOptions,
   VideoGenerationOptions,
   WorldGenerationOptions,
   LiveVideoGenerationOptions,
@@ -12,6 +13,7 @@ export type GenerationKind =
   | 'image'
   | 'audio'
   | 'tts'
+  | 'voice'
   | 'video'
   | 'transcription'
   | 'world'
@@ -21,6 +23,7 @@ type GenerationInputByKind = {
   image: Omit<ImageGenerationOptions, 'logger' | 'model'>
   audio: Omit<AudioGenerationOptions, 'logger' | 'model'>
   tts: Omit<TTSOptions, 'logger' | 'model'>
+  voice: Omit<VoiceGenerationOptions, 'logger' | 'model'>
   video: Omit<VideoGenerationOptions, 'logger' | 'model'>
   transcription: Omit<TranscriptionOptions, 'logger' | 'model'>
   world: Omit<WorldGenerationOptions, 'logger' | 'model'>
@@ -38,6 +41,7 @@ const generationKinds = [
   'image',
   'audio',
   'tts',
+  'voice',
   'video',
   'transcription',
   'world',
@@ -69,6 +73,31 @@ function assertGenerationKind(kind: unknown): asserts kind is GenerationKind {
   }
 }
 
+/**
+ * The input field(s) that identify a generation body for a kind. Most kinds
+ * have exactly one; `voice` accepts either of its two creation modes, so any
+ * one of its keys is enough.
+ */
+function requiredKeysForKind(kind: GenerationKind): Array<string> {
+  // Enumerated rather than defaulted so a new generation kind has to declare
+  // the field that identifies its body instead of silently inheriting
+  // `prompt`.
+  switch (kind) {
+    case 'tts':
+      return ['text']
+    case 'transcription':
+      return ['audio']
+    case 'voice':
+      return ['prompt', 'referenceAudio']
+    case 'image':
+    case 'audio':
+    case 'video':
+    case 'world':
+    case 'liveVideo':
+      return ['prompt']
+  }
+}
+
 function assertInputForKind(
   kind: GenerationKind,
   input: unknown,
@@ -77,21 +106,19 @@ function assertInputForKind(
     throw new Error(`Generation ${kind} input must be an object.`)
   }
 
-  const requiredKey =
-    kind === 'tts' ? 'text' : kind === 'transcription' ? 'audio' : 'prompt'
+  const requiredKeys = requiredKeysForKind(kind)
 
-  if (!hasOwnKey(input, requiredKey)) {
-    throw new Error(`Generation ${kind} input must include ${requiredKey}.`)
+  if (!requiredKeys.some((key) => hasOwnKey(input, key))) {
+    throw new Error(
+      `Generation ${kind} input must include ${requiredKeys.join(' or ')}.`,
+    )
   }
 }
 
 function isInputForKind(kind: GenerationKind, input: unknown): boolean {
   if (!isRecord(input)) return false
 
-  const requiredKey =
-    kind === 'tts' ? 'text' : kind === 'transcription' ? 'audio' : 'prompt'
-
-  return hasOwnKey(input, requiredKey)
+  return requiredKeysForKind(kind).some((key) => hasOwnKey(input, key))
 }
 
 function forwardedPropsFromEnvelope(

package/src/index.ts

@@ -3,11 +3,17 @@ export {
   chat,
   summarize,
   rerank,
+  decide,
+  choice,
+  score,
+  boolean,
   generateImage,
   generateAudio,
   generateVideo,
   getVideoJobStatus,
   generateSpeech,
+  listVoices,
+  generateVoice,
   generateTranscription,
   embed,
   generateWorld,
@@ -22,6 +28,7 @@ export { createImageOptions } from './activities/generateImage/index'
 export { createAudioOptions } from './activities/generateAudio/index'
 export { createVideoOptions } from './activities/generateVideo/index'
 export { createSpeechOptions } from './activities/generateSpeech/index'
+export { createVoiceOptions } from './activities/generateVoice/index'
 export { createTranscriptionOptions } from './activities/generateTranscription/index'
 export { createEmbedOptions } from './activities/embed/index'
 export { createWorldOptions } from './activities/generateWorld/index'
@@ -40,6 +47,9 @@ export type {
   AudioAdapter,
   AnyTTSAdapter,
   TTSAdapter,
+  TTSCapabilities,
+  AnyVoiceAdapter,
+  VoiceAdapter,
   AnyTranscriptionAdapter,
   TranscriptionAdapter,
   AnyVideoAdapter,
@@ -48,6 +58,14 @@ export type {
   EmbeddingAdapter,
   AnyRerankAdapter,
   RerankAdapter,
+  AnyEvaluateAdapter,
+  EvaluateAdapter,
+  ChoiceAnswer,
+  ScoreAnswer,
+  BooleanAnswer,
+  EvaluateResult,
+  WireQuestion,
+  WireAnswer,
   AnyWorldAdapter,
   WorldAdapter,
   AnyLiveVideoAdapter,
@@ -57,6 +75,9 @@ export type {
 // Rerank adapter base + types
 export { BaseRerankAdapter } from './activities/rerank/adapter'
 
+// Evaluate adapter base + types
+export { BaseEvaluateAdapter } from './activities/evaluate/adapter'
+
 // Tool definition
 export {
   toolDefinition,

package/src/middlewares/otel.ts

@@ -87,9 +87,11 @@ const OPERATION_NAME: Record<GenerationActivity, string> = {
   video: 'video_generation',
   audio: 'audio_generation',
   tts: 'text_to_speech',
+  voice: 'voice_generation',
   transcription: 'transcription',
   embedding: 'embeddings',
   rerank: 'rerank',
+  evaluate: 'evaluate',
   summarize: 'summarize',
   world: 'world_generation',
   liveVideo: 'live_video_generation',

package/src/realtime/index.ts

@@ -22,7 +22,7 @@ export type * from './types'
  * // On the server (e.g. inside a server route or framework server
  * // function), mint an ephemeral token for the client:
  * const token = await realtimeToken({
- *   adapter: openaiRealtimeToken({ model: 'gpt-realtime' }),
+ *   adapter: openaiRealtimeToken({ model: 'gpt-realtime-2.1' }),
  * })
  * ```
  */

package/src/types.ts

@@ -2350,6 +2350,56 @@ export interface LiveVideoGenerationResult {
 // Text-to-Speech (TTS) Types
 // ============================================================================
 
+/**
+ * One turn of a multi-voice dialogue request.
+ *
+ * Providers that expose a dedicated dialogue endpoint (ElevenLabs
+ * `textToDialogue`, Gemini multi-speaker) take these natively instead of a
+ * single `text` + `voice` pair.
+ */
+export interface TTSTurn {
+  /** The text this voice speaks. */
+  text: string
+  /** Provider voice id (ElevenLabs) or voice name (Gemini) for this turn. */
+  voice: string
+}
+
+/**
+ * Timings for the generated audio, returned when `timestamps: true` was
+ * requested and the adapter declares `capabilities.timestamps`.
+ *
+ * Granularity differs per provider — ElevenLabs reports characters, BytePlus
+ * reports words — so `unit` says which, and the three arrays are parallel.
+ * All times are **seconds**; adapters convert.
+ */
+export interface TTSAlignment {
+  /** Granularity of each entry. */
+  unit: 'character' | 'word'
+  /** Entry text, in audio order. */
+  texts: Array<string>
+  /** Start of each entry in seconds. Same length as `texts`. */
+  startSeconds: Array<number>
+  /** End of each entry in seconds. Same length as `texts`. */
+  endSeconds: Array<number>
+}
+
+/**
+ * A stretch of audio attributable to one turn (multi-voice) or one utterance
+ * (single voice). This is what tells a consumer which turn is where.
+ */
+export interface TTSSegment {
+  /** Start of the segment in seconds. */
+  startSeconds: number
+  /** End of the segment in seconds. */
+  endSeconds: number
+  /** Index into the request's `turns`, when the provider reports it. */
+  turnIndex?: number
+  /** Voice heard in this segment, when the provider reports it. */
+  voice?: string
+  /** Text spoken in this segment, when the provider reports it. */
+  text?: string
+}
+
 /**
  * Options for text-to-speech generation.
  * These are the common options supported across providers.
@@ -2357,8 +2407,24 @@ export interface LiveVideoGenerationResult {
 export interface TTSOptions<TProviderOptions extends object = object> {
   /** The model to use for TTS generation */
   model: string
-  /** The text to convert to speech */
+  /**
+   * The text to convert to speech. When the caller passed `turns`, the
+   * activity fills this with the turn texts joined by newlines so adapters
+   * that only read `text` still receive the full script.
+   */
   text: string
+  /**
+   * Multi-voice dialogue turns, when the caller asked for dialogue. Only
+   * adapters that declare `capabilities.maxSpeakers` ever see this — the
+   * activity rejects `turns` for the rest.
+   */
+  turns?: Array<TTSTurn>
+  /**
+   * Ask for `alignment` / `segments` on the result. Rejected by the activity
+   * unless the adapter declares `capabilities.timestamps`, because on some
+   * providers this is a different endpoint rather than free metadata.
+   */
+  timestamps?: boolean
   /** The voice to use for generation */
   voice?: string
   /** The output audio format */
@@ -2393,8 +2459,18 @@ export interface TTSResult {
   audio: string
   /** Audio format of the generated audio */
   format: string
-  /** Duration of the audio in seconds, if available */
+  /** Duration of the audio file in seconds, if available */
   duration?: number
+  /**
+   * Character- or word-level timings, present when `timestamps: true` was
+   * requested. Use this rather than `duration` to find where *speech* ends.
+   */
+  alignment?: TTSAlignment
+  /**
+   * Per-turn (or per-utterance) spans of the audio, present when
+   * `timestamps: true` was requested and the provider reports segmentation.
+   */
+  segments?: Array<TTSSegment>
   /** Content type of the audio (e.g., 'audio/mp3') */
   contentType?: string
   /** Token usage information (if provided by the adapter) */
@@ -2403,6 +2479,174 @@ export interface TTSResult {
   artifacts?: Array<PersistedArtifactRef>
 }
 
+// ============================================================================
+// Voice Catalog Types
+// ============================================================================
+
+/**
+ * Where a voice in a provider's catalog came from.
+ *
+ * `'premade'` is the provider's own stock catalog. `'generated'` and
+ * `'cloned'` are voices the account made, which is what `generateVoice()`
+ * produces. `'professional'` covers a provider's curated or paid tiers.
+ */
+export type VoiceOrigin = 'premade' | 'generated' | 'cloned' | 'professional'
+
+/** One voice from a provider's catalog. */
+export interface CatalogVoice {
+  /** Pass this to `generateSpeech()` as `voice` */
+  voiceId: string
+  /** Display name, when the provider stores one */
+  name?: string
+  /** Where the voice came from */
+  origin?: VoiceOrigin
+  /** Provider description of the voice */
+  description?: string
+  /** URL of a sample, when the provider hosts one */
+  previewUrl?: string
+  /** Provider labels, such as accent, age, or use case */
+  labels?: Record<string, string>
+}
+
+/** Options for listing a provider's voices. */
+export interface ListVoicesOptions {
+  /**
+   * Restrict the result to voices of these origins. Adapters filter server
+   * side when the provider supports it, and in memory otherwise.
+   */
+  origins?: Array<VoiceOrigin>
+  /**
+   * Effective abort signal. Adapters forward this to the provider SDK when
+   * supported.
+   */
+  abortSignal?: AbortSignal
+}
+
+/** Result of listing a provider's voices. */
+export interface ListVoicesResult {
+  /** The voices available to this account */
+  voices: Array<CatalogVoice>
+}
+
+// ============================================================================
+// Voice Creation Types
+// ============================================================================
+
+/**
+ * Options for creating a voice.
+ *
+ * Providers create voices in one of two ways, and some support both:
+ * - **design** — synthesize a brand new voice from a text {@link prompt}.
+ * - **clone** — derive a voice from {@link referenceAudio} of a real speaker.
+ *
+ * At least one of `prompt` / `referenceAudio` is required; which ones an
+ * adapter accepts depends on the model. An adapter may require both — the
+ * only adapter today, `elevenlabsVoiceDesign`, always needs `prompt` and
+ * takes `referenceAudio` as an additional design reference.
+ */
+export interface VoiceGenerationOptions<
+  TProviderOptions extends object = object,
+> {
+  /** The model to use for voice creation */
+  model: string
+  /** Text description of the voice to create, for design-capable models */
+  prompt?: string
+  /**
+   * Reference audio of the speaker to clone - base64 string, base64 data URL,
+   * File, Blob, or ArrayBuffer. For clone-capable models. Remote URLs are not
+   * accepted; read the file and pass the bytes.
+   */
+  referenceAudio?: string | File | Blob | ArrayBuffer
+  /**
+   * Name to store the voice under in the provider's voice library. Providers
+   * differ on what this implies — ElevenLabs only persists a designed voice
+   * when a name is given. Read {@link GeneratedVoice.saved} to find out what
+   * actually happened.
+   */
+  name?: string
+  /** Human-readable description stored alongside the voice */
+  description?: string
+  /** Model-specific options for voice creation */
+  modelOptions?: TProviderOptions
+  /**
+   * Internal logger threaded from the generateVoice() entry point. Adapters
+   * must call logger.request() before the SDK call and logger.errors() in
+   * catch blocks.
+   */
+  logger: InternalLogger
+  /**
+   * Effective abort signal composed by the activity from caller `abortSignal`
+   * and/or `timeout`. Adapters should forward this to the provider SDK when
+   * supported. Request-specific - never store on a global client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+/**
+ * A single voice produced by {@link VoiceGenerationOptions}.
+ */
+export interface GeneratedVoice {
+  /**
+   * The provider's voice identifier. Pass it straight back as the `voice`
+   * option on `generateSpeech()`.
+   */
+  voiceId: string
+  /** Base64-encoded preview audio, when the provider returns one */
+  audio?: string
+  /** Audio format of the preview (e.g. 'mp3') */
+  format?: string
+  /** Content type of the preview (e.g. 'audio/mpeg') */
+  contentType?: string
+  /** Duration of the preview in seconds, if available */
+  duration?: number
+  /** Language of the preview, if reported */
+  language?: string
+  /**
+   * Whether the voice is persisted in the provider's voice library. Unsaved
+   * voices are previews and generally expire.
+   */
+  saved: boolean
+  /**
+   * Whether the voice can be used in `generateSpeech()` yet. Required so a
+   * caller never has to guess: every adapter states it outright.
+   */
+  status: VoiceTrainingStatus
+}
+
+/**
+ * Whether a created voice is usable.
+ *
+ * - `'ready'` — usable in `generateSpeech()` now. Every adapter today returns
+ *   this, because they all finish the voice inside `generateVoice()`.
+ * - `'training'` — the provider accepted the request but is still building
+ *   the voice, so it is not usable yet. Reserved for providers that train
+ *   asynchronously; no adapter returns it yet, and reading the state back
+ *   will land with the first adapter that needs it.
+ * - `'failed'` — the provider finished without producing a usable voice.
+ */
+export type VoiceTrainingStatus = 'ready' | 'training' | 'failed'
+
+/**
+ * Result of voice creation.
+ *
+ * Design models typically return several candidates to choose between; clone
+ * models return exactly one.
+ */
+export interface VoiceResult {
+  /** Unique identifier for the generation */
+  id: string
+  /** Model used for generation */
+  model: string
+  /** The voices produced, best-first when the provider ranks them */
+  voices: Array<GeneratedVoice>
+  /** The line spoken in the previews, when the provider generated one */
+  previewText?: string
+  /** Token usage information (if provided by the adapter) */
+  usage?: TokenUsage
+  /** Persisted artifact references for generated assets, when available */
+  artifacts?: Array<PersistedArtifactRef>
+}
+
 // ============================================================================
 // Transcription (Speech-to-Text) Types
 // ============================================================================