@tanstack/ai-elevenlabs 0.4.4 → 0.6.1

package/src/adapters/speech.ts

@@ -6,8 +6,18 @@ import {
   parseOutputFormat,
   readStreamToArrayBuffer,
 } from '../utils/client'
-import type { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
-import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type { ElevenLabs, ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
+import type {
+  CatalogVoice,
+  ListVoicesOptions,
+  ListVoicesResult,
+  TTSAlignment,
+  TTSCapabilities,
+  TTSOptions,
+  TTSResult,
+  TTSSegment,
+  VoiceOrigin,
+} from '@tanstack/ai'
 import type { ElevenLabsClientConfig } from '../utils/client'
 import type { ElevenLabsOutputFormat, ElevenLabsTTSModel } from '../model-meta'
 
@@ -82,6 +92,15 @@ export class ElevenLabsSpeechAdapter<
 > extends BaseTTSAdapter<TModel, ElevenLabsSpeechProviderOptions> {
   readonly name = 'elevenlabs' as const
 
+  /**
+   * `textToDialogue` accepts up to 10 distinct voice ids, and both endpoints
+   * have a `…WithTimestamps` twin that returns character alignment.
+   */
+  override readonly capabilities: TTSCapabilities = {
+    maxSpeakers: 10,
+    timestamps: true,
+  }
+
   private readonly client: ElevenLabsClient
 
   constructor(model: TModel, config?: ElevenLabsClientConfig) {
@@ -98,12 +117,6 @@ export class ElevenLabsSpeechAdapter<
       { provider: 'elevenlabs', model: this.model },
     )
     try {
-      const voiceId = options.voice ?? options.modelOptions?.voiceId
-      if (!voiceId) {
-        throw new Error(
-          'ElevenLabs TTS requires a voice. Pass `voice` on generateSpeech() or `voiceId` in modelOptions.',
-        )
-      }
       const {
         outputFormat,
         voiceSettings,
@@ -121,46 +134,115 @@ export class ElevenLabsSpeechAdapter<
       const effectiveOutputFormat =
         outputFormat ?? inferOutputFormatFromResponseFormat(options.format)
       const wrapAsWav = outputFormat == null && options.format === 'wav'
-
-      const stream = await this.client.textToSpeech.convert(voiceId, {
-        text: options.text,
+      const { format, contentType } = wrapAsWav
+        ? { format: 'wav', contentType: 'audio/wav' }
+        : parseOutputFormat(effectiveOutputFormat)
+      // `format: 'wav'` asks ElevenLabs for pcm_44100 and wraps it here, on
+      // every endpoint: the byte-stream pair hands back raw bytes, the
+      // timestamped pair hands back the same bytes as base64.
+      const encodeAudio = (buffer: ArrayBuffer) =>
+        arrayBufferToBase64(wrapAsWav ? wrapPcmAsWav(buffer) : buffer)
+      const encodeAudioBase64 = (audioBase64: string) =>
+        wrapAsWav ? encodeAudio(base64ToArrayBuffer(audioBase64)) : audioBase64
+      const base = {
         modelId: this.model,
         ...(effectiveOutputFormat
           ? { outputFormat: effectiveOutputFormat }
           : {}),
+        ...(languageCode ? { languageCode } : {}),
+        ...(seed != null ? { seed } : {}),
+        ...(applyTextNormalization ? { applyTextNormalization } : {}),
+      }
+
+      // Four endpoints, picked by (turns?, timestamps?). The dialogue pair
+      // takes `inputs` instead of a voice id in the path, and the timestamped
+      // pair returns JSON (base64 audio + alignment) instead of a byte stream.
+      if (options.turns) {
+        const inputs = options.turns.map((turn) => ({
+          text: turn.text,
+          voiceId: turn.voice,
+        }))
+        const dialogue = {
+          ...base,
+          inputs,
+          ...(voiceSettings?.stability != null
+            ? { settings: { stability: voiceSettings.stability } }
+            : {}),
+        }
+
+        if (options.timestamps) {
+          const response =
+            await this.client.textToDialogue.convertWithTimestamps(dialogue)
+          return {
+            id: generateId(this.name),
+            model: this.model,
+            audio: encodeAudioBase64(response.audioBase64),
+            format,
+            contentType,
+            ...toAlignmentFields(response.alignment, response.voiceSegments),
+          }
+        }
+
+        const stream = await this.client.textToDialogue.convert(dialogue)
+        return {
+          id: generateId(this.name),
+          model: this.model,
+          audio: encodeAudio(await readStreamToArrayBuffer(stream)),
+          format,
+          contentType,
+        }
+      }
+
+      const voiceId = options.voice ?? options.modelOptions?.voiceId
+      if (!voiceId) {
+        throw new Error(
+          'ElevenLabs TTS requires a voice. Pass `voice` on generateSpeech() or `voiceId` in modelOptions.',
+        )
+      }
+      const single = {
+        ...base,
+        text: options.text,
         ...(voiceSettings
           ? { voiceSettings: mapVoiceSettings(voiceSettings, options.speed) }
           : options.speed != null
             ? { voiceSettings: { speed: options.speed } }
             : {}),
-        ...(languageCode ? { languageCode } : {}),
-        ...(seed != null ? { seed } : {}),
         ...(previousText ? { previousText } : {}),
         ...(nextText ? { nextText } : {}),
         ...(previousRequestIds ? { previousRequestIds } : {}),
         ...(nextRequestIds ? { nextRequestIds } : {}),
-        ...(applyTextNormalization ? { applyTextNormalization } : {}),
         ...(applyLanguageTextNormalization != null
           ? { applyLanguageTextNormalization }
           : {}),
+        ...(enableLogging != null ? { enableLogging } : {}),
+      }
+
+      if (options.timestamps) {
+        const response = await this.client.textToSpeech.convertWithTimestamps(
+          voiceId,
+          single,
+        )
+        return {
+          id: generateId(this.name),
+          model: this.model,
+          audio: encodeAudioBase64(response.audioBase64),
+          format,
+          contentType,
+          ...toAlignmentFields(response.alignment, undefined),
+        }
+      }
+
+      const stream = await this.client.textToSpeech.convert(voiceId, {
+        ...single,
         ...(optimizeStreamingLatency != null
           ? { optimizeStreamingLatency }
           : {}),
-        ...(enableLogging != null ? { enableLogging } : {}),
       })
 
-      const buffer = await readStreamToArrayBuffer(stream)
-      const base64 = arrayBufferToBase64(
-        wrapAsWav ? wrapPcmAsWav(buffer) : buffer,
-      )
-      const { format, contentType } = wrapAsWav
-        ? { format: 'wav', contentType: 'audio/wav' }
-        : parseOutputFormat(effectiveOutputFormat)
-
       return {
         id: generateId(this.name),
         model: this.model,
-        audio: base64,
+        audio: encodeAudio(await readStreamToArrayBuffer(stream)),
         format,
         contentType,
       }
@@ -173,11 +255,73 @@ export class ElevenLabsSpeechAdapter<
     }
   }
 
+  /**
+   * List the voices this API key can use, via `GET /v1/voices`.
+   *
+   * The catalog is per-account and grows every time `generateVoice()` saves a
+   * voice, so it has to be read at runtime rather than shipped as a const.
+   */
+  override async listVoices(
+    options?: ListVoicesOptions,
+  ): Promise<ListVoicesResult> {
+    const response = await this.client.voices.getAll(
+      {},
+      options?.abortSignal ? { abortSignal: options.abortSignal } : {},
+    )
+
+    const voices = response.voices.map(toCatalogVoice)
+    const origins = options?.origins
+    // ElevenLabs has no origin filter on /v1/voices, so narrow in memory.
+    return {
+      voices: origins
+        ? voices.filter(
+            (voice) => voice.origin && origins.includes(voice.origin),
+          )
+        : voices,
+    }
+  }
+
   protected override generateId(): string {
     return generateId(this.name)
   }
 }
 
+/**
+ * Map the ElevenLabs timestamp payload onto the core `alignment` / `segments`
+ * fields. `voiceSegments` only comes back from the dialogue endpoint; its
+ * character indices slice the alignment into per-turn text.
+ */
+function toAlignmentFields(
+  alignment: ElevenLabs.CharacterAlignmentResponseModel | undefined,
+  voiceSegments: Array<ElevenLabs.VoiceSegment> | undefined,
+): { alignment?: TTSAlignment; segments?: Array<TTSSegment> } {
+  const fields: { alignment?: TTSAlignment; segments?: Array<TTSSegment> } = {}
+  if (alignment) {
+    fields.alignment = {
+      unit: 'character',
+      texts: alignment.characters,
+      startSeconds: alignment.characterStartTimesSeconds,
+      endSeconds: alignment.characterEndTimesSeconds,
+    }
+  }
+  if (voiceSegments && voiceSegments.length > 0) {
+    fields.segments = voiceSegments.map((segment) => ({
+      startSeconds: segment.startTimeSeconds,
+      endSeconds: segment.endTimeSeconds,
+      turnIndex: segment.dialogueInputIndex,
+      voice: segment.voiceId,
+      ...(alignment
+        ? {
+            text: alignment.characters
+              .slice(segment.characterStartIndex, segment.characterEndIndex)
+              .join(''),
+          }
+        : {}),
+    }))
+  }
+  return fields
+}
+
 function mapVoiceSettings(
   settings: ElevenLabsVoiceSettings,
   speedOverride: number | undefined,
@@ -199,6 +343,44 @@ function mapVoiceSettings(
   }
 }
 
+/**
+ * ElevenLabs voice categories map onto {@link VoiceOrigin} with two joins:
+ * `famous` and `high_quality` are both curated tiers, so both read as
+ * `'professional'`. An unrecognized category is dropped rather than guessed,
+ * which keeps an `origins` filter from silently matching the wrong thing.
+ */
+function toVoiceOrigin(
+  category: ElevenLabs.VoiceCategory | undefined,
+): VoiceOrigin | undefined {
+  switch (category) {
+    case 'premade':
+      return 'premade'
+    case 'generated':
+      return 'generated'
+    case 'cloned':
+      return 'cloned'
+    case 'professional':
+    case 'famous':
+    case 'high_quality':
+      return 'professional'
+    case undefined:
+    default:
+      return undefined
+  }
+}
+
+function toCatalogVoice(voice: ElevenLabs.Voice): CatalogVoice {
+  const origin = toVoiceOrigin(voice.category)
+  return {
+    voiceId: voice.voiceId,
+    ...(voice.name ? { name: voice.name } : {}),
+    ...(origin ? { origin } : {}),
+    ...(voice.description ? { description: voice.description } : {}),
+    ...(voice.previewUrl ? { previewUrl: voice.previewUrl } : {}),
+    ...(voice.labels ? { labels: voice.labels } : {}),
+  }
+}
+
 /**
  * Map the standard TTSOptions `format` (mp3/opus/aac/flac/wav/pcm) to a
  * reasonable ElevenLabs `outputFormat` so callers don't need to know the
@@ -226,6 +408,13 @@ function inferOutputFormatFromResponseFormat(
   }
 }
 
+function base64ToArrayBuffer(base64: string): ArrayBuffer {
+  const binary = atob(base64)
+  const bytes = new Uint8Array(binary.length)
+  for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i)
+  return bytes.buffer
+}
+
 /** Wrap ElevenLabs pcm_44100 (16-bit little-endian mono) in a RIFF/WAV container. */
 function wrapPcmAsWav(pcm: ArrayBuffer): ArrayBuffer {
   const wav = new ArrayBuffer(44 + pcm.byteLength)

package/src/adapters/voice.ts

@@ -0,0 +1,299 @@
+import { BaseVoiceAdapter } from '@tanstack/ai/adapters'
+import {
+  arrayBufferToBase64,
+  createElevenLabsClient,
+  generateId,
+  parseOutputFormat,
+} from '../utils/client'
+import type { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
+import type {
+  GeneratedVoice,
+  VoiceGenerationOptions,
+  VoiceResult,
+} from '@tanstack/ai'
+import type { ElevenLabsClientConfig } from '../utils/client'
+import type {
+  ElevenLabsOutputFormat,
+  ElevenLabsVoiceModel,
+} from '../model-meta'
+
+/**
+ * Provider-specific voice-design options. Fields map 1:1 onto the SDK's
+ * `VoiceDesignRequestModel` — mirroring the names so ElevenLabs'
+ * documentation stays useful here.
+ * @see https://elevenlabs.io/docs/api-reference/text-to-voice/design
+ */
+export interface ElevenLabsVoiceProviderOptions {
+  /** Output audio format for the previews, `codec_samplerate[_bitrate]`. */
+  outputFormat?: ElevenLabsOutputFormat
+  /** Line the previews speak, 100..1000 characters. */
+  text?: string
+  /** Let ElevenLabs write the preview line from the description. */
+  autoGenerateText?: boolean
+  /** Preview loudness, -1 (quietest) to 1 (loudest). 0 is roughly -24 LUFS. */
+  loudness?: number
+  /** Deterministic sampling seed — same seed and inputs produce the same voice. */
+  seed?: number
+  /** How closely to follow the description. High values can sound robotic. */
+  guidanceScale?: number
+  /** Higher quality trades variety for fidelity. */
+  quality?: number
+  /** Let ElevenLabs expand a short description into a detailed one. */
+  shouldEnhance?: boolean
+  /**
+   * Balance of description against reference audio, 0 (almost all reference)
+   * to 1 (almost all description). `eleven_ttv_v3` only.
+   */
+  promptStrength?: number
+  /** Metadata stored on the voice. Only used when the voice is saved. */
+  labels?: Record<string, string>
+  /** Remixing session to attach these generations to. */
+  remixingSessionId?: string
+  /** Remixing session iteration to attach these generations to. */
+  remixingSessionIterationId?: string
+}
+
+/**
+ * ElevenLabs voice-design adapter built on the official
+ * `@elevenlabs/elevenlabs-js` SDK.
+ *
+ * ElevenLabs designs voices in two steps — generate previews, then promote one
+ * into a real voice — but that is an implementation detail. Pass `name` to get
+ * a saved voice back; leave it off to audition previews first.
+ *
+ * @example Audition previews
+ * ```ts
+ * const result = await generateVoice({
+ *   adapter: elevenlabsVoiceDesign('eleven_ttv_v3'),
+ *   prompt: 'A warm, gravelly narrator in his sixties with a slight Irish lilt',
+ * })
+ * ```
+ *
+ * @example Save the voice
+ * ```ts
+ * const result = await generateVoice({
+ *   adapter: elevenlabsVoiceDesign('eleven_ttv_v3'),
+ *   prompt: 'A bright, upbeat product demo host',
+ *   name: 'Demo Host',
+ * })
+ * ```
+ */
+export class ElevenLabsVoiceAdapter<
+  TModel extends ElevenLabsVoiceModel,
+> extends BaseVoiceAdapter<TModel, ElevenLabsVoiceProviderOptions> {
+  readonly name = 'elevenlabs' as const
+
+  private readonly client: ElevenLabsClient
+
+  constructor(model: TModel, config?: ElevenLabsClientConfig) {
+    super(model, config ?? {})
+    this.client = createElevenLabsClient(config)
+  }
+
+  async generateVoice(
+    options: VoiceGenerationOptions<ElevenLabsVoiceProviderOptions>,
+  ): Promise<VoiceResult> {
+    const { logger } = options
+    logger.request(
+      `activity=generateVoice provider=elevenlabs model=${this.model}`,
+      { provider: 'elevenlabs', model: this.model },
+    )
+    try {
+      // `voiceDescription` is required by the design endpoint, so reference
+      // audio alone is not enough here — unlike clone-only providers.
+      if (!options.prompt) {
+        throw new Error(
+          'ElevenLabs voice design requires a `prompt` describing the voice. Reference audio alone is not supported; pass both to guide the design with a real speaker.',
+        )
+      }
+
+      const opts = options.modelOptions ?? {}
+      const referenceAudioBase64 = await toReferenceAudioBase64(
+        options.referenceAudio,
+      )
+      if (referenceAudioBase64 && this.model !== 'eleven_ttv_v3') {
+        throw new Error(
+          `ElevenLabs only accepts reference audio on eleven_ttv_v3, but this adapter is using "${this.model}".`,
+        )
+      }
+
+      const requestOptions = options.abortSignal
+        ? { abortSignal: options.abortSignal }
+        : {}
+
+      const previewResponse = await this.client.textToVoice.design(
+        {
+          voiceDescription: options.prompt,
+          modelId: this.model,
+          ...(opts.outputFormat ? { outputFormat: opts.outputFormat } : {}),
+          ...(opts.text ? { text: opts.text } : {}),
+          ...(opts.autoGenerateText != null
+            ? { autoGenerateText: opts.autoGenerateText }
+            : {}),
+          ...(opts.loudness != null ? { loudness: opts.loudness } : {}),
+          ...(opts.seed != null ? { seed: opts.seed } : {}),
+          ...(opts.guidanceScale != null
+            ? { guidanceScale: opts.guidanceScale }
+            : {}),
+          ...(opts.quality != null ? { quality: opts.quality } : {}),
+          ...(opts.shouldEnhance != null
+            ? { shouldEnhance: opts.shouldEnhance }
+            : {}),
+          ...(opts.promptStrength != null
+            ? { promptStrength: opts.promptStrength }
+            : {}),
+          ...(referenceAudioBase64 ? { referenceAudioBase64 } : {}),
+          ...(opts.remixingSessionId
+            ? { remixingSessionId: opts.remixingSessionId }
+            : {}),
+          ...(opts.remixingSessionIterationId
+            ? { remixingSessionIterationId: opts.remixingSessionIterationId }
+            : {}),
+        },
+        requestOptions,
+      )
+
+      const { format, contentType } = parseOutputFormat(opts.outputFormat)
+      const voices: Array<GeneratedVoice> = previewResponse.previews.map(
+        (preview) => ({
+          voiceId: preview.generatedVoiceId,
+          audio: preview.audioBase64,
+          format,
+          contentType: preview.mediaType || contentType,
+          duration: preview.durationSecs,
+          ...(preview.language ? { language: preview.language } : {}),
+          saved: false,
+          // The design endpoint returns a finished preview; nothing trains.
+          status: 'ready' as const,
+        }),
+      )
+
+      // A caller who passed `name` asked for a persisted voice. Returning an
+      // empty list would report success for a request that produced nothing.
+      if (voices.length === 0) {
+        throw new Error(
+          'ElevenLabs returned no voice previews for this description. Try a longer, more specific prompt.',
+        )
+      }
+
+      if (options.name) {
+        voices[0] = await this.promoteFirstPreview(
+          voices,
+          options.name,
+          options.description ?? options.prompt,
+          opts.labels,
+          requestOptions,
+        )
+      }
+
+      return {
+        id: generateId(this.name),
+        model: this.model,
+        voices,
+        previewText: previewResponse.text,
+      }
+    } catch (error) {
+      logger.errors('elevenlabs.generateVoice fatal', {
+        error,
+        source: 'elevenlabs.generateVoice',
+      })
+      throw error
+    }
+  }
+
+  /**
+   * Promote the best preview into a real library voice. The remaining
+   * generated ids go along as `playedNotSelectedVoiceIds` — ElevenLabs uses
+   * them as RLHF signal for future designs.
+   */
+  private async promoteFirstPreview(
+    voices: ReadonlyArray<GeneratedVoice>,
+    voiceName: string,
+    voiceDescription: string,
+    labels: Record<string, string> | undefined,
+    requestOptions: { abortSignal?: AbortSignal },
+  ): Promise<GeneratedVoice> {
+    const [best, ...rest] = voices
+    if (!best) {
+      throw new Error('No preview to promote into a library voice.')
+    }
+
+    const saved = await this.client.textToVoice.create(
+      {
+        voiceName,
+        voiceDescription,
+        generatedVoiceId: best.voiceId,
+        ...(labels ? { labels } : {}),
+        ...(rest.length > 0
+          ? { playedNotSelectedVoiceIds: rest.map((voice) => voice.voiceId) }
+          : {}),
+      },
+      requestOptions,
+    )
+
+    return { ...best, voiceId: saved.voiceId, saved: true }
+  }
+
+  protected override generateId(): string {
+    return generateId(this.name)
+  }
+}
+
+/**
+ * Normalize reference audio to the bare base64 the design endpoint wants.
+ *
+ * https URLs are rejected rather than fetched: ElevenLabs has no URL field
+ * here, and downloading caller-supplied media into memory to inline it is a
+ * footgun on large files.
+ */
+async function toReferenceAudioBase64(
+  audio: VoiceGenerationOptions['referenceAudio'],
+): Promise<string | undefined> {
+  if (audio == null) return undefined
+  if (audio instanceof ArrayBuffer) return arrayBufferToBase64(audio)
+  if (typeof audio !== 'string') {
+    return arrayBufferToBase64(await audio.arrayBuffer())
+  }
+
+  if (audio.startsWith('data:')) {
+    const commaIndex = audio.indexOf(',')
+    const header = commaIndex === -1 ? '' : audio.slice(5, commaIndex)
+    if (commaIndex === -1 || !/;base64$/i.test(header)) {
+      throw new Error(
+        'ElevenLabs voice design needs base64 reference audio. Pass a base64 data URL, a base64 string, a Blob, or an ArrayBuffer.',
+      )
+    }
+    return audio.slice(commaIndex + 1)
+  }
+
+  if (/^https?:\/\//i.test(audio)) {
+    throw new Error(
+      'ElevenLabs voice design does not accept reference audio URLs. Read the file yourself and pass a Blob, ArrayBuffer, or base64 string.',
+    )
+  }
+
+  return audio
+}
+
+/**
+ * Create an ElevenLabs voice-design adapter using `ELEVENLABS_API_KEY` from env.
+ */
+export function elevenlabsVoiceDesign<TModel extends ElevenLabsVoiceModel>(
+  model: TModel,
+  config?: ElevenLabsClientConfig,
+): ElevenLabsVoiceAdapter<TModel> {
+  return new ElevenLabsVoiceAdapter(model, config)
+}
+
+/**
+ * Create an ElevenLabs voice-design adapter with an explicit API key.
+ */
+export function createElevenLabsVoiceDesign<
+  TModel extends ElevenLabsVoiceModel,
+>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<ElevenLabsClientConfig, 'apiKey'>,
+): ElevenLabsVoiceAdapter<TModel> {
+  return new ElevenLabsVoiceAdapter(model, { apiKey, ...config })
+}

package/src/index.ts

@@ -38,6 +38,17 @@ export {
   type ElevenLabsMusicCompositionPlan,
 } from './adapters/audio'
 
+// ============================================================================
+// Voice (Voice Design) Adapter
+// ============================================================================
+
+export {
+  ElevenLabsVoiceAdapter,
+  createElevenLabsVoiceDesign,
+  elevenlabsVoiceDesign,
+  type ElevenLabsVoiceProviderOptions,
+} from './adapters/voice'
+
 // ============================================================================
 // Transcription (Speech-to-Text) Adapter
 // ============================================================================
@@ -57,6 +68,7 @@ export {
   ELEVENLABS_TTS_MODELS,
   ELEVENLABS_AUDIO_MODELS,
   ELEVENLABS_TRANSCRIPTION_MODELS,
+  ELEVENLABS_VOICE_MODELS,
   isElevenLabsMusicModel,
   isElevenLabsSoundEffectsModel,
   type ElevenLabsTTSModel,
@@ -64,6 +76,7 @@ export {
   type ElevenLabsMusicModel,
   type ElevenLabsSoundEffectsModel,
   type ElevenLabsTranscriptionModel,
+  type ElevenLabsVoiceModel,
   type ElevenLabsOutputFormat,
 } from './model-meta'