@tanstack/ai-elevenlabs 0.1.8 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-elevenlabs",
-  "version": "0.1.8",
+  "version": "0.2.1",
   "description": "ElevenLabs adapter for TanStack AI realtime voice",
   "author": "",
   "license": "MIT",
@@ -15,7 +15,14 @@
     "voice",
     "realtime",
     "tanstack",
-    "adapter"
+    "adapter",
+    "tts",
+    "text-to-speech",
+    "audio-generation",
+    "music",
+    "sound-effects",
+    "transcription",
+    "speech-to-text"
   ],
   "type": "module",
   "module": "./dist/esm/index.js",
@@ -31,16 +38,17 @@
     "src"
   ],
   "dependencies": {
-    "@11labs/client": "^0.2.0"
+    "@elevenlabs/client": "^1.3.1",
+    "@elevenlabs/elevenlabs-js": "^2.44.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "^0.14.0",
-    "@tanstack/ai-client": "^0.8.0"
+    "@tanstack/ai": "^0.15.0",
+    "@tanstack/ai-client": "^0.9.0"
   },
   "devDependencies": {
     "@vitest/coverage-v8": "4.0.14",
-    "@tanstack/ai": "0.14.0",
-    "@tanstack/ai-client": "0.8.0"
+    "@tanstack/ai": "0.15.0",
+    "@tanstack/ai-client": "0.9.0"
   },
   "scripts": {
     "build": "vite build",

package/src/adapters/audio.ts

@@ -0,0 +1,257 @@
+import { BaseAudioAdapter } from '@tanstack/ai/adapters'
+import {
+  arrayBufferToBase64,
+  createElevenLabsClient,
+  generateId,
+  parseOutputFormat,
+  readStreamToArrayBuffer,
+} from '../utils/client'
+import {
+  isElevenLabsMusicModel,
+  isElevenLabsSoundEffectsModel,
+} from '../model-meta'
+import type { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
+import type {
+  AudioGenerationOptions,
+  AudioGenerationResult,
+} from '@tanstack/ai'
+import type { ElevenLabsClientConfig } from '../utils/client'
+import type {
+  ElevenLabsAudioModel,
+  ElevenLabsMusicModel,
+  ElevenLabsOutputFormat,
+  ElevenLabsSoundEffectsModel,
+} from '../model-meta'
+
+/**
+ * Structured composition plan for ElevenLabs music generation. Mutually
+ * exclusive with a free-form `prompt` on the `generateAudio()` call — when
+ * supplied, `prompt` is ignored by ElevenLabs.
+ *
+ * We mirror the SDK's camelCase naming. Lengths are in milliseconds.
+ * @see https://elevenlabs.io/docs/api-reference/music/compose
+ */
+export interface ElevenLabsMusicCompositionPlan {
+  /** Positive global style descriptors (mood, instruments, tempo, …). */
+  positiveGlobalStyles?: Array<string>
+  /** Negative global style descriptors — styles to avoid. */
+  negativeGlobalStyles?: Array<string>
+  /** Section definitions (verse/chorus/bridge/…) with local style hints. */
+  sections?: Array<{
+    sectionName: string
+    positiveLocalStyles?: Array<string>
+    negativeLocalStyles?: Array<string>
+    durationMs?: number
+    lines?: Array<string>
+  }>
+}
+
+/**
+ * Provider options common to all ElevenLabs audio endpoints.
+ */
+interface CommonAudioOptions {
+  /** Output audio format. Defaults to `mp3_44100_128`. */
+  outputFormat?: ElevenLabsOutputFormat
+}
+
+/**
+ * Provider options for music generation (`music_v1`).
+ */
+export interface ElevenLabsMusicProviderOptions extends CommonAudioOptions {
+  /** Structured composition plan. Mutually exclusive with `prompt`/`duration`. */
+  compositionPlan?: ElevenLabsMusicCompositionPlan
+  /** Deterministic sampling seed (incompatible with `prompt`). */
+  seed?: number
+  /** Force the output to be purely instrumental (prompt-mode only). */
+  forceInstrumental?: boolean
+  /** Strictly respect section durations in `compositionPlan`. */
+  respectSectionsDurations?: boolean
+}
+
+/**
+ * Provider options for sound-effect generation (`eleven_text_to_sound_v*`).
+ */
+export interface ElevenLabsSoundEffectsProviderOptions extends CommonAudioOptions {
+  /** Prompt influence, 0..1. Default 0.3. Higher = more prompt adherence. */
+  promptInfluence?: number
+  /** Generate a loopable SFX (v2 only). */
+  loop?: boolean
+}
+
+/**
+ * Union of per-model provider options. We keep both branches on one type so
+ * the adapter stays tree-shakeable; callers narrow by model at the factory.
+ */
+export type ElevenLabsAudioProviderOptions =
+  | (ElevenLabsMusicProviderOptions & ElevenLabsSoundEffectsProviderOptions)
+  | ElevenLabsMusicProviderOptions
+  | ElevenLabsSoundEffectsProviderOptions
+
+/**
+ * ElevenLabs audio generation adapter. Dispatches to music or SFX endpoints
+ * based on the model id. Music → `client.music.compose`, SFX →
+ * `client.textToSoundEffects.convert`.
+ *
+ * @example
+ * ```ts
+ * const music = elevenlabsAudio('music_v1')
+ * await generateAudio({ adapter: music, prompt: 'lo-fi beat', duration: 15 })
+ *
+ * const sfx = elevenlabsAudio('eleven_text_to_sound_v2')
+ * await generateAudio({ adapter: sfx, prompt: 'glass shattering', duration: 3 })
+ * ```
+ */
+export class ElevenLabsAudioAdapter<
+  TModel extends ElevenLabsAudioModel,
+> extends BaseAudioAdapter<TModel, ElevenLabsAudioProviderOptions> {
+  readonly name = 'elevenlabs' as const
+
+  private client: ElevenLabsClient
+
+  constructor(model: TModel, config?: ElevenLabsClientConfig) {
+    super(model, config ?? {})
+    this.client = createElevenLabsClient(config)
+  }
+
+  async generateAudio(
+    options: AudioGenerationOptions<ElevenLabsAudioProviderOptions>,
+  ): Promise<AudioGenerationResult> {
+    const { logger } = options
+    logger.request(
+      `activity=generateAudio provider=elevenlabs model=${this.model}`,
+      { provider: 'elevenlabs', model: this.model },
+    )
+    try {
+      if (isElevenLabsMusicModel(this.model)) {
+        return await this.runMusic(options)
+      }
+      if (isElevenLabsSoundEffectsModel(this.model)) {
+        return await this.runSoundEffects(options)
+      }
+      throw new Error(
+        `Unsupported ElevenLabs audio model "${this.model}". Expected one of: music_v1, eleven_text_to_sound_v2, eleven_text_to_sound_v1.`,
+      )
+    } catch (error) {
+      logger.errors('elevenlabs.generateAudio fatal', {
+        error,
+        source: 'elevenlabs.generateAudio',
+      })
+      throw error
+    }
+  }
+
+  private async runMusic(
+    options: AudioGenerationOptions<ElevenLabsAudioProviderOptions>,
+  ): Promise<AudioGenerationResult> {
+    // Gated by isElevenLabsMusicModel() in generateAudio().
+    const modelId = this.model as ElevenLabsMusicModel
+    const music = (options.modelOptions ?? {}) as ElevenLabsMusicProviderOptions
+    const outputFormat = music.outputFormat
+
+    const stream = await this.client.music.compose({
+      modelId,
+      ...(options.prompt && !music.compositionPlan
+        ? { prompt: options.prompt }
+        : {}),
+      ...(music.compositionPlan
+        ? { compositionPlan: toMusicPrompt(music.compositionPlan) }
+        : {}),
+      ...(options.duration != null && !music.compositionPlan
+        ? { musicLengthMs: Math.round(options.duration * 1000) }
+        : {}),
+      ...(outputFormat ? { outputFormat } : {}),
+      ...(music.seed != null ? { seed: music.seed } : {}),
+      ...(music.forceInstrumental != null
+        ? { forceInstrumental: music.forceInstrumental }
+        : {}),
+      ...(music.respectSectionsDurations != null
+        ? { respectSectionsDurations: music.respectSectionsDurations }
+        : {}),
+    })
+
+    return this.finalize(stream, outputFormat, options.duration)
+  }
+
+  private async runSoundEffects(
+    options: AudioGenerationOptions<ElevenLabsAudioProviderOptions>,
+  ): Promise<AudioGenerationResult> {
+    // Gated by isElevenLabsSoundEffectsModel() in generateAudio().
+    const modelId = this.model as ElevenLabsSoundEffectsModel
+    const sfx = (options.modelOptions ??
+      {}) as ElevenLabsSoundEffectsProviderOptions
+    const outputFormat = sfx.outputFormat
+
+    const stream = await this.client.textToSoundEffects.convert({
+      text: options.prompt,
+      modelId,
+      ...(options.duration != null
+        ? { durationSeconds: options.duration }
+        : {}),
+      ...(outputFormat ? { outputFormat } : {}),
+      ...(sfx.promptInfluence != null
+        ? { promptInfluence: sfx.promptInfluence }
+        : {}),
+      ...(sfx.loop != null ? { loop: sfx.loop } : {}),
+    })
+
+    return this.finalize(stream, outputFormat, options.duration)
+  }
+
+  private async finalize(
+    stream: ReadableStream<Uint8Array>,
+    outputFormat: ElevenLabsOutputFormat | undefined,
+    duration: number | undefined,
+  ): Promise<AudioGenerationResult> {
+    const buffer = await readStreamToArrayBuffer(stream)
+    const base64 = arrayBufferToBase64(buffer)
+    const { contentType } = parseOutputFormat(outputFormat)
+    return {
+      id: generateId(this.name),
+      model: this.model,
+      audio: {
+        b64Json: base64,
+        contentType,
+        ...(duration != null ? { duration } : {}),
+      },
+    }
+  }
+
+  protected override generateId(): string {
+    return generateId(this.name)
+  }
+}
+
+function toMusicPrompt(plan: ElevenLabsMusicCompositionPlan) {
+  return {
+    positiveGlobalStyles: plan.positiveGlobalStyles ?? [],
+    negativeGlobalStyles: plan.negativeGlobalStyles ?? [],
+    sections: (plan.sections ?? []).map((section) => ({
+      sectionName: section.sectionName,
+      positiveLocalStyles: section.positiveLocalStyles ?? [],
+      negativeLocalStyles: section.negativeLocalStyles ?? [],
+      durationMs: section.durationMs ?? 10000,
+      lines: section.lines ?? [],
+    })),
+  }
+}
+
+/**
+ * Create an ElevenLabs audio adapter using `ELEVENLABS_API_KEY` from env.
+ */
+export function elevenlabsAudio<TModel extends ElevenLabsAudioModel>(
+  model: TModel,
+  config?: ElevenLabsClientConfig,
+): ElevenLabsAudioAdapter<TModel> {
+  return new ElevenLabsAudioAdapter(model, config)
+}
+
+/**
+ * Create an ElevenLabs audio adapter with an explicit API key.
+ */
+export function createElevenLabsAudio<TModel extends ElevenLabsAudioModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<ElevenLabsClientConfig, 'apiKey'>,
+): ElevenLabsAudioAdapter<TModel> {
+  return new ElevenLabsAudioAdapter(model, { apiKey, ...config })
+}

package/src/adapters/speech.ts

@@ -0,0 +1,240 @@
+import { BaseTTSAdapter } from '@tanstack/ai/adapters'
+import {
+  arrayBufferToBase64,
+  createElevenLabsClient,
+  generateId,
+  parseOutputFormat,
+  readStreamToArrayBuffer,
+} from '../utils/client'
+import type { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
+import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type { ElevenLabsClientConfig } from '../utils/client'
+import type { ElevenLabsOutputFormat, ElevenLabsTTSModel } from '../model-meta'
+
+/**
+ * ElevenLabs voice settings overrides. All fields are optional — omitted
+ * values fall back to the voice's stored defaults.
+ * @see https://elevenlabs.io/docs/api-reference/text-to-speech/convert
+ */
+export interface ElevenLabsVoiceSettings {
+  /** Voice stability, 0..1. Default 0.5. */
+  stability?: number
+  /** Similarity boost, 0..1. Default 0.75. */
+  similarityBoost?: number
+  /** Style exaggeration, 0..1. Default 0. */
+  style?: number
+  /** Playback speed. Default 1.0. */
+  speed?: number
+  /** Clarity/presence boost. Default true. */
+  useSpeakerBoost?: boolean
+}
+
+/**
+ * Provider-specific TTS options. `voice` on `generateSpeech()` takes priority
+ * over `voiceId` here, but we expose the same field for callers that prefer
+ * to keep voice configuration inside the adapter config.
+ */
+export interface ElevenLabsSpeechProviderOptions {
+  /** ElevenLabs voice ID to synthesize. Required if `generateSpeech().voice` is not set. */
+  voiceId?: string
+  /** Output audio format encoded as `codec_samplerate[_bitrate]`. Defaults to `mp3_44100_128`. */
+  outputFormat?: ElevenLabsOutputFormat
+  /** Voice-settings overrides for this request only. */
+  voiceSettings?: ElevenLabsVoiceSettings
+  /** ISO-639-1 language code to enforce (e.g. `'en'`, `'ja'`). */
+  languageCode?: string
+  /** Deterministic sampling seed, 0..4294967295. */
+  seed?: number
+  /** Previous text for stitching adjacent clips. */
+  previousText?: string
+  /** Next text for stitching adjacent clips. */
+  nextText?: string
+  /** Previous request IDs for stitching (max 3). */
+  previousRequestIds?: Array<string>
+  /** Next request IDs for stitching (max 3). */
+  nextRequestIds?: Array<string>
+  /** Text normalization toggle. Default `'auto'`. */
+  applyTextNormalization?: 'auto' | 'on' | 'off'
+  /** Language-specific text normalization (currently Japanese only, adds latency). */
+  applyLanguageTextNormalization?: boolean
+  /** Latency optimization level, 0..4. */
+  optimizeStreamingLatency?: number
+  /** Enable logging. Set false for zero-retention mode (enterprise only). */
+  enableLogging?: boolean
+}
+
+/**
+ * ElevenLabs text-to-speech adapter built on the official
+ * `@elevenlabs/elevenlabs-js` SDK.
+ *
+ * @example
+ * ```ts
+ * const adapter = elevenlabsSpeech('eleven_multilingual_v2')
+ * const result = await generateSpeech({
+ *   adapter,
+ *   text: 'Hello, world!',
+ *   voice: '21m00Tcm4TlvDq8ikWAM',
+ * })
+ * ```
+ */
+export class ElevenLabsSpeechAdapter<
+  TModel extends ElevenLabsTTSModel,
+> extends BaseTTSAdapter<TModel, ElevenLabsSpeechProviderOptions> {
+  readonly name = 'elevenlabs' as const
+
+  private client: ElevenLabsClient
+
+  constructor(model: TModel, config?: ElevenLabsClientConfig) {
+    super(model, config ?? {})
+    this.client = createElevenLabsClient(config)
+  }
+
+  async generateSpeech(
+    options: TTSOptions<ElevenLabsSpeechProviderOptions>,
+  ): Promise<TTSResult> {
+    const { logger } = options
+    logger.request(
+      `activity=generateSpeech provider=elevenlabs model=${this.model}`,
+      { provider: 'elevenlabs', model: this.model },
+    )
+    try {
+      const voiceId = options.voice ?? options.modelOptions?.voiceId
+      if (!voiceId) {
+        throw new Error(
+          'ElevenLabs TTS requires a voice. Pass `voice` on generateSpeech() or `voiceId` in modelOptions.',
+        )
+      }
+      const {
+        outputFormat,
+        voiceSettings,
+        languageCode,
+        seed,
+        previousText,
+        nextText,
+        previousRequestIds,
+        nextRequestIds,
+        applyTextNormalization,
+        applyLanguageTextNormalization,
+        optimizeStreamingLatency,
+        enableLogging,
+      } = options.modelOptions ?? {}
+      const effectiveOutputFormat =
+        outputFormat ?? inferOutputFormatFromResponseFormat(options.format)
+
+      const stream = await this.client.textToSpeech.convert(voiceId, {
+        text: options.text,
+        modelId: this.model,
+        ...(effectiveOutputFormat
+          ? { outputFormat: effectiveOutputFormat }
+          : {}),
+        ...(voiceSettings
+          ? { voiceSettings: mapVoiceSettings(voiceSettings, options.speed) }
+          : options.speed != null
+            ? { voiceSettings: { speed: options.speed } }
+            : {}),
+        ...(languageCode ? { languageCode } : {}),
+        ...(seed != null ? { seed } : {}),
+        ...(previousText ? { previousText } : {}),
+        ...(nextText ? { nextText } : {}),
+        ...(previousRequestIds ? { previousRequestIds } : {}),
+        ...(nextRequestIds ? { nextRequestIds } : {}),
+        ...(applyTextNormalization ? { applyTextNormalization } : {}),
+        ...(applyLanguageTextNormalization != null
+          ? { applyLanguageTextNormalization }
+          : {}),
+        ...(optimizeStreamingLatency != null
+          ? { optimizeStreamingLatency }
+          : {}),
+        ...(enableLogging != null ? { enableLogging } : {}),
+      })
+
+      const buffer = await readStreamToArrayBuffer(stream)
+      const base64 = arrayBufferToBase64(buffer)
+      const { format, contentType } = parseOutputFormat(effectiveOutputFormat)
+
+      return {
+        id: generateId(this.name),
+        model: this.model,
+        audio: base64,
+        format,
+        contentType,
+      }
+    } catch (error) {
+      logger.errors('elevenlabs.generateSpeech fatal', {
+        error,
+        source: 'elevenlabs.generateSpeech',
+      })
+      throw error
+    }
+  }
+
+  protected override generateId(): string {
+    return generateId(this.name)
+  }
+}
+
+function mapVoiceSettings(
+  settings: ElevenLabsVoiceSettings,
+  speedOverride: number | undefined,
+): Record<string, unknown> {
+  return {
+    ...(settings.stability != null ? { stability: settings.stability } : {}),
+    ...(settings.similarityBoost != null
+      ? { similarityBoost: settings.similarityBoost }
+      : {}),
+    ...(settings.style != null ? { style: settings.style } : {}),
+    ...(speedOverride != null
+      ? { speed: speedOverride }
+      : settings.speed != null
+        ? { speed: settings.speed }
+        : {}),
+    ...(settings.useSpeakerBoost != null
+      ? { useSpeakerBoost: settings.useSpeakerBoost }
+      : {}),
+  }
+}
+
+/**
+ * Map the standard TTSOptions `format` (mp3/opus/aac/flac/wav/pcm) to a
+ * reasonable ElevenLabs `outputFormat` so callers don't need to know the
+ * full codec/samplerate string for the common case.
+ */
+function inferOutputFormatFromResponseFormat(
+  format: TTSOptions['format'] | undefined,
+): ElevenLabsOutputFormat | undefined {
+  switch (format) {
+    case 'mp3':
+      return 'mp3_44100_128'
+    case 'pcm':
+      return 'pcm_44100'
+    case 'opus':
+      return 'opus_48000_128'
+    case undefined:
+      return undefined
+    default:
+      // `aac` / `flac` / `wav` are not native ElevenLabs formats —
+      // fall back to mp3 rather than blowing up mid-request.
+      return 'mp3_44100_128'
+  }
+}
+
+/**
+ * Create an ElevenLabs speech adapter using `ELEVENLABS_API_KEY` from env.
+ */
+export function elevenlabsSpeech<TModel extends ElevenLabsTTSModel>(
+  model: TModel,
+  config?: ElevenLabsClientConfig,
+): ElevenLabsSpeechAdapter<TModel> {
+  return new ElevenLabsSpeechAdapter(model, config)
+}
+
+/**
+ * Create an ElevenLabs speech adapter with an explicit API key.
+ */
+export function createElevenLabsSpeech<TModel extends ElevenLabsTTSModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<ElevenLabsClientConfig, 'apiKey'>,
+): ElevenLabsSpeechAdapter<TModel> {
+  return new ElevenLabsSpeechAdapter(model, { apiKey, ...config })
+}