@tanstack/ai-elevenlabs 0.1.7 → 0.2.0

package/src/adapters/transcription.ts

@@ -0,0 +1,338 @@
+import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'
+import {
+  createElevenLabsClient,
+  dataUrlToBlob,
+  generateId,
+} from '../utils/client'
+import type { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'
+import type {
+  TranscriptionOptions,
+  TranscriptionResult,
+  TranscriptionSegment,
+  TranscriptionWord,
+} from '@tanstack/ai'
+import type { ElevenLabsClientConfig } from '../utils/client'
+import type { ElevenLabsTranscriptionModel } from '../model-meta'
+
+/**
+ * Provider-specific options for ElevenLabs Scribe transcription. Fields map
+ * 1:1 onto the SDK's `BodySpeechToTextV1SpeechToTextPost` — mirroring the
+ * names so documentation stays useful.
+ * @see https://elevenlabs.io/docs/api-reference/speech-to-text/convert
+ */
+export interface ElevenLabsTranscriptionProviderOptions {
+  /** Annotate non-speech events like (laughter), (footsteps), …. */
+  tagAudioEvents?: boolean
+  /** Maximum number of speakers in the audio (1..32). */
+  numSpeakers?: number
+  /** Timestamp granularity for words. */
+  timestampsGranularity?: 'word' | 'character' | 'none'
+  /** Enable speaker diarization. */
+  diarize?: boolean
+  /** Diarization threshold (requires `diarize=true` and no `numSpeakers`). */
+  diarizationThreshold?: number
+  /** Detect speaker roles (agent/customer). Requires diarize=true. */
+  detectSpeakerRoles?: boolean
+  /** Bias the model towards these keyterms (max 1000). */
+  keyterms?: Array<string>
+  /**
+   * Entity detection: `'all'`, a category (`'pii'`, `'phi'`, `'pci'`,
+   * `'other'`, `'offensive_language'`), or a specific entity type.
+   */
+  entityDetection?: string
+  /** Redact entities from the transcript text. Must be a subset of `entityDetection`. */
+  entityRedaction?: string
+  /** How redacted entities are formatted. */
+  entityRedactionMode?: string
+  /** Whether to skip filler words / non-speech sounds (scribe_v2 only). */
+  noVerbatim?: boolean
+  /** Sampling temperature (0..2). */
+  temperature?: number
+  /** Deterministic sampling seed (0..2147483647). */
+  seed?: number
+  /** Use `false` for zero-retention mode (enterprise only). */
+  enableLogging?: boolean
+  /** Multi-channel audio with one speaker per channel. Max 5 channels. */
+  useMultiChannel?: boolean
+  /**
+   * Hint for audio format. Use `'pcm_s16le_16'` to skip encoding for 16-bit
+   * PCM @ 16kHz mono little-endian inputs (lower latency).
+   */
+  fileFormat?: 'pcm_s16le_16' | 'other'
+}
+
+/**
+ * ElevenLabs speech-to-text adapter built on the official SDK's Scribe family.
+ *
+ * @example
+ * ```ts
+ * const adapter = elevenlabsTranscription('scribe_v1')
+ * const result = await generateTranscription({
+ *   adapter,
+ *   audio: fileInput,
+ *   language: 'en',
+ * })
+ * ```
+ */
+export class ElevenLabsTranscriptionAdapter<
+  TModel extends ElevenLabsTranscriptionModel,
+> extends BaseTranscriptionAdapter<
+  TModel,
+  ElevenLabsTranscriptionProviderOptions
+> {
+  readonly name = 'elevenlabs' as const
+
+  private client: ElevenLabsClient
+
+  constructor(model: TModel, config?: ElevenLabsClientConfig) {
+    super(model, config ?? {})
+    this.client = createElevenLabsClient(config)
+  }
+
+  async transcribe(
+    options: TranscriptionOptions<ElevenLabsTranscriptionProviderOptions>,
+  ): Promise<TranscriptionResult> {
+    const { logger } = options
+    logger.request(
+      `activity=generateTranscription provider=elevenlabs model=${this.model}`,
+      { provider: 'elevenlabs', model: this.model },
+    )
+    try {
+      const modelOpts = options.modelOptions ?? {}
+      const audioInput = normalizeAudioInput(options.audio)
+
+      const response = await this.client.speechToText.convert({
+        modelId: this.model,
+        ...(audioInput.kind === 'file'
+          ? { file: audioInput.value }
+          : { cloudStorageUrl: audioInput.value }),
+        ...(options.language ? { languageCode: options.language } : {}),
+        ...(modelOpts.tagAudioEvents != null
+          ? { tagAudioEvents: modelOpts.tagAudioEvents }
+          : {}),
+        ...(modelOpts.numSpeakers != null
+          ? { numSpeakers: modelOpts.numSpeakers }
+          : {}),
+        ...(modelOpts.timestampsGranularity
+          ? { timestampsGranularity: modelOpts.timestampsGranularity }
+          : {}),
+        ...(modelOpts.diarize != null ? { diarize: modelOpts.diarize } : {}),
+        ...(modelOpts.diarizationThreshold != null
+          ? { diarizationThreshold: modelOpts.diarizationThreshold }
+          : {}),
+        ...(modelOpts.detectSpeakerRoles != null
+          ? { detectSpeakerRoles: modelOpts.detectSpeakerRoles }
+          : {}),
+        ...(modelOpts.keyterms ? { keyterms: modelOpts.keyterms } : {}),
+        ...(modelOpts.entityDetection
+          ? { entityDetection: modelOpts.entityDetection }
+          : {}),
+        ...(modelOpts.entityRedaction
+          ? { entityRedaction: modelOpts.entityRedaction }
+          : {}),
+        ...(modelOpts.entityRedactionMode
+          ? { entityRedactionMode: modelOpts.entityRedactionMode }
+          : {}),
+        ...(modelOpts.noVerbatim != null
+          ? { noVerbatim: modelOpts.noVerbatim }
+          : {}),
+        ...(modelOpts.temperature != null
+          ? { temperature: modelOpts.temperature }
+          : {}),
+        ...(modelOpts.seed != null ? { seed: modelOpts.seed } : {}),
+        ...(modelOpts.enableLogging != null
+          ? { enableLogging: modelOpts.enableLogging }
+          : {}),
+        ...(modelOpts.useMultiChannel != null
+          ? { useMultiChannel: modelOpts.useMultiChannel }
+          : {}),
+        ...(modelOpts.fileFormat ? { fileFormat: modelOpts.fileFormat } : {}),
+      } as Parameters<ElevenLabsClient['speechToText']['convert']>[0])
+
+      return this.transformResponse(response)
+    } catch (error) {
+      logger.errors('elevenlabs.generateTranscription fatal', {
+        error,
+        source: 'elevenlabs.generateTranscription',
+      })
+      throw error
+    }
+  }
+
+  private transformResponse(
+    response: Awaited<ReturnType<ElevenLabsClient['speechToText']['convert']>>,
+  ): TranscriptionResult {
+    // The SDK types this as a union of single- and multi-channel responses.
+    // We treat multi-channel as "join the channel transcripts" — consumers
+    // who care about per-channel detail can re-parse from `modelOptions`.
+    const data = response as unknown as {
+      text?: string
+      languageCode?: string
+      languageProbability?: number
+      words?: Array<{
+        text: string
+        start?: number
+        end?: number
+        type: string
+        speakerId?: string
+      }>
+      audioDurationSecs?: number
+      transcripts?: Array<{
+        text?: string
+        languageCode?: string
+        words?: Array<{
+          text: string
+          start?: number
+          end?: number
+          type: string
+          speakerId?: string
+        }>
+        audioDurationSecs?: number
+      }>
+    }
+
+    if (data.transcripts) {
+      const joinedText = data.transcripts
+        .map((t) => t.text ?? '')
+        .filter(Boolean)
+        .join('\n')
+      const joinedWords = data.transcripts.flatMap((t) => t.words ?? [])
+      const duration = data.transcripts.reduce(
+        (max, t) => Math.max(max, t.audioDurationSecs ?? 0),
+        0,
+      )
+      const firstLang = data.transcripts.find(
+        (t) => t.languageCode,
+      )?.languageCode
+      return {
+        id: generateId(this.name),
+        model: this.model,
+        text: joinedText,
+        ...(firstLang ? { language: firstLang } : {}),
+        ...(duration ? { duration } : {}),
+        ...buildWordsAndSegments(joinedWords),
+      }
+    }
+
+    return {
+      id: generateId(this.name),
+      model: this.model,
+      text: data.text ?? '',
+      ...(data.languageCode ? { language: data.languageCode } : {}),
+      ...(data.audioDurationSecs ? { duration: data.audioDurationSecs } : {}),
+      ...buildWordsAndSegments(data.words ?? []),
+    }
+  }
+
+  protected override generateId(): string {
+    return generateId(this.name)
+  }
+}
+
+type NormalizedAudio =
+  | { kind: 'file'; value: Blob }
+  | { kind: 'url'; value: string }
+
+function normalizeAudioInput(
+  audio: TranscriptionOptions['audio'],
+): NormalizedAudio {
+  if (audio instanceof ArrayBuffer) {
+    return { kind: 'file', value: new Blob([audio]) }
+  }
+  if (typeof audio === 'string') {
+    const blob = dataUrlToBlob(audio)
+    if (blob) return { kind: 'file', value: blob }
+    return { kind: 'url', value: audio }
+  }
+  // Blob or File both fit the SDK's `Uploadable` contract.
+  return { kind: 'file', value: audio }
+}
+
+function buildWordsAndSegments(
+  words: Array<{
+    text: string
+    start?: number
+    end?: number
+    type: string
+    speakerId?: string
+  }>,
+): {
+  words?: Array<TranscriptionWord>
+  segments?: Array<TranscriptionSegment>
+} {
+  const timedWords = words.filter(
+    (w) =>
+      typeof w.start === 'number' &&
+      typeof w.end === 'number' &&
+      w.type !== 'spacing',
+  )
+  if (timedWords.length === 0) return {}
+
+  const outWords: Array<TranscriptionWord> = timedWords.map((w) => ({
+    word: w.text,
+    start: w.start!,
+    end: w.end!,
+  }))
+
+  // Group contiguous words that share a speaker into segments. If no speaker
+  // is ever set, we still emit one segment per sentence-ish grouping.
+  const segments: Array<TranscriptionSegment> = []
+  let current: {
+    start: number
+    end: number
+    text: string
+    speaker?: string
+  } | null = null
+
+  for (const w of timedWords) {
+    if (!current) {
+      current = {
+        start: w.start!,
+        end: w.end!,
+        text: w.text,
+        ...(w.speakerId ? { speaker: w.speakerId } : {}),
+      }
+      continue
+    }
+    if (w.speakerId && current.speaker !== w.speakerId) {
+      segments.push({ id: segments.length, ...current })
+      current = {
+        start: w.start!,
+        end: w.end!,
+        text: w.text,
+        speaker: w.speakerId,
+      }
+      continue
+    }
+    current.end = w.end!
+    current.text = current.text ? `${current.text} ${w.text}` : w.text
+  }
+  if (current) segments.push({ id: segments.length, ...current })
+
+  return { words: outWords, segments }
+}
+
+/**
+ * Create an ElevenLabs transcription adapter using `ELEVENLABS_API_KEY` from env.
+ */
+export function elevenlabsTranscription<
+  TModel extends ElevenLabsTranscriptionModel,
+>(
+  model: TModel,
+  config?: ElevenLabsClientConfig,
+): ElevenLabsTranscriptionAdapter<TModel> {
+  return new ElevenLabsTranscriptionAdapter(model, config)
+}
+
+/**
+ * Create an ElevenLabs transcription adapter with an explicit API key.
+ */
+export function createElevenLabsTranscription<
+  TModel extends ElevenLabsTranscriptionModel,
+>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<ElevenLabsClientConfig, 'apiKey'>,
+): ElevenLabsTranscriptionAdapter<TModel> {
+  return new ElevenLabsTranscriptionAdapter(model, { apiKey, ...config })
+}

package/src/index.ts

@@ -11,3 +11,67 @@ export type {
   ElevenLabsVADConfig,
   ElevenLabsClientTool,
 } from './realtime/index'
+
+// ============================================================================
+// Speech (Text-to-Speech) Adapter
+// ============================================================================
+
+export {
+  ElevenLabsSpeechAdapter,
+  createElevenLabsSpeech,
+  elevenlabsSpeech,
+  type ElevenLabsSpeechProviderOptions,
+  type ElevenLabsVoiceSettings,
+} from './adapters/speech'
+
+// ============================================================================
+// Audio (Music + Sound Effects) Adapter
+// ============================================================================
+
+export {
+  ElevenLabsAudioAdapter,
+  createElevenLabsAudio,
+  elevenlabsAudio,
+  type ElevenLabsAudioProviderOptions,
+  type ElevenLabsMusicProviderOptions,
+  type ElevenLabsSoundEffectsProviderOptions,
+  type ElevenLabsMusicCompositionPlan,
+} from './adapters/audio'
+
+// ============================================================================
+// Transcription (Speech-to-Text) Adapter
+// ============================================================================
+
+export {
+  ElevenLabsTranscriptionAdapter,
+  createElevenLabsTranscription,
+  elevenlabsTranscription,
+  type ElevenLabsTranscriptionProviderOptions,
+} from './adapters/transcription'
+
+// ============================================================================
+// Model Metadata
+// ============================================================================
+
+export {
+  ELEVENLABS_TTS_MODELS,
+  ELEVENLABS_AUDIO_MODELS,
+  ELEVENLABS_TRANSCRIPTION_MODELS,
+  isElevenLabsMusicModel,
+  isElevenLabsSoundEffectsModel,
+  type ElevenLabsTTSModel,
+  type ElevenLabsAudioModel,
+  type ElevenLabsMusicModel,
+  type ElevenLabsSoundEffectsModel,
+  type ElevenLabsTranscriptionModel,
+  type ElevenLabsOutputFormat,
+} from './model-meta'
+
+// ============================================================================
+// Utilities
+// ============================================================================
+
+export {
+  getElevenLabsApiKeyFromEnv,
+  type ElevenLabsClientConfig,
+} from './utils/index'

package/src/model-meta.ts

@@ -0,0 +1,79 @@
+import type { ElevenLabs } from '@elevenlabs/elevenlabs-js'
+
+/**
+ * ElevenLabs model identifiers. The lists below are the source of truth —
+ * callers are blocked from passing unknown model IDs. Keep them in sync with
+ * the ElevenLabs SDK via the automated update pipeline.
+ */
+
+/**
+ * Text-to-speech models.
+ * @see https://elevenlabs.io/docs/models
+ */
+export const ELEVENLABS_TTS_MODELS = [
+  'eleven_v3',
+  'eleven_multilingual_v2',
+  'eleven_flash_v2_5',
+  'eleven_flash_v2',
+  'eleven_turbo_v2_5',
+  'eleven_turbo_v2',
+  'eleven_monolingual_v1',
+] as const
+
+export type ElevenLabsTTSModel = (typeof ELEVENLABS_TTS_MODELS)[number]
+
+/**
+ * Audio generation models — music (`music_v1`) + sound effects
+ * (`eleven_text_to_sound_v*`) share one `generateAudio` adapter.
+ * The adapter dispatches by model id so callers pick behavior via the model.
+ *
+ * @see https://elevenlabs.io/docs/overview/capabilities/music
+ * @see https://elevenlabs.io/docs/overview/capabilities/sound-effects
+ */
+export const ELEVENLABS_AUDIO_MODELS = [
+  'music_v1',
+  'eleven_text_to_sound_v2',
+  'eleven_text_to_sound_v1',
+] as const
+
+export type ElevenLabsAudioModel = (typeof ELEVENLABS_AUDIO_MODELS)[number]
+
+/** Music models within the audio family. */
+export type ElevenLabsMusicModel = 'music_v1'
+/** SFX models within the audio family. */
+export type ElevenLabsSoundEffectsModel =
+  | 'eleven_text_to_sound_v2'
+  | 'eleven_text_to_sound_v1'
+
+export function isElevenLabsMusicModel(
+  model: string,
+): model is ElevenLabsMusicModel {
+  return model === 'music_v1'
+}
+
+export function isElevenLabsSoundEffectsModel(
+  model: string,
+): model is ElevenLabsSoundEffectsModel {
+  return model.startsWith('eleven_text_to_sound_')
+}
+
+/**
+ * Speech-to-text (transcription) models — Scribe family.
+ * @see https://elevenlabs.io/docs/overview/capabilities/speech-to-text
+ */
+export const ELEVENLABS_TRANSCRIPTION_MODELS = [
+  'scribe_v2',
+  'scribe_v1',
+] as const
+
+export type ElevenLabsTranscriptionModel =
+  (typeof ELEVENLABS_TRANSCRIPTION_MODELS)[number]
+
+/**
+ * Supported `output_format` strings, encoded as `codec_samplerate[_bitrate]`.
+ * Aliased to the SDK's `AllowedOutputFormats` so the list stays in sync
+ * automatically whenever the `@elevenlabs/elevenlabs-js` dependency is bumped.
+ *
+ * @see https://elevenlabs.io/docs/api-reference/text-to-speech/convert
+ */
+export type ElevenLabsOutputFormat = ElevenLabs.AllowedOutputFormats

package/src/realtime/adapter.ts

@@ -1,4 +1,4 @@
-import { Conversation } from '@11labs/client'
+import { Conversation } from '@elevenlabs/client'
 import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
 import type {
   AnyClientTool,
@@ -18,7 +18,7 @@ import type { ElevenLabsRealtimeOptions } from './types'
 /**
  * Creates an ElevenLabs realtime adapter for client-side use.
  *
- * Wraps the @11labs/client SDK for voice conversations.
+ * Wraps the @elevenlabs/client SDK for voice conversations.
  *
  * @param options - Optional configuration
  * @returns A RealtimeAdapter for use with RealtimeClient
@@ -91,7 +91,7 @@ async function createElevenLabsConnection(
   }
 
   // Convert TanStack tool definitions to ElevenLabs clientTools format.
-  // @11labs/client@0.2.0 expects plain async functions, not objects.
+  // @elevenlabs/client expects plain async functions, not objects.
   const elevenLabsClientTools: Record<
     string,
     (params: unknown) => Promise<string>

package/src/realtime/token.ts

@@ -1,40 +1,19 @@
+import {
+  createElevenLabsClient,
+  getElevenLabsAgentIdFromEnv,
+} from '../utils/client'
 import type { RealtimeToken, RealtimeTokenAdapter } from '@tanstack/ai'
 import type { ElevenLabsRealtimeTokenOptions } from './types'
 
-const ELEVENLABS_API_URL = 'https://api.elevenlabs.io/v1'
-
-/**
- * Get ElevenLabs API key from environment
- */
-function getElevenLabsApiKey(): string {
-  // Check process.env (Node.js)
-  if (typeof process !== 'undefined' && process.env.ELEVENLABS_API_KEY) {
-    return process.env.ELEVENLABS_API_KEY
-  }
-
-  // Check window.env (Browser with injected env)
-  if (
-    typeof window !== 'undefined' &&
-    (window as unknown as { env?: { ELEVENLABS_API_KEY?: string } }).env
-      ?.ELEVENLABS_API_KEY
-  ) {
-    return (window as unknown as { env: { ELEVENLABS_API_KEY: string } }).env
-      .ELEVENLABS_API_KEY
-  }
-
-  throw new Error(
-    'ELEVENLABS_API_KEY not found in environment variables. ' +
-      'Please set ELEVENLABS_API_KEY in your environment.',
-  )
-}
-
 /**
  * Creates an ElevenLabs realtime token adapter.
  *
- * This adapter generates signed URLs for client-side connections.
- * The signed URL is valid for 30 minutes.
+ * Uses the official `@elevenlabs/elevenlabs-js` SDK to request a signed URL
+ * for client-side conversation connections. The signed URL is valid for
+ * 30 minutes.
  *
- * @param options - Configuration options including agentId
+ * @param options - Configuration. `agentId` falls back to
+ *   `ELEVENLABS_AGENT_ID` in the environment when omitted.
  * @returns A RealtimeTokenAdapter for use with realtimeToken()
  *
  * @example
@@ -42,51 +21,37 @@ function getElevenLabsApiKey(): string {
  * import { realtimeToken } from '@tanstack/ai'
  * import { elevenlabsRealtimeToken } from '@tanstack/ai-elevenlabs'
  *
+ * // Reads ELEVENLABS_AGENT_ID from env:
+ * const token = await realtimeToken({ adapter: elevenlabsRealtimeToken() })
+ *
+ * // Or pass explicitly:
  * const token = await realtimeToken({
- *   adapter: elevenlabsRealtimeToken({
- *     agentId: 'your-agent-id',
- *   }),
+ *   adapter: elevenlabsRealtimeToken({ agentId: 'your-agent-id' }),
  * })
  * ```
  */
 export function elevenlabsRealtimeToken(
-  options: ElevenLabsRealtimeTokenOptions,
+  options: ElevenLabsRealtimeTokenOptions = {},
 ): RealtimeTokenAdapter {
-  const apiKey = getElevenLabsApiKey()
+  const client = createElevenLabsClient()
 
   return {
     provider: 'elevenlabs',
 
     async generateToken(): Promise<RealtimeToken> {
-      const { agentId, overrides } = options
+      const { overrides } = options
+      const agentId = options.agentId ?? getElevenLabsAgentIdFromEnv()
 
-      // Get signed URL from ElevenLabs
-      const response = await fetch(
-        `${ELEVENLABS_API_URL}/convai/conversation/get_signed_url?agent_id=${agentId}`,
-        {
-          method: 'GET',
-          headers: {
-            'xi-api-key': apiKey,
-          },
-        },
+      const response = await client.conversationalAi.conversations.getSignedUrl(
+        { agentId },
       )
 
-      if (!response.ok) {
-        const errorText = await response.text()
-        throw new Error(
-          `ElevenLabs signed URL request failed: ${response.status} ${errorText}`,
-        )
-      }
-
-      const data = await response.json()
-      const signedUrl = data.signed_url as string
-
       // Signed URLs are valid for 30 minutes
       const expiresAt = Date.now() + 30 * 60 * 1000
 
       return {
         provider: 'elevenlabs',
-        token: signedUrl,
+        token: response.signedUrl,
         expiresAt,
         config: {
           voice: overrides?.voiceId,

package/src/realtime/types.ts

@@ -4,8 +4,11 @@ import type { DebugOption } from '@tanstack/ai'
  * Options for the ElevenLabs realtime token adapter
  */
 export interface ElevenLabsRealtimeTokenOptions {
-  /** Agent ID configured in ElevenLabs dashboard */
-  agentId: string
+  /**
+   * Agent ID configured in ElevenLabs dashboard. Falls back to
+   * `ELEVENLABS_AGENT_ID` in the environment when omitted.
+   */
+  agentId?: string
   /** Optional override values for the agent */
   overrides?: {
     /** Custom voice ID to use */