@ai-sdk/elevenlabs 2.0.7 → 2.0.9

package/src/elevenlabs-speech-model.ts

@@ -0,0 +1,258 @@
+import { SpeechModelV3, SharedV3Warning } from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createBinaryResponseHandler,
+  parseProviderOptions,
+  postJsonToApi,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { ElevenLabsConfig } from './elevenlabs-config';
+import { elevenlabsFailedResponseHandler } from './elevenlabs-error';
+import { ElevenLabsSpeechAPITypes } from './elevenlabs-speech-api-types';
+import {
+  ElevenLabsSpeechModelId,
+  ElevenLabsSpeechVoiceId,
+} from './elevenlabs-speech-options';
+
+// Schema for camelCase input from users
+const ElevenLabsProviderOptionsSchema = z.object({
+  languageCode: z.string().optional(),
+  voiceSettings: z
+    .object({
+      stability: z.number().min(0).max(1).optional(),
+      similarityBoost: z.number().min(0).max(1).optional(),
+      style: z.number().min(0).max(1).optional(),
+      useSpeakerBoost: z.boolean().optional(),
+    })
+    .optional(),
+  pronunciationDictionaryLocators: z
+    .array(
+      z.object({
+        pronunciationDictionaryId: z.string(),
+        versionId: z.string().optional(),
+      }),
+    )
+    .max(3)
+    .optional(),
+  seed: z.number().min(0).max(4294967295).optional(),
+  previousText: z.string().optional(),
+  nextText: z.string().optional(),
+  previousRequestIds: z.array(z.string()).max(3).optional(),
+  nextRequestIds: z.array(z.string()).max(3).optional(),
+  applyTextNormalization: z.enum(['auto', 'on', 'off']).optional(),
+  applyLanguageTextNormalization: z.boolean().optional(),
+  enableLogging: z.boolean().optional(),
+});
+
+export type ElevenLabsSpeechCallOptions = z.infer<
+  typeof ElevenLabsProviderOptionsSchema
+>;
+
+interface ElevenLabsSpeechModelConfig extends ElevenLabsConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+export class ElevenLabsSpeechModel implements SpeechModelV3 {
+  readonly specificationVersion = 'v3';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: ElevenLabsSpeechModelId,
+    private readonly config: ElevenLabsSpeechModelConfig,
+  ) {}
+
+  private async getArgs({
+    text,
+    voice = '21m00Tcm4TlvDq8ikWAM',
+    outputFormat = 'mp3_44100_128',
+    instructions,
+    language,
+    speed,
+    providerOptions,
+  }: Parameters<SpeechModelV3['doGenerate']>[0]) {
+    const warnings: SharedV3Warning[] = [];
+
+    // Parse provider options
+    const elevenLabsOptions = await parseProviderOptions({
+      provider: 'elevenlabs',
+      providerOptions,
+      schema: ElevenLabsProviderOptionsSchema,
+    });
+
+    // Create request body
+    const requestBody: ElevenLabsSpeechAPITypes = {
+      text,
+      model_id: this.modelId,
+    };
+
+    // Prepare query parameters
+    const queryParams: Record<string, string> = {};
+
+    // Map outputFormat to ElevenLabs format (as query param)
+    if (outputFormat) {
+      const formatMap: Record<string, string> = {
+        mp3: 'mp3_44100_128',
+        mp3_32: 'mp3_44100_32',
+        mp3_64: 'mp3_44100_64',
+        mp3_96: 'mp3_44100_96',
+        mp3_128: 'mp3_44100_128',
+        mp3_192: 'mp3_44100_192',
+        pcm: 'pcm_44100',
+        pcm_16000: 'pcm_16000',
+        pcm_22050: 'pcm_22050',
+        pcm_24000: 'pcm_24000',
+        pcm_44100: 'pcm_44100',
+        ulaw: 'ulaw_8000',
+      };
+
+      const mappedFormat = formatMap[outputFormat] || outputFormat;
+      queryParams.output_format = mappedFormat;
+    }
+
+    // Add language code if provided
+    if (language) {
+      requestBody.language_code = language;
+    }
+
+    const voiceSettings: typeof requestBody.voice_settings = {};
+
+    if (speed != null) {
+      voiceSettings.speed = speed;
+    }
+
+    // Add provider-specific options - map from camelCase to snake_case
+    if (elevenLabsOptions) {
+      if (elevenLabsOptions.voiceSettings) {
+        // Map camelCase voice settings to snake_case for API
+        if (elevenLabsOptions.voiceSettings.stability != null) {
+          voiceSettings.stability = elevenLabsOptions.voiceSettings.stability;
+        }
+        if (elevenLabsOptions.voiceSettings.similarityBoost != null) {
+          voiceSettings.similarity_boost =
+            elevenLabsOptions.voiceSettings.similarityBoost;
+        }
+        if (elevenLabsOptions.voiceSettings.style != null) {
+          voiceSettings.style = elevenLabsOptions.voiceSettings.style;
+        }
+        if (elevenLabsOptions.voiceSettings.useSpeakerBoost != null) {
+          voiceSettings.use_speaker_boost =
+            elevenLabsOptions.voiceSettings.useSpeakerBoost;
+        }
+      }
+      // Add language code from provider options if not already set
+      if (elevenLabsOptions.languageCode && !requestBody.language_code) {
+        requestBody.language_code = elevenLabsOptions.languageCode;
+      }
+
+      // Map pronunciation dictionary locators
+      if (elevenLabsOptions.pronunciationDictionaryLocators) {
+        requestBody.pronunciation_dictionary_locators =
+          elevenLabsOptions.pronunciationDictionaryLocators.map(locator => ({
+            pronunciation_dictionary_id: locator.pronunciationDictionaryId,
+            ...(locator.versionId && { version_id: locator.versionId }),
+          }));
+      }
+
+      if (elevenLabsOptions.seed != null) {
+        requestBody.seed = elevenLabsOptions.seed;
+      }
+      if (elevenLabsOptions.previousText) {
+        requestBody.previous_text = elevenLabsOptions.previousText;
+      }
+      if (elevenLabsOptions.nextText) {
+        requestBody.next_text = elevenLabsOptions.nextText;
+      }
+
+      // Add previous and next request IDs
+      if (elevenLabsOptions.previousRequestIds) {
+        requestBody.previous_request_ids = elevenLabsOptions.previousRequestIds;
+      }
+      if (elevenLabsOptions.nextRequestIds) {
+        requestBody.next_request_ids = elevenLabsOptions.nextRequestIds;
+      }
+
+      // Add text normalization options
+      if (elevenLabsOptions.applyTextNormalization) {
+        requestBody.apply_text_normalization =
+          elevenLabsOptions.applyTextNormalization;
+      }
+      if (elevenLabsOptions.applyLanguageTextNormalization != null) {
+        requestBody.apply_language_text_normalization =
+          elevenLabsOptions.applyLanguageTextNormalization;
+      }
+
+      // enable_logging is a query parameter
+      if (elevenLabsOptions.enableLogging != null) {
+        queryParams.enable_logging = String(elevenLabsOptions.enableLogging);
+      }
+    }
+
+    // Only add voice_settings if there are settings to add
+    if (Object.keys(voiceSettings).length > 0) {
+      requestBody.voice_settings = voiceSettings;
+    }
+
+    if (instructions) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'instructions',
+        details: `ElevenLabs speech models do not support instructions. Instructions parameter was ignored.`,
+      });
+    }
+
+    return {
+      requestBody,
+      queryParams,
+      warnings,
+      voiceId: voice as ElevenLabsSpeechVoiceId,
+    };
+  }
+
+  async doGenerate(
+    options: Parameters<SpeechModelV3['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<SpeechModelV3['doGenerate']>>> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const { requestBody, queryParams, warnings, voiceId } =
+      await this.getArgs(options);
+
+    const {
+      value: audio,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postJsonToApi({
+      url: (() => {
+        const baseUrl = this.config.url({
+          path: `/v1/text-to-speech/${voiceId}`,
+          modelId: this.modelId,
+        });
+        const queryString = new URLSearchParams(queryParams).toString();
+        return queryString ? `${baseUrl}?${queryString}` : baseUrl;
+      })(),
+      headers: combineHeaders(this.config.headers(), options.headers),
+      body: requestBody,
+      failedResponseHandler: elevenlabsFailedResponseHandler,
+      successfulResponseHandler: createBinaryResponseHandler(),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    return {
+      audio,
+      warnings,
+      request: {
+        body: JSON.stringify(requestBody),
+      },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+    };
+  }
+}

package/src/elevenlabs-speech-options.ts

@@ -0,0 +1,12 @@
+export type ElevenLabsSpeechModelId =
+  | 'eleven_v3'
+  | 'eleven_multilingual_v2'
+  | 'eleven_flash_v2_5'
+  | 'eleven_flash_v2'
+  | 'eleven_turbo_v2_5'
+  | 'eleven_turbo_v2'
+  | 'eleven_monolingual_v1'
+  | 'eleven_multilingual_v1'
+  | (string & {});
+
+export type ElevenLabsSpeechVoiceId = string;

package/src/elevenlabs-transcription-model.test.ts

@@ -0,0 +1,389 @@
+import { createTestServer } from '@ai-sdk/test-server/with-vitest';
+import { ElevenLabsTranscriptionModel } from './elevenlabs-transcription-model';
+import { createElevenLabs } from './elevenlabs-provider';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { describe, it, expect, vi } from 'vitest';
+
+vi.mock('./version', () => ({
+  VERSION: '0.0.0-test',
+}));
+
+const audioData = await readFile(path.join(__dirname, 'transcript-test.mp3'));
+const provider = createElevenLabs({ apiKey: 'test-api-key' });
+const model = provider.transcription('scribe_v1');
+
+const server = createTestServer({
+  'https://api.elevenlabs.io/v1/speech-to-text': {},
+});
+
+describe('doGenerate', () => {
+  function prepareJsonResponse({
+    headers,
+  }: {
+    headers?: Record<string, string>;
+  } = {}) {
+    server.urls['https://api.elevenlabs.io/v1/speech-to-text'].response = {
+      type: 'json-value',
+      headers,
+      body: {
+        language_code: 'en',
+        language_probability: 0.98,
+        text: 'Hello world!',
+        words: [
+          {
+            text: 'Hello',
+            type: 'word',
+            start: 0,
+            end: 0.5,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+          {
+            text: ' ',
+            type: 'spacing',
+            start: 0.5,
+            end: 0.5,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+          {
+            text: 'world!',
+            type: 'word',
+            start: 0.5,
+            end: 1.2,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+        ],
+        additional_formats: [
+          {
+            requested_format: 'requested_format',
+            file_extension: 'file_extension',
+            content_type: 'content_type',
+            is_base64_encoded: true,
+            content: 'content',
+          },
+        ],
+      },
+    };
+  }
+
+  it('should pass the model', async () => {
+    prepareJsonResponse();
+
+    await model.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+    });
+
+    expect(await server.calls[0].requestBodyMultipart).toMatchObject({
+      model_id: 'scribe_v1',
+    });
+  });
+
+  it('should pass headers', async () => {
+    prepareJsonResponse();
+
+    const provider = createElevenLabs({
+      apiKey: 'test-api-key',
+      headers: {
+        'Custom-Provider-Header': 'provider-header-value',
+      },
+    });
+
+    await provider.transcription('scribe_v1').doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+      headers: {
+        'Custom-Request-Header': 'request-header-value',
+      },
+    });
+
+    expect(server.calls[0].requestHeaders).toMatchObject({
+      'xi-api-key': 'test-api-key',
+      'content-type': expect.stringMatching(
+        /^multipart\/form-data; boundary=----formdata-undici-\d+$/,
+      ),
+      'custom-provider-header': 'provider-header-value',
+      'custom-request-header': 'request-header-value',
+    });
+    expect(server.calls[0].requestUserAgent).toContain(
+      `ai-sdk/elevenlabs/0.0.0-test`,
+    );
+  });
+
+  it('should extract the transcription text', async () => {
+    prepareJsonResponse();
+
+    const result = await model.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+    });
+
+    expect(result.text).toBe('Hello world!');
+  });
+
+  it('should include response data with timestamp, modelId and headers', async () => {
+    prepareJsonResponse({
+      headers: {
+        'x-request-id': 'test-request-id',
+        'x-ratelimit-remaining': '123',
+      },
+    });
+
+    const testDate = new Date(0);
+    const customModel = new ElevenLabsTranscriptionModel('scribe_v1', {
+      provider: 'test-provider',
+      url: () => 'https://api.elevenlabs.io/v1/speech-to-text',
+      headers: () => ({}),
+      _internal: {
+        currentDate: () => testDate,
+      },
+    });
+
+    const result = await customModel.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+    });
+
+    expect(result.response).toMatchObject({
+      timestamp: testDate,
+      modelId: 'scribe_v1',
+      headers: {
+        'content-type': 'application/json',
+        'x-request-id': 'test-request-id',
+        'x-ratelimit-remaining': '123',
+      },
+    });
+  });
+
+  it('should use real date when no custom date provider is specified', async () => {
+    prepareJsonResponse();
+
+    const testDate = new Date(0);
+    const customModel = new ElevenLabsTranscriptionModel('scribe_v1', {
+      provider: 'test-provider',
+      url: () => 'https://api.elevenlabs.io/v1/speech-to-text',
+      headers: () => ({}),
+      _internal: {
+        currentDate: () => testDate,
+      },
+    });
+
+    const result = await customModel.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+    });
+
+    expect(result.response.timestamp.getTime()).toEqual(testDate.getTime());
+    expect(result.response.modelId).toBe('scribe_v1');
+  });
+
+  it('should work when no additional formats are returned', async () => {
+    server.urls['https://api.elevenlabs.io/v1/speech-to-text'].response = {
+      type: 'json-value',
+      body: {
+        language_code: 'en',
+        language_probability: 0.98,
+        text: 'Hello world!',
+        words: [
+          {
+            text: 'Hello',
+            type: 'word',
+            start: 0,
+            end: 0.5,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+          {
+            text: ' ',
+            type: 'spacing',
+            start: 0.5,
+            end: 0.5,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+          {
+            text: 'world!',
+            type: 'word',
+            start: 0.5,
+            end: 1.2,
+            speaker_id: 'speaker_1',
+            characters: [
+              {
+                text: 'text',
+                start: 0,
+                end: 0.1,
+              },
+            ],
+          },
+        ],
+      },
+    };
+
+    const testDate = new Date(0);
+    const customModel = new ElevenLabsTranscriptionModel('scribe_v1', {
+      provider: 'test-provider',
+      url: () => 'https://api.elevenlabs.io/v1/speech-to-text',
+      headers: () => ({}),
+      _internal: {
+        currentDate: () => testDate,
+      },
+    });
+
+    const result = await customModel.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+    });
+
+    expect(result).toMatchInlineSnapshot(`
+      {
+        "durationInSeconds": 1.2,
+        "language": "en",
+        "response": {
+          "body": {
+            "language_code": "en",
+            "language_probability": 0.98,
+            "text": "Hello world!",
+            "words": [
+              {
+                "characters": [
+                  {
+                    "end": 0.1,
+                    "start": 0,
+                    "text": "text",
+                  },
+                ],
+                "end": 0.5,
+                "speaker_id": "speaker_1",
+                "start": 0,
+                "text": "Hello",
+                "type": "word",
+              },
+              {
+                "characters": [
+                  {
+                    "end": 0.1,
+                    "start": 0,
+                    "text": "text",
+                  },
+                ],
+                "end": 0.5,
+                "speaker_id": "speaker_1",
+                "start": 0.5,
+                "text": " ",
+                "type": "spacing",
+              },
+              {
+                "characters": [
+                  {
+                    "end": 0.1,
+                    "start": 0,
+                    "text": "text",
+                  },
+                ],
+                "end": 1.2,
+                "speaker_id": "speaker_1",
+                "start": 0.5,
+                "text": "world!",
+                "type": "word",
+              },
+            ],
+          },
+          "headers": {
+            "content-length": "467",
+            "content-type": "application/json",
+          },
+          "modelId": "scribe_v1",
+          "timestamp": 1970-01-01T00:00:00.000Z,
+        },
+        "segments": [
+          {
+            "endSecond": 0.5,
+            "startSecond": 0,
+            "text": "Hello",
+          },
+          {
+            "endSecond": 0.5,
+            "startSecond": 0.5,
+            "text": " ",
+          },
+          {
+            "endSecond": 1.2,
+            "startSecond": 0.5,
+            "text": "world!",
+          },
+        ],
+        "text": "Hello world!",
+        "warnings": [],
+      }
+    `);
+  });
+
+  it('should pass provider options correctly', async () => {
+    prepareJsonResponse();
+
+    await model.doGenerate({
+      audio: audioData,
+      mediaType: 'audio/wav',
+      providerOptions: {
+        elevenlabs: {
+          languageCode: 'en',
+          fileFormat: 'pcm_s16le_16',
+          tagAudioEvents: false,
+          numSpeakers: 2,
+          timestampsGranularity: 'character',
+          diarize: true,
+        },
+      },
+    });
+
+    expect(await server.calls[0].requestBodyMultipart).toMatchInlineSnapshot(`
+      {
+        "diarize": "true",
+        "file": File {
+          Symbol(kHandle): Blob {},
+          Symbol(kLength): 40169,
+          Symbol(kType): "audio/wav",
+        },
+        "file_format": "pcm_s16le_16",
+        "language_code": "en",
+        "model_id": "scribe_v1",
+        "num_speakers": "2",
+        "tag_audio_events": "false",
+        "timestamps_granularity": "character",
+      }
+    `);
+  });
+});