@mastra/voice-elevenlabs 0.1.0 → 0.1.1-alpha.1

package/.turbo/turbo-build.log

@@ -1,18 +1,18 @@
 
-> @mastra/voice-elevenlabs@0.1.0-alpha.4 build /home/runner/work/mastra/mastra/voice/elevenlabs
+> @mastra/voice-elevenlabs@0.1.1-alpha.1 build /home/runner/work/mastra/mastra/voice/elevenlabs
 > tsup src/index.ts --format esm --experimental-dts --clean --treeshake
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.3.6
 TSC Build start
-TSC ⚡️ Build success in 7370ms
+TSC ⚡️ Build success in 8591ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.7.3
 Writing package typings: /home/runner/work/mastra/mastra/voice/elevenlabs/dist/_tsup-dts-rollup.d.ts
-DTS ⚡️ Build success in 5039ms
+DTS ⚡️ Build success in 5735ms
 CLI Cleaning output folder
 ESM Build start
-ESM dist/index.js 3.36 KB
-ESM ⚡️ Build success in 287ms
+ESM dist/index.js 5.00 KB
+ESM ⚡️ Build success in 286ms

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @mastra/voice-elevenlabs
 
+## 0.1.1-alpha.1
+
+### Patch Changes
+
+- 705d69b: Add STT for ElevenlabsVoice
+- Updated dependencies [0d185b1]
+- Updated dependencies [ed55f1d]
+- Updated dependencies [8d13b14]
+- Updated dependencies [3ee4831]
+- Updated dependencies [108793c]
+- Updated dependencies [5f28f44]
+  - @mastra/core@0.4.3-alpha.1
+
+## 0.1.1-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [06aa827]
+  - @mastra/core@0.4.3-alpha.0
+
 ## 0.1.0
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.ts

@@ -1,6 +1,8 @@
 import { MastraVoice } from '@mastra/core/voice';
 
-declare type ElevenLabsModel = 'eleven_multilingual_v2' | 'eleven_flash_v2_5' | 'eleven_flash_v2' | 'eleven_multilingual_sts_v2' | 'eleven_english_sts_v2';
+declare type ElevenLabsListenOptions = SpeechToTextOptions & RequestOptions;
+
+declare type ElevenLabsModel = 'eleven_multilingual_v2' | 'eleven_flash_v2_5' | 'eleven_flash_v2' | 'eleven_multilingual_sts_v2' | 'eleven_english_sts_v2' | 'scribe_v1';
 
 export declare class ElevenLabsVoice extends MastraVoice {
     private client;
@@ -13,8 +15,9 @@ export declare class ElevenLabsVoice extends MastraVoice {
      *
      * @throws {Error} If the ELEVENLABS_API_KEY is not set in the environment variables.
      */
-    constructor({ speechModel, speaker }?: {
+    constructor({ speechModel, listeningModel, speaker, }?: {
         speechModel?: ElevenLabsVoiceConfig;
+        listeningModel?: ElevenLabsVoiceConfig;
         speaker?: string;
     });
     /**
@@ -45,7 +48,23 @@ export declare class ElevenLabsVoice extends MastraVoice {
     speak(input: string | NodeJS.ReadableStream, options?: {
         speaker?: string;
     }): Promise<NodeJS.ReadableStream>;
-    listen(_input: NodeJS.ReadableStream | Buffer, _options?: Record<string, unknown>): Promise<string>;
+    /**
+     * Converts audio input to text using ElevenLabs Speech-to-Text API.
+     *
+     * @param input - A readable stream containing the audio data to transcribe
+     * @param options - Configuration options for the transcription
+     * @param options.language_code - ISO language code (e.g., 'en', 'fr', 'es')
+     * @param options.tag_audio_events - Whether to tag audio events like [MUSIC], [LAUGHTER], etc.
+     * @param options.num_speakers - Number of speakers to detect in the audio
+     * @param options.filetype - Audio file format (e.g., 'mp3', 'wav', 'ogg')
+     * @param options.timeoutInSeconds - Request timeout in seconds
+     * @param options.maxRetries - Maximum number of retry attempts
+     * @param options.abortSignal - Signal to abort the request
+     *
+     * @returns A Promise that resolves to the transcribed text
+     *
+     */
+    listen(input: NodeJS.ReadableStream, options?: ElevenLabsListenOptions): Promise<string>;
 }
 
 declare interface ElevenLabsVoiceConfig {
@@ -53,4 +72,19 @@ declare interface ElevenLabsVoiceConfig {
     apiKey?: string;
 }
 
+declare interface RequestOptions {
+    timeoutInSeconds?: number;
+    maxRetries?: number;
+    abortSignal?: AbortSignal;
+    apiKey?: string | undefined;
+    headers?: Record<string, string>;
+}
+
+declare interface SpeechToTextOptions {
+    language_code?: string;
+    tag_audio_events?: boolean;
+    num_speakers?: number;
+    filetype?: string;
+}
+
 export { }

package/dist/index.js

@@ -1,3 +1,4 @@
+import { File } from 'node:buffer';
 import { MastraVoice } from '@mastra/core/voice';
 import { ElevenLabsClient } from 'elevenlabs';
 
@@ -13,13 +14,21 @@ var ElevenLabsVoice = class extends MastraVoice {
    *
    * @throws {Error} If the ELEVENLABS_API_KEY is not set in the environment variables.
    */
-  constructor({ speechModel, speaker } = {}) {
+  constructor({
+    speechModel,
+    listeningModel,
+    speaker
+  } = {}) {
     const apiKey = speechModel?.apiKey ?? process.env.ELEVENLABS_API_KEY;
     super({
       speechModel: {
         name: speechModel?.name ?? "eleven_multilingual_v2",
         apiKey: speechModel?.apiKey
       },
+      listeningModel: {
+        name: listeningModel?.name ?? "scribe_v1",
+        apiKey: listeningModel?.apiKey
+      },
       speaker
     });
     if (!apiKey) {
@@ -86,8 +95,44 @@ var ElevenLabsVoice = class extends MastraVoice {
     }, "voice.elevenlabs.speak")();
     return res;
   }
-  async listen(_input, _options) {
-    throw new Error("ElevenLabs does not support transcription");
+  /**
+   * Converts audio input to text using ElevenLabs Speech-to-Text API.
+   *
+   * @param input - A readable stream containing the audio data to transcribe
+   * @param options - Configuration options for the transcription
+   * @param options.language_code - ISO language code (e.g., 'en', 'fr', 'es')
+   * @param options.tag_audio_events - Whether to tag audio events like [MUSIC], [LAUGHTER], etc.
+   * @param options.num_speakers - Number of speakers to detect in the audio
+   * @param options.filetype - Audio file format (e.g., 'mp3', 'wav', 'ogg')
+   * @param options.timeoutInSeconds - Request timeout in seconds
+   * @param options.maxRetries - Maximum number of retry attempts
+   * @param options.abortSignal - Signal to abort the request
+   *
+   * @returns A Promise that resolves to the transcribed text
+   *
+   */
+  async listen(input, options) {
+    const res = await this.traced(async () => {
+      const chunks = [];
+      for await (const chunk of input) {
+        chunks.push(Buffer.from(chunk));
+      }
+      const buffer = Buffer.concat(chunks);
+      const { language_code, tag_audio_events, num_speakers, filetype, ...requestOptions } = options || {};
+      const file = new File([buffer], `audio.${filetype || "mp3"}`);
+      const transcription = await this.client.speechToText.convert(
+        {
+          file,
+          model_id: this.listeningModel?.name,
+          language_code,
+          tag_audio_events,
+          num_speakers
+        },
+        requestOptions
+      );
+      return transcription.text;
+    }, "voice.elevenlabs.listen")();
+    return res;
   }
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/voice-elevenlabs",
-  "version": "0.1.0",
+  "version": "0.1.1-alpha.1",
   "description": "Mastra ElevenLabs voice integration",
   "type": "module",
   "main": "dist/index.js",
@@ -17,7 +17,7 @@
   "dependencies": {
     "elevenlabs": "^1.50.2",
     "zod": "^3.24.1",
-    "@mastra/core": "^0.4.2"
+    "@mastra/core": "^0.4.3-alpha.1"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.49.2",

package/src/index.test.ts

@@ -1,6 +1,5 @@
-import { createWriteStream, writeFileSync, mkdirSync } from 'fs';
+import { createWriteStream, writeFileSync, mkdirSync, createReadStream } from 'fs';
 import path from 'path';
-import { Readable } from 'stream';
 import { describe, expect, it, beforeAll } from 'vitest';
 
 import { ElevenLabsVoice } from './index.js';
@@ -10,7 +9,6 @@ describe('ElevenLabsVoice Integration Tests', () => {
   const outputDir = path.join(process.cwd(), 'test-outputs');
 
   beforeAll(() => {
-    // Create output directory if it doesn't exist
     try {
       mkdirSync(outputDir, { recursive: true });
     } catch (err) {
@@ -100,14 +98,44 @@ describe('ElevenLabsVoice Integration Tests', () => {
   });
 
   describe('listen', () => {
-    it('should throw error as transcription is not supported', async () => {
-      const dummyStream = new Readable({
-        read() {
-          this.push(null);
+    it('should convert audio to text', async () => {
+      const outputPath = path.join(outputDir, 'elevenlabs-speech-test-params.mp3');
+      const audio = createReadStream(outputPath);
+      const result = await voice.listen(audio);
+
+      if (typeof result !== 'string') {
+        return expect(result).toBeInstanceOf(String);
+      }
+
+      expect(typeof result).toBe('string');
+      expect(result.length).toBeGreaterThan(0);
+    });
+
+    it('should handle API errors gracefully', async () => {
+      // Create a voice instance with an invalid API key to force an error
+      const invalidVoice = new ElevenLabsVoice({
+        listeningModel: {
+          name: 'eleven_multilingual_v2',
+          apiKey: 'invalid-api-key',
         },
       });
 
-      await expect(voice.listen(dummyStream)).rejects.toThrow('ElevenLabs does not support transcription');
+      const outputPath = path.join(outputDir, 'elevenlabs-speech-test-params.mp3');
+      const audio = createReadStream(outputPath);
+
+      // The API call should fail with an authentication error
+      await expect(invalidVoice.listen(audio)).rejects.toThrow();
+    });
+
+    it('should handle invalid audio input', async () => {
+      // Create a path to a non-existent file
+      const nonExistentPath = path.join(outputDir, 'non-existent-file.mp3');
+
+      // Attempting to create a read stream from a non-existent file should throw
+      await expect(async () => {
+        const audio = createReadStream(nonExistentPath);
+        await voice.listen(audio);
+      }).rejects.toThrow();
     });
   });
 });

package/src/index.ts

@@ -1,3 +1,4 @@
+import { File } from 'node:buffer';
 import { MastraVoice } from '@mastra/core/voice';
 import { ElevenLabsClient } from 'elevenlabs';
 
@@ -6,13 +7,32 @@ type ElevenLabsModel =
   | 'eleven_flash_v2_5'
   | 'eleven_flash_v2'
   | 'eleven_multilingual_sts_v2'
-  | 'eleven_english_sts_v2';
+  | 'eleven_english_sts_v2'
+  | 'scribe_v1';
 
 interface ElevenLabsVoiceConfig {
   name?: ElevenLabsModel;
   apiKey?: string;
 }
 
+interface SpeechToTextOptions {
+  language_code?: string;
+  tag_audio_events?: boolean;
+  num_speakers?: number;
+  filetype?: string;
+}
+
+interface RequestOptions {
+  timeoutInSeconds?: number;
+  maxRetries?: number;
+  abortSignal?: AbortSignal;
+  apiKey?: string | undefined;
+  headers?: Record<string, string>;
+}
+
+// Combined options type
+type ElevenLabsListenOptions = SpeechToTextOptions & RequestOptions;
+
 export class ElevenLabsVoice extends MastraVoice {
   private client: ElevenLabsClient;
 
@@ -25,13 +45,21 @@ export class ElevenLabsVoice extends MastraVoice {
    *
    * @throws {Error} If the ELEVENLABS_API_KEY is not set in the environment variables.
    */
-  constructor({ speechModel, speaker }: { speechModel?: ElevenLabsVoiceConfig; speaker?: string } = {}) {
+  constructor({
+    speechModel,
+    listeningModel,
+    speaker,
+  }: { speechModel?: ElevenLabsVoiceConfig; listeningModel?: ElevenLabsVoiceConfig; speaker?: string } = {}) {
     const apiKey = speechModel?.apiKey ?? process.env.ELEVENLABS_API_KEY;
     super({
       speechModel: {
         name: speechModel?.name ?? 'eleven_multilingual_v2',
         apiKey: speechModel?.apiKey,
       },
+      listeningModel: {
+        name: listeningModel?.name ?? 'scribe_v1',
+        apiKey: listeningModel?.apiKey,
+      },
       speaker,
     });
 
@@ -110,7 +138,48 @@ export class ElevenLabsVoice extends MastraVoice {
     return res;
   }
 
-  async listen(_input: NodeJS.ReadableStream | Buffer, _options?: Record<string, unknown>): Promise<string> {
-    throw new Error('ElevenLabs does not support transcription');
+  /**
+   * Converts audio input to text using ElevenLabs Speech-to-Text API.
+   *
+   * @param input - A readable stream containing the audio data to transcribe
+   * @param options - Configuration options for the transcription
+   * @param options.language_code - ISO language code (e.g., 'en', 'fr', 'es')
+   * @param options.tag_audio_events - Whether to tag audio events like [MUSIC], [LAUGHTER], etc.
+   * @param options.num_speakers - Number of speakers to detect in the audio
+   * @param options.filetype - Audio file format (e.g., 'mp3', 'wav', 'ogg')
+   * @param options.timeoutInSeconds - Request timeout in seconds
+   * @param options.maxRetries - Maximum number of retry attempts
+   * @param options.abortSignal - Signal to abort the request
+   *
+   * @returns A Promise that resolves to the transcribed text
+   *
+   */
+  async listen(input: NodeJS.ReadableStream, options?: ElevenLabsListenOptions): Promise<string> {
+    const res = await this.traced(async () => {
+      const chunks: Buffer[] = [];
+      for await (const chunk of input) {
+        chunks.push(Buffer.from(chunk));
+      }
+      const buffer = Buffer.concat(chunks);
+
+      const { language_code, tag_audio_events, num_speakers, filetype, ...requestOptions } = options || {};
+
+      const file = new File([buffer], `audio.${filetype || 'mp3'}`);
+
+      const transcription = await this.client.speechToText.convert(
+        {
+          file: file,
+          model_id: this.listeningModel?.name as ElevenLabsModel,
+          language_code,
+          tag_audio_events,
+          num_speakers,
+        },
+        requestOptions,
+      );
+
+      return transcription.text;
+    }, 'voice.elevenlabs.listen')();
+
+    return res;
   }
 }