@elevenlabs/elevenlabs-js 2.19.0 → 2.20.0

package/dist/wrapper/ElevenLabsClient.js

@@ -38,6 +38,7 @@ const Client_1 = require("../Client");
 const errors = __importStar(require("../errors"));
 const webhooks_1 = require("./webhooks");
 const music_1 = require("./music");
+const speechToText_1 = require("./speechToText");
 class ElevenLabsClient extends Client_1.ElevenLabsClient {
     constructor(options = {}) {
         var _a;
@@ -63,5 +64,11 @@ class ElevenLabsClient extends Client_1.ElevenLabsClient {
         }
         return this._customMusic;
     }
+    get speechToText() {
+        if (!this._customSpeechToText) {
+            this._customSpeechToText = new speechToText_1.SpeechToText(this._options);
+        }
+        return this._customSpeechToText;
+    }
 }
 exports.ElevenLabsClient = ElevenLabsClient;

package/dist/wrapper/index.d.ts

@@ -2,3 +2,5 @@ export { ElevenLabsClient } from "./ElevenLabsClient";
 export { Music, type SongMetadata, type MultipartResponse } from "./music";
 export { play } from "./play";
 export { stream } from "./stream";
+export { SpeechToText } from "./speechToText";
+export { RealtimeConnection, RealtimeEvents, AudioFormat, CommitStrategy, type AudioOptions, type UrlOptions } from "./realtime";

package/dist/wrapper/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stream = exports.play = exports.Music = exports.ElevenLabsClient = void 0;
+exports.CommitStrategy = exports.AudioFormat = exports.RealtimeEvents = exports.RealtimeConnection = exports.SpeechToText = exports.stream = exports.play = exports.Music = exports.ElevenLabsClient = void 0;
 var ElevenLabsClient_1 = require("./ElevenLabsClient");
 Object.defineProperty(exports, "ElevenLabsClient", { enumerable: true, get: function () { return ElevenLabsClient_1.ElevenLabsClient; } });
 var music_1 = require("./music");
@@ -9,3 +9,10 @@ var play_1 = require("./play");
 Object.defineProperty(exports, "play", { enumerable: true, get: function () { return play_1.play; } });
 var stream_1 = require("./stream");
 Object.defineProperty(exports, "stream", { enumerable: true, get: function () { return stream_1.stream; } });
+var speechToText_1 = require("./speechToText");
+Object.defineProperty(exports, "SpeechToText", { enumerable: true, get: function () { return speechToText_1.SpeechToText; } });
+var realtime_1 = require("./realtime");
+Object.defineProperty(exports, "RealtimeConnection", { enumerable: true, get: function () { return realtime_1.RealtimeConnection; } });
+Object.defineProperty(exports, "RealtimeEvents", { enumerable: true, get: function () { return realtime_1.RealtimeEvents; } });
+Object.defineProperty(exports, "AudioFormat", { enumerable: true, get: function () { return realtime_1.AudioFormat; } });
+Object.defineProperty(exports, "CommitStrategy", { enumerable: true, get: function () { return realtime_1.CommitStrategy; } });

package/dist/wrapper/realtime/connection.d.ts

@@ -0,0 +1,179 @@
+import WebSocket from "ws";
+import type { ChildProcess } from "node:child_process";
+/**
+ * Events emitted by the RealtimeConnection.
+ */
+export declare enum RealtimeEvents {
+    /** Emitted when the session is successfully started */
+    SESSION_STARTED = "session_started",
+    /** Emitted when a partial (interim) transcript is available */
+    PARTIAL_TRANSCRIPT = "partial_transcript",
+    /** Emitted when a final transcript is available */
+    FINAL_TRANSCRIPT = "final_transcript",
+    /** Emitted when a final transcript with timestamps is available */
+    FINAL_TRANSCRIPT_WITH_TIMESTAMPS = "final_transcript_with_timestamps",
+    /** Emitted when an error occurs */
+    ERROR = "error",
+    /** Emitted when the WebSocket connection is opened */
+    OPEN = "open",
+    /** Emitted when the WebSocket connection is closed */
+    CLOSE = "close"
+}
+/**
+ * Manages a real-time transcription WebSocket connection.
+ *
+ * @remarks
+ * **Node.js only**: This class uses Node.js-specific WebSocket implementation.
+ *
+ * @example
+ * ```typescript
+ * const connection = await client.speechToText.realtime.connect({
+ *     modelId: "scribe_realtime_v2",
+ *     audioFormat: AudioFormat.PCM_16000,
+ *     sampleRate: 16000,
+ * });
+ *
+ * connection.on(RealtimeEvents.SESSION_STARTED, (data) => {
+ *     console.log("Session started");
+ * });
+ *
+ * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
+ *     console.log("Partial:", data.transcript);
+ * });
+ *
+ * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+ *     console.log("Final:", data.transcript);
+ *     connection.close();
+ * });
+ *
+ * // Send audio data
+ * connection.send({ audioBase64: base64String });
+ *
+ * // Commit and close
+ * connection.commit();
+  * ```
+ */
+export declare class RealtimeConnection {
+    private websocket;
+    private eventEmitter;
+    private ffmpegProcess;
+    private currentSampleRate;
+    constructor(sampleRate: number);
+    /**
+     * @internal
+     * Used internally by ScribeRealtime to attach the WebSocket after connection is created.
+     */
+    setWebSocket(websocket: WebSocket): void;
+    /**
+     * @internal
+     * Used internally by ScribeRealtime to attach ffmpeg process for cleanup.
+     */
+    setFfmpegProcess(ffmpegProcess: ChildProcess): void;
+    /**
+     * Attaches an event listener for the specified event.
+     *
+     * @param event - The event to listen for (use RealtimeEvents enum)
+     * @param listener - The callback function to execute when the event fires
+     *
+     * @example
+     * ```typescript
+     * connection.on(RealtimeEvents.SESSION_STARTED, (data) => {
+     *     console.log("Session started", data);
+     * });
+     *
+     * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
+     *     console.log("Partial:", data.transcript);
+     * });
+     *
+     * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+     *     console.log("Final:", data.transcript);
+     * });
+     * ```
+     */
+    on(event: RealtimeEvents, listener: (...args: unknown[]) => void): void;
+    /**
+     * Removes an event listener for the specified event.
+     *
+     * @param event - The event to stop listening for
+     * @param listener - The callback function to remove
+     *
+     * @example
+     * ```typescript
+     * const handler = (data) => console.log(data);
+     * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, handler);
+     *
+     * // Later, remove the listener
+     * connection.off(RealtimeEvents.PARTIAL_TRANSCRIPT, handler);
+     * ```
+     */
+    off(event: RealtimeEvents, listener: (...args: unknown[]) => void): void;
+    /**
+     * Sends audio data to the transcription service.
+     *
+     * @param data - Audio data configuration
+     * @param data.audioBase64 - Base64-encoded audio data
+     * @param data.commit - Whether to commit the transcription after this chunk. You likely want to use connection.commit() instead (default: false)
+     * @param data.sampleRate - Sample rate of the audio (default: configured sample rate)
+     *
+     * @throws {Error} If the WebSocket connection is not open
+     *
+     * @example
+     * ```typescript
+     * // Send audio chunk without committing
+     * connection.send({
+     *     audioBase64: base64EncodedAudio,
+     * });
+     *
+     * // Send audio chunk with custom sample rate
+     * connection.send({
+     *     audioBase64: base64EncodedAudio,
+     *     sampleRate: 16000,
+     * });
+     * ```
+     */
+    send(data: {
+        audioBase64: string;
+        commit?: boolean;
+        sampleRate?: number;
+    }): void;
+    /**
+     * Commits the transcription, signaling that all audio has been sent.
+     * This finalizes the transcription and triggers a FINAL_TRANSCRIPT event.
+     *
+     * @throws {Error} If the WebSocket connection is not open
+     *
+     * @remarks
+     * Only needed when using CommitStrategy.MANUAL.
+     * When using CommitStrategy.VAD, commits are handled automatically by the server.
+     *
+     * @example
+     * ```typescript
+     * // Send all audio chunks
+     * for (const chunk of audioChunks) {
+     *     connection.send({ audioBase64: chunk });
+     * }
+     *
+     * // Finalize the transcription
+     * connection.commit();
+     * ```
+     */
+    commit(): void;
+    /**
+     * Closes the WebSocket connection and cleans up resources.
+     * This will terminate any ongoing transcription and stop ffmpeg processes if running.
+     *
+     * @remarks
+     * After calling close(), this connection cannot be reused.
+     * Create a new connection if you need to start transcribing again.
+     *
+     * @example
+     * ```typescript
+     * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+     *     console.log("Final:", data.transcript);
+     *     connection.close();
+     * });
+     * ```
+     */
+    close(): void;
+    private cleanup;
+}

package/dist/wrapper/realtime/connection.js

@@ -0,0 +1,261 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeConnection = exports.RealtimeEvents = void 0;
+const ws_1 = __importDefault(require("ws"));
+const node_events_1 = require("node:events");
+/**
+ * Events emitted by the RealtimeConnection.
+ */
+var RealtimeEvents;
+(function (RealtimeEvents) {
+    /** Emitted when the session is successfully started */
+    RealtimeEvents["SESSION_STARTED"] = "session_started";
+    /** Emitted when a partial (interim) transcript is available */
+    RealtimeEvents["PARTIAL_TRANSCRIPT"] = "partial_transcript";
+    /** Emitted when a final transcript is available */
+    RealtimeEvents["FINAL_TRANSCRIPT"] = "final_transcript";
+    /** Emitted when a final transcript with timestamps is available */
+    RealtimeEvents["FINAL_TRANSCRIPT_WITH_TIMESTAMPS"] = "final_transcript_with_timestamps";
+    /** Emitted when an error occurs */
+    RealtimeEvents["ERROR"] = "error";
+    /** Emitted when the WebSocket connection is opened */
+    RealtimeEvents["OPEN"] = "open";
+    /** Emitted when the WebSocket connection is closed */
+    RealtimeEvents["CLOSE"] = "close";
+})(RealtimeEvents || (exports.RealtimeEvents = RealtimeEvents = {}));
+/**
+ * Manages a real-time transcription WebSocket connection.
+ *
+ * @remarks
+ * **Node.js only**: This class uses Node.js-specific WebSocket implementation.
+ *
+ * @example
+ * ```typescript
+ * const connection = await client.speechToText.realtime.connect({
+ *     modelId: "scribe_realtime_v2",
+ *     audioFormat: AudioFormat.PCM_16000,
+ *     sampleRate: 16000,
+ * });
+ *
+ * connection.on(RealtimeEvents.SESSION_STARTED, (data) => {
+ *     console.log("Session started");
+ * });
+ *
+ * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
+ *     console.log("Partial:", data.transcript);
+ * });
+ *
+ * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+ *     console.log("Final:", data.transcript);
+ *     connection.close();
+ * });
+ *
+ * // Send audio data
+ * connection.send({ audioBase64: base64String });
+ *
+ * // Commit and close
+ * connection.commit();
+  * ```
+ */
+class RealtimeConnection {
+    constructor(sampleRate) {
+        this.websocket = null;
+        this.eventEmitter = new node_events_1.EventEmitter();
+        this.ffmpegProcess = null;
+        this.currentSampleRate = 16000;
+        this.currentSampleRate = sampleRate;
+    }
+    /**
+     * @internal
+     * Used internally by ScribeRealtime to attach the WebSocket after connection is created.
+     */
+    setWebSocket(websocket) {
+        this.websocket = websocket;
+        // If WebSocket is already open, emit OPEN event immediately
+        if (this.websocket.readyState === ws_1.default.OPEN) {
+            this.eventEmitter.emit(RealtimeEvents.OPEN);
+        }
+        else {
+            // Otherwise, wait for the open event
+            this.websocket.on("open", () => {
+                this.eventEmitter.emit(RealtimeEvents.OPEN);
+            });
+        }
+        this.websocket.on("message", (event) => {
+            const data = JSON.parse(event.toString());
+            switch (data.message_type) {
+                case "session_started":
+                    this.eventEmitter.emit(RealtimeEvents.SESSION_STARTED, data);
+                    break;
+                case "partial_transcript":
+                    this.eventEmitter.emit(RealtimeEvents.PARTIAL_TRANSCRIPT, data);
+                    break;
+                case "final_transcript":
+                    this.eventEmitter.emit(RealtimeEvents.FINAL_TRANSCRIPT, data);
+                    break;
+                case "final_transcript_with_timestamps":
+                    this.eventEmitter.emit(RealtimeEvents.FINAL_TRANSCRIPT_WITH_TIMESTAMPS, data);
+                    this.close();
+                    break;
+            }
+        });
+        this.websocket.on("error", (error) => {
+            this.eventEmitter.emit(RealtimeEvents.ERROR, error);
+        });
+        this.websocket.on("close", () => {
+            this.eventEmitter.emit(RealtimeEvents.CLOSE);
+            this.cleanup();
+        });
+    }
+    /**
+     * @internal
+     * Used internally by ScribeRealtime to attach ffmpeg process for cleanup.
+     */
+    setFfmpegProcess(ffmpegProcess) {
+        this.ffmpegProcess = ffmpegProcess;
+    }
+    /**
+     * Attaches an event listener for the specified event.
+     *
+     * @param event - The event to listen for (use RealtimeEvents enum)
+     * @param listener - The callback function to execute when the event fires
+     *
+     * @example
+     * ```typescript
+     * connection.on(RealtimeEvents.SESSION_STARTED, (data) => {
+     *     console.log("Session started", data);
+     * });
+     *
+     * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
+     *     console.log("Partial:", data.transcript);
+     * });
+     *
+     * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+     *     console.log("Final:", data.transcript);
+     * });
+     * ```
+     */
+    on(event, listener) {
+        this.eventEmitter.on(event, listener);
+    }
+    /**
+     * Removes an event listener for the specified event.
+     *
+     * @param event - The event to stop listening for
+     * @param listener - The callback function to remove
+     *
+     * @example
+     * ```typescript
+     * const handler = (data) => console.log(data);
+     * connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, handler);
+     *
+     * // Later, remove the listener
+     * connection.off(RealtimeEvents.PARTIAL_TRANSCRIPT, handler);
+     * ```
+     */
+    off(event, listener) {
+        this.eventEmitter.off(event, listener);
+    }
+    /**
+     * Sends audio data to the transcription service.
+     *
+     * @param data - Audio data configuration
+     * @param data.audioBase64 - Base64-encoded audio data
+     * @param data.commit - Whether to commit the transcription after this chunk. You likely want to use connection.commit() instead (default: false)
+     * @param data.sampleRate - Sample rate of the audio (default: configured sample rate)
+     *
+     * @throws {Error} If the WebSocket connection is not open
+     *
+     * @example
+     * ```typescript
+     * // Send audio chunk without committing
+     * connection.send({
+     *     audioBase64: base64EncodedAudio,
+     * });
+     *
+     * // Send audio chunk with custom sample rate
+     * connection.send({
+     *     audioBase64: base64EncodedAudio,
+     *     sampleRate: 16000,
+     * });
+     * ```
+     */
+    send(data) {
+        var _a, _b;
+        if (!this.websocket || this.websocket.readyState !== ws_1.default.OPEN) {
+            throw new Error("WebSocket is not connected");
+        }
+        const message = {
+            message_type: "input_audio_chunk",
+            audio_base_64: data.audioBase64,
+            commit: (_a = data.commit) !== null && _a !== void 0 ? _a : false,
+            sample_rate: (_b = data.sampleRate) !== null && _b !== void 0 ? _b : this.currentSampleRate,
+        };
+        this.websocket.send(JSON.stringify(message));
+    }
+    /**
+     * Commits the transcription, signaling that all audio has been sent.
+     * This finalizes the transcription and triggers a FINAL_TRANSCRIPT event.
+     *
+     * @throws {Error} If the WebSocket connection is not open
+     *
+     * @remarks
+     * Only needed when using CommitStrategy.MANUAL.
+     * When using CommitStrategy.VAD, commits are handled automatically by the server.
+     *
+     * @example
+     * ```typescript
+     * // Send all audio chunks
+     * for (const chunk of audioChunks) {
+     *     connection.send({ audioBase64: chunk });
+     * }
+     *
+     * // Finalize the transcription
+     * connection.commit();
+     * ```
+     */
+    commit() {
+        if (!this.websocket || this.websocket.readyState !== ws_1.default.OPEN) {
+            throw new Error("WebSocket is not connected");
+        }
+        const message = {
+            message_type: "input_audio_chunk",
+            audio_base_64: "",
+            commit: true,
+            sample_rate: this.currentSampleRate,
+        };
+        this.websocket.send(JSON.stringify(message));
+    }
+    /**
+     * Closes the WebSocket connection and cleans up resources.
+     * This will terminate any ongoing transcription and stop ffmpeg processes if running.
+     *
+     * @remarks
+     * After calling close(), this connection cannot be reused.
+     * Create a new connection if you need to start transcribing again.
+     *
+     * @example
+     * ```typescript
+     * connection.on(RealtimeEvents.FINAL_TRANSCRIPT, (data) => {
+     *     console.log("Final:", data.transcript);
+     *     connection.close();
+     * });
+     * ```
+     */
+    close() {
+        this.cleanup();
+        if (this.websocket) {
+            this.websocket.close();
+        }
+    }
+    cleanup() {
+        if (this.ffmpegProcess) {
+            this.ffmpegProcess.kill();
+            this.ffmpegProcess = null;
+        }
+    }
+}
+exports.RealtimeConnection = RealtimeConnection;

package/dist/wrapper/realtime/index.d.ts

@@ -0,0 +1,2 @@
+export { RealtimeConnection, RealtimeEvents } from "./connection";
+export { ScribeRealtime, AudioFormat, CommitStrategy, type AudioOptions, type UrlOptions } from "./scribe";

package/dist/wrapper/realtime/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CommitStrategy = exports.AudioFormat = exports.ScribeRealtime = exports.RealtimeEvents = exports.RealtimeConnection = void 0;
+var connection_1 = require("./connection");
+Object.defineProperty(exports, "RealtimeConnection", { enumerable: true, get: function () { return connection_1.RealtimeConnection; } });
+Object.defineProperty(exports, "RealtimeEvents", { enumerable: true, get: function () { return connection_1.RealtimeEvents; } });
+var scribe_1 = require("./scribe");
+Object.defineProperty(exports, "ScribeRealtime", { enumerable: true, get: function () { return scribe_1.ScribeRealtime; } });
+Object.defineProperty(exports, "AudioFormat", { enumerable: true, get: function () { return scribe_1.AudioFormat; } });
+Object.defineProperty(exports, "CommitStrategy", { enumerable: true, get: function () { return scribe_1.CommitStrategy; } });

package/dist/wrapper/realtime/scribe.d.ts

@@ -0,0 +1,107 @@
+import type { SpeechToText } from "../../api/resources/speechToText/client/Client";
+import { RealtimeConnection } from "./connection";
+export declare enum AudioFormat {
+    PCM_8000 = "pcm_8000",
+    PCM_16000 = "pcm_16000",
+    PCM_22050 = "pcm_22050",
+    PCM_24000 = "pcm_24000",
+    PCM_44100 = "pcm_44100",
+    PCM_48000 = "pcm_48000",
+    ULAW_8000 = "ulaw_8000"
+}
+export declare enum CommitStrategy {
+    MANUAL = "manual",
+    VAD = "vad"
+}
+interface BaseOptions {
+    /**
+     * Strategy for committing transcriptions.
+     * @default CommitStrategy.MANUAL
+     */
+    commitStrategy?: CommitStrategy;
+    /**
+     * Silence threshold in seconds for VAD (Voice Activity Detection).
+     * Must be a positive number between 0.3 and 3.0
+     */
+    vadSilenceThresholdSecs?: number;
+    /**
+     * Threshold for voice activity detection.
+     * Must be between 0.1 and 0.9.
+     */
+    vadThreshold?: number;
+    /**
+     * Minimum speech duration in milliseconds.
+     * Must be a positive integer between 50 and 2000.
+     */
+    minSpeechDurationMs?: number;
+    /**
+     * Minimum silence duration in milliseconds.
+     * Must be a positive integer between 50 and 2000.
+     */
+    minSilenceDurationMs?: number;
+    /**
+     * Model ID to use for transcription.
+     * Must be a valid model ID.
+     */
+    modelId: string;
+}
+export interface AudioOptions extends BaseOptions {
+    audioFormat: AudioFormat;
+    sampleRate: number;
+    url?: never;
+}
+/**
+ * Options for streaming audio from a URL.
+ * @remarks
+ * **Node.js only**: Requires ffmpeg to be installed and available in PATH.
+ * This will not work in browsers, Deno, or Cloudflare Workers.
+ */
+export interface UrlOptions extends BaseOptions {
+    url: string;
+    audioFormat?: never;
+    sampleRate?: never;
+}
+/**
+ * Real-time speech-to-text transcription client.
+ * @remarks
+ * **Node.js only**: This class uses Node.js-specific APIs (WebSocket from 'ws', child_process).
+ * It will not work in browsers, Deno, or Cloudflare Workers without modifications.
+ */
+export declare class ScribeRealtime {
+    private options;
+    constructor(options?: SpeechToText.Options);
+    private getWebSocketUri;
+    private checkFfmpegInstalled;
+    private buildWebSocketUri;
+    /**
+     * Establishes a WebSocket connection for real-time speech-to-text transcription.
+     *
+     * @param options - Configuration options for the connection
+     * @returns A promise that resolves to a RealtimeConnection instance
+     *
+     * @remarks
+     * **Node.js only**: This method uses Node.js-specific APIs.
+     *
+     * When using `UrlOptions` with a URL, ffmpeg must be installed and available in PATH.
+     * The SDK will automatically convert the stream to 16kHz mono PCM format.
+     *
+     * @example
+     * ```typescript
+     * // Manual audio streaming
+     * const connection = await client.speechToText.realtime.connect({
+     *     modelId: "scribe_realtime_v2",
+     *     audioFormat: AudioFormat.PCM_16000,
+     *     sampleRate: 16000,
+     * });
+     *
+     * // Automatic URL streaming (requires ffmpeg)
+     * const connection = await client.speechToText.realtime.connect({
+     *     modelId: "scribe_realtime_v2",
+     *     url: "https://example.com/stream.mp3",
+     * });
+     * ```
+     */
+    connect(options: AudioOptions | UrlOptions): Promise<RealtimeConnection>;
+    private streamFromUrl;
+}
+export {};