@elevenlabs/elevenlabs-js 2.5.0 → 2.6.0

package/api/resources/conversationalAi/conversation/Conversation.js

@@ -0,0 +1,280 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Conversation = void 0;
+const events_1 = require("events");
+const ws_1 = __importDefault(require("ws"));
+const Client_1 = require("../../../../Client");
+const ClientTools_1 = require("./ClientTools");
+const events_2 = require("./events");
+const WebSocketInterface_1 = require("./interfaces/WebSocketInterface");
+/**
+ * Conversational AI session for Node.js.
+ *
+ * BETA: This API is subject to change without regard to backwards compatibility.
+ */
+class Conversation extends events_1.EventEmitter {
+    constructor(options) {
+        super();
+        this.shouldStop = false;
+        this.lastInterruptId = 0;
+        this.client = options.client || new Client_1.ElevenLabsClient();
+        this.agentId = options.agentId;
+        this.requiresAuth = options.requiresAuth;
+        this.audioInterface = options.audioInterface;
+        this.clientTools = options.clientTools || new ClientTools_1.ClientTools();
+        this.config = options.config || { extraBody: {}, conversationConfigOverride: {}, dynamicVariables: {} };
+        this.conversationClient = options.conversationClient || this.client;
+        this.webSocketFactory = options.webSocketFactory || new WebSocketInterface_1.DefaultWebSocketFactory();
+        this.callbackAgentResponse = options.callbackAgentResponse;
+        this.callbackAgentResponseCorrection = options.callbackAgentResponseCorrection;
+        this.callbackUserTranscript = options.callbackUserTranscript;
+        this.callbackLatencyMeasurement = options.callbackLatencyMeasurement;
+    }
+    /**
+     * Starts the conversation session.
+     *
+     * Will run until `endSession` is called.
+     */
+    startSession() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.ws) {
+                throw new Error("Session already started");
+            }
+            const wsUrl = this.requiresAuth ? yield this._getSignedUrl() : this._getWssUrl();
+            return new Promise((resolve, reject) => {
+                this.ws = this.webSocketFactory.create(wsUrl, {
+                    perMessageDeflate: false,
+                    maxPayload: 16 * 1024 * 1024, // 16MB max message size
+                });
+                this.ws.on("open", () => {
+                    this._onWebSocketOpen();
+                    resolve();
+                });
+                this.ws.on("message", (data) => {
+                    this._onWebSocketMessage(data);
+                });
+                this.ws.on("error", (error) => {
+                    this.emit("error", error);
+                    reject(error);
+                });
+                this.ws.on("close", (code, reason) => {
+                    this._onWebSocketClose(code, reason);
+                });
+            });
+        });
+    }
+    /**
+     * Ends the conversation session and cleans up resources.
+     */
+    endSession() {
+        this.shouldStop = true;
+        if (this.audioInterface) {
+            this.audioInterface.stop();
+        }
+        if (this.ws) {
+            this.ws.close();
+            this.ws = undefined;
+        }
+        this.emit("session_ended", this.conversationId);
+    }
+    /**
+     * Send a text message from the user to the agent.
+     *
+     * @param text The text message to send to the agent
+     */
+    sendUserMessage(text) {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.USER_MESSAGE,
+            text,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Register user activity to prevent session timeout.
+     *
+     * This sends a ping to the orchestrator to reset the timeout timer.
+     */
+    registerUserActivity() {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.USER_ACTIVITY,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Send a contextual update to the conversation.
+     *
+     * Contextual updates are non-interrupting content that is sent to the server
+     * to update the conversation state without directly prompting the agent.
+     *
+     * @param text The contextual information to send to the conversation
+     */
+    sendContextualUpdate(text) {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.CONTEXTUAL_UPDATE,
+            text,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Get the conversation ID if available.
+     *
+     * @returns The conversation ID or undefined if not yet available
+     */
+    getConversationId() {
+        return this.conversationId;
+    }
+    /**
+     * Check if the session is currently active.
+     *
+     * @returns True if the session is active
+     */
+    isSessionActive() {
+        return !!this.ws && this.ws.readyState === ws_1.default.OPEN && !this.shouldStop;
+    }
+    _onWebSocketOpen() {
+        // Send conversation initiation data
+        const initEvent = {
+            type: events_2.ClientToOrchestratorEvent.CONVERSATION_INITIATION_CLIENT_DATA,
+            custom_llm_extra_body: this.config.extraBody,
+            conversation_config_override: this.config.conversationConfigOverride,
+            dynamic_variables: this.config.dynamicVariables,
+        };
+        this.ws.send(JSON.stringify(initEvent));
+        // Set up audio input callback
+        this.inputCallback = (audio) => {
+            if (this.shouldStop || !this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+                return;
+            }
+            try {
+                const audioEvent = {
+                    user_audio_chunk: audio.toString("base64"),
+                };
+                this.ws.send(JSON.stringify(audioEvent));
+            }
+            catch (error) {
+                console.error("Error sending user audio chunk:", error);
+                this.endSession();
+            }
+        };
+        // Start audio interface
+        this.audioInterface.start(this.inputCallback);
+        this.emit("session_started");
+    }
+    _onWebSocketMessage(data) {
+        if (this.shouldStop) {
+            return;
+        }
+        try {
+            const message = JSON.parse(data.toString());
+            this._handleMessage(message);
+        }
+        catch (error) {
+            console.error("Error parsing WebSocket message:", error);
+        }
+    }
+    _onWebSocketClose(code, reason) {
+        this.emit("session_ended", this.conversationId, code, reason.toString());
+    }
+    _handleMessage(message) {
+        const messageType = message.type;
+        switch (messageType) {
+            case "conversation_initiation_metadata":
+                const event = message.conversation_initiation_metadata_event;
+                if (!this.conversationId) {
+                    this.conversationId = event.conversation_id;
+                    this.emit("conversation_started", this.conversationId);
+                }
+                break;
+            case "audio":
+                const audioEvent = message.audio_event;
+                if (parseInt(audioEvent.event_id) <= this.lastInterruptId) {
+                    return;
+                }
+                const audio = Buffer.from(audioEvent.audio_base_64, "base64");
+                this.audioInterface.output(audio);
+                break;
+            case "agent_response":
+                if (this.callbackAgentResponse) {
+                    const responseEvent = message.agent_response_event;
+                    this.callbackAgentResponse(responseEvent.agent_response.trim());
+                }
+                break;
+            case "agent_response_correction":
+                if (this.callbackAgentResponseCorrection) {
+                    const correctionEvent = message.agent_response_correction_event;
+                    this.callbackAgentResponseCorrection(correctionEvent.original_agent_response.trim(), correctionEvent.corrected_agent_response.trim());
+                }
+                break;
+            case "user_transcript":
+                if (this.callbackUserTranscript) {
+                    const transcriptEvent = message.user_transcription_event;
+                    this.callbackUserTranscript(transcriptEvent.user_transcript.trim());
+                }
+                break;
+            case "interruption":
+                const interruptionEvent = message.interruption_event;
+                this.lastInterruptId = parseInt(interruptionEvent.event_id);
+                this.audioInterface.interrupt();
+                break;
+            case "ping":
+                const pingEvent = message.ping_event;
+                const pongEvent = {
+                    type: events_2.ClientToOrchestratorEvent.PONG,
+                    event_id: pingEvent.event_id,
+                };
+                this.ws.send(JSON.stringify(pongEvent));
+                if (this.callbackLatencyMeasurement && pingEvent.ping_ms) {
+                    this.callbackLatencyMeasurement(parseInt(pingEvent.ping_ms));
+                }
+                break;
+            case "client_tool_call":
+                const toolCall = message.client_tool_call;
+                const toolName = toolCall.tool_name;
+                const parameters = Object.assign({ tool_call_id: toolCall.tool_call_id }, toolCall.parameters);
+                this.clientTools.executeToolAsync(toolName, parameters, (response) => {
+                    if (!this.shouldStop && this.ws && this.ws.readyState === ws_1.default.OPEN) {
+                        this.ws.send(JSON.stringify(response));
+                    }
+                });
+                break;
+            default:
+                // Ignore unknown message types
+                break;
+        }
+    }
+    _getWssUrl() {
+        // Default to production environment WebSocket URL
+        const baseWsUrl = "wss://api.elevenlabs.io";
+        return `${baseWsUrl}/v1/convai/conversation?agent_id=${this.agentId}`;
+    }
+    _getSignedUrl() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.conversationClient.conversationalAi.conversations.getSignedUrl({
+                agentId: this.agentId,
+            });
+            return response.signedUrl;
+        });
+    }
+}
+exports.Conversation = Conversation;

package/api/resources/conversationalAi/conversation/ConversationConfig.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Configuration options for the Conversation.
+ */
+export interface ConversationInitiationData {
+    /** Extra body parameters to include in the conversation initiation request */
+    extraBody?: Record<string, any>;
+    /** Configuration overrides for the conversation */
+    conversationConfigOverride?: Record<string, any>;
+    /** Dynamic variables to use in the conversation */
+    dynamicVariables?: Record<string, any>;
+}
+/**
+ * Creates a default conversation initiation data object.
+ *
+ * @param options Optional partial configuration to override defaults
+ * @returns Complete conversation initiation data object
+ */
+export declare function createConversationInitiationData(options?: Partial<ConversationInitiationData>): ConversationInitiationData;

package/api/resources/conversationalAi/conversation/ConversationConfig.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createConversationInitiationData = createConversationInitiationData;
+/**
+ * Creates a default conversation initiation data object.
+ *
+ * @param options Optional partial configuration to override defaults
+ * @returns Complete conversation initiation data object
+ */
+function createConversationInitiationData(options = {}) {
+    return {
+        extraBody: options.extraBody || {},
+        conversationConfigOverride: options.conversationConfigOverride || {},
+        dynamicVariables: options.dynamicVariables || {},
+    };
+}

package/api/resources/conversationalAi/conversation/DefaultAudioInterface.d.ts

@@ -0,0 +1,63 @@
+import { AudioInterface } from "./AudioInterface";
+/**
+ * Default audio interface implementation for Node.js using basic audio processing.
+ *
+ * Note: This is a basic implementation. For production use, consider using
+ * professional audio libraries like 'naudiodon', 'mic'/'speaker', or similar.
+ *
+ * This implementation provides a foundation that can be extended with actual
+ * audio capture and playback capabilities.
+ */
+export declare class DefaultAudioInterface extends AudioInterface {
+    private static readonly INPUT_FRAMES_PER_BUFFER;
+    private static readonly OUTPUT_FRAMES_PER_BUFFER;
+    private static readonly SAMPLE_RATE;
+    private static readonly CHANNELS;
+    private inputCallback?;
+    private outputQueue;
+    private shouldStop;
+    private inputInterval?;
+    private outputInterval?;
+    /**
+     * Starts the audio interface.
+     *
+     * @param inputCallback Function to call with audio chunks from the microphone
+     */
+    start(inputCallback: (audio: Buffer) => void): void;
+    /**
+     * Stops the audio interface and cleans up resources.
+     */
+    stop(): void;
+    /**
+     * Output audio to the user.
+     *
+     * @param audio Audio data to output to the speaker
+     */
+    output(audio: Buffer): void;
+    /**
+     * Interruption signal to stop any audio output.
+     */
+    interrupt(): void;
+    /**
+     * Starts audio input processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'mic', 'naudiodon', or 'node-record-lpcm16' to capture
+     * actual microphone input.
+     */
+    private _startAudioInput;
+    /**
+     * Starts audio output processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'speaker', 'naudiodon', or similar to play audio
+     * through the system speakers.
+     */
+    private _startAudioOutput;
+}
+/**
+ * Creates a new DefaultAudioInterface instance.
+ *
+ * @returns A new DefaultAudioInterface instance
+ */
+export declare function createDefaultAudioInterface(): DefaultAudioInterface;

package/api/resources/conversationalAi/conversation/DefaultAudioInterface.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefaultAudioInterface = void 0;
+exports.createDefaultAudioInterface = createDefaultAudioInterface;
+const AudioInterface_1 = require("./AudioInterface");
+/**
+ * Default audio interface implementation for Node.js using basic audio processing.
+ *
+ * Note: This is a basic implementation. For production use, consider using
+ * professional audio libraries like 'naudiodon', 'mic'/'speaker', or similar.
+ *
+ * This implementation provides a foundation that can be extended with actual
+ * audio capture and playback capabilities.
+ */
+class DefaultAudioInterface extends AudioInterface_1.AudioInterface {
+    constructor() {
+        super(...arguments);
+        this.outputQueue = [];
+        this.shouldStop = false;
+    }
+    /**
+     * Starts the audio interface.
+     *
+     * @param inputCallback Function to call with audio chunks from the microphone
+     */
+    start(inputCallback) {
+        this.inputCallback = inputCallback;
+        this.shouldStop = false;
+        this.outputQueue = [];
+        // Start audio input simulation
+        // In a real implementation, this would capture from the microphone
+        this._startAudioInput();
+        // Start audio output processing
+        this._startAudioOutput();
+    }
+    /**
+     * Stops the audio interface and cleans up resources.
+     */
+    stop() {
+        this.shouldStop = true;
+        if (this.inputInterval) {
+            clearInterval(this.inputInterval);
+            this.inputInterval = undefined;
+        }
+        if (this.outputInterval) {
+            clearInterval(this.outputInterval);
+            this.outputInterval = undefined;
+        }
+        this.outputQueue = [];
+        this.inputCallback = undefined;
+    }
+    /**
+     * Output audio to the user.
+     *
+     * @param audio Audio data to output to the speaker
+     */
+    output(audio) {
+        if (!this.shouldStop) {
+            this.outputQueue.push(audio);
+        }
+    }
+    /**
+     * Interruption signal to stop any audio output.
+     */
+    interrupt() {
+        // Clear the output queue to stop any audio that is currently playing
+        this.outputQueue.length = 0;
+    }
+    /**
+     * Starts audio input processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'mic', 'naudiodon', or 'node-record-lpcm16' to capture
+     * actual microphone input.
+     */
+    _startAudioInput() {
+        // Simulate audio input by generating silent audio chunks
+        // In production, replace this with actual microphone capture
+        const chunkSize = DefaultAudioInterface.INPUT_FRAMES_PER_BUFFER * 2; // 16-bit = 2 bytes per sample
+        this.inputInterval = setInterval(() => {
+            if (this.shouldStop || !this.inputCallback) {
+                return;
+            }
+            // Generate silent audio chunk (all zeros)
+            // In a real implementation, this would be actual microphone data
+            const silentChunk = Buffer.alloc(chunkSize);
+            this.inputCallback(silentChunk);
+        }, 250); // 250ms intervals for 4000 samples at 16kHz
+    }
+    /**
+     * Starts audio output processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'speaker', 'naudiodon', or similar to play audio
+     * through the system speakers.
+     */
+    _startAudioOutput() {
+        this.outputInterval = setInterval(() => {
+            if (this.shouldStop) {
+                return;
+            }
+            // Process queued audio for output
+            if (this.outputQueue.length > 0) {
+                const audioChunk = this.outputQueue.shift();
+                if (audioChunk) {
+                    // In a real implementation, this would play the audio
+                    // For now, we just log that audio would be played
+                    console.debug(`Playing audio chunk of ${audioChunk.length} bytes`);
+                }
+            }
+        }, 62.5); // ~62.5ms intervals for 1000 samples at 16kHz
+    }
+}
+exports.DefaultAudioInterface = DefaultAudioInterface;
+DefaultAudioInterface.INPUT_FRAMES_PER_BUFFER = 4000; // 250ms @ 16kHz
+DefaultAudioInterface.OUTPUT_FRAMES_PER_BUFFER = 1000; // 62.5ms @ 16kHz
+DefaultAudioInterface.SAMPLE_RATE = 16000;
+DefaultAudioInterface.CHANNELS = 1;
+/**
+ * Creates a new DefaultAudioInterface instance.
+ *
+ * @returns A new DefaultAudioInterface instance
+ */
+function createDefaultAudioInterface() {
+    return new DefaultAudioInterface();
+}

package/api/resources/conversationalAi/conversation/__tests__/Conversation.test.d.ts

@@ -0,0 +1 @@
+export {};