@elevenlabs/elevenlabs-js 2.4.1 → 2.6.0

package/api/resources/conversationalAi/conversation/ClientTools.js

@@ -0,0 +1,132 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClientTools = void 0;
+const events_1 = require("./events");
+/**
+ * Handles registration and execution of client-side tools that can be called by the agent.
+ *
+ * Supports both synchronous and asynchronous tools running in a dedicated event loop,
+ * ensuring non-blocking operation of the main conversation thread.
+ */
+class ClientTools {
+    constructor() {
+        this.tools = new Map();
+    }
+    /**
+     * Register a new tool that can be called by the AI agent.
+     *
+     * @param toolName Unique identifier for the tool
+     * @param handler Function that implements the tool's logic
+     * @param isAsync Whether the handler is an async function (auto-detected if not specified)
+     */
+    register(toolName, handler, isAsync) {
+        if (!handler || typeof handler !== "function") {
+            throw new Error("Handler must be a function");
+        }
+        if (this.tools.has(toolName)) {
+            throw new Error(`Tool '${toolName}' is already registered`);
+        }
+        // Auto-detect if the function is async if not specified
+        const isAsyncHandler = isAsync !== undefined ? isAsync : handler.constructor.name === "AsyncFunction";
+        this.tools.set(toolName, {
+            handler,
+            isAsync: isAsyncHandler,
+        });
+    }
+    /**
+     * Execute a registered tool with the given parameters.
+     *
+     * @param toolName Name of the tool to execute
+     * @param parameters Parameters to pass to the tool
+     * @returns The result of the tool execution
+     */
+    handle(toolName, parameters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const tool = this.tools.get(toolName);
+            if (!tool) {
+                throw new Error(`Tool '${toolName}' is not registered`);
+            }
+            if (tool.isAsync) {
+                return yield tool.handler(parameters);
+            }
+            else {
+                return tool.handler(parameters);
+            }
+        });
+    }
+    /**
+     * Execute a tool and send its result via the provided callback.
+     *
+     * This method is non-blocking and handles both sync and async tools.
+     *
+     * @param toolName Name of the tool to execute
+     * @param parameters Parameters to pass to the tool (should include tool_call_id)
+     * @param callback Function to call with the result
+     */
+    executeToolAsync(toolName, parameters, callback) {
+        // Run the tool execution in the next tick to avoid blocking
+        setImmediate(() => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const result = yield this.handle(toolName, parameters);
+                const response = {
+                    type: events_1.ClientToOrchestratorEvent.CLIENT_TOOL_RESULT,
+                    tool_call_id: parameters.tool_call_id,
+                    result: result !== null && result !== void 0 ? result : `Client tool: ${toolName} called successfully.`,
+                    is_error: false,
+                };
+                callback(response);
+            }
+            catch (error) {
+                const response = {
+                    type: events_1.ClientToOrchestratorEvent.CLIENT_TOOL_RESULT,
+                    tool_call_id: parameters.tool_call_id,
+                    result: error.message || String(error),
+                    is_error: true,
+                };
+                callback(response);
+            }
+        }));
+    }
+    /**
+     * Get a list of all registered tool names.
+     *
+     * @returns Array of tool names
+     */
+    getRegisteredTools() {
+        return Array.from(this.tools.keys());
+    }
+    /**
+     * Check if a tool is registered.
+     *
+     * @param toolName Name of the tool to check
+     * @returns True if the tool is registered
+     */
+    isToolRegistered(toolName) {
+        return this.tools.has(toolName);
+    }
+    /**
+     * Unregister a tool.
+     *
+     * @param toolName Name of the tool to unregister
+     * @returns True if the tool was found and removed
+     */
+    unregister(toolName) {
+        return this.tools.delete(toolName);
+    }
+    /**
+     * Clear all registered tools.
+     */
+    clear() {
+        this.tools.clear();
+    }
+}
+exports.ClientTools = ClientTools;

package/api/resources/conversationalAi/conversation/Conversation.d.ts

@@ -0,0 +1,94 @@
+import { EventEmitter } from "events";
+import { ElevenLabsClient } from "../../../../Client";
+import { AudioInterface } from "./AudioInterface";
+import { ClientTools } from "./ClientTools";
+import { ConversationInitiationData } from "./ConversationConfig";
+import { WebSocketFactory } from "./interfaces/WebSocketInterface";
+import { ConversationClient } from "./interfaces/ConversationClient";
+/**
+ * Conversational AI session for Node.js.
+ *
+ * BETA: This API is subject to change without regard to backwards compatibility.
+ */
+export declare class Conversation extends EventEmitter {
+    private client;
+    private agentId;
+    private requiresAuth;
+    private audioInterface;
+    private clientTools;
+    private config;
+    private webSocketFactory;
+    private conversationClient;
+    private callbackAgentResponse?;
+    private callbackAgentResponseCorrection?;
+    private callbackUserTranscript?;
+    private callbackLatencyMeasurement?;
+    private ws?;
+    private shouldStop;
+    private conversationId?;
+    private lastInterruptId;
+    private inputCallback?;
+    constructor(options: {
+        client?: ElevenLabsClient;
+        conversationClient?: ConversationClient;
+        webSocketFactory?: WebSocketFactory;
+        agentId: string;
+        requiresAuth: boolean;
+        audioInterface: AudioInterface;
+        config?: ConversationInitiationData;
+        clientTools?: ClientTools;
+        callbackAgentResponse?: (response: string) => void;
+        callbackAgentResponseCorrection?: (original: string, corrected: string) => void;
+        callbackUserTranscript?: (transcript: string) => void;
+        callbackLatencyMeasurement?: (latencyMs: number) => void;
+    });
+    /**
+     * Starts the conversation session.
+     *
+     * Will run until `endSession` is called.
+     */
+    startSession(): Promise<void>;
+    /**
+     * Ends the conversation session and cleans up resources.
+     */
+    endSession(): void;
+    /**
+     * Send a text message from the user to the agent.
+     *
+     * @param text The text message to send to the agent
+     */
+    sendUserMessage(text: string): void;
+    /**
+     * Register user activity to prevent session timeout.
+     *
+     * This sends a ping to the orchestrator to reset the timeout timer.
+     */
+    registerUserActivity(): void;
+    /**
+     * Send a contextual update to the conversation.
+     *
+     * Contextual updates are non-interrupting content that is sent to the server
+     * to update the conversation state without directly prompting the agent.
+     *
+     * @param text The contextual information to send to the conversation
+     */
+    sendContextualUpdate(text: string): void;
+    /**
+     * Get the conversation ID if available.
+     *
+     * @returns The conversation ID or undefined if not yet available
+     */
+    getConversationId(): string | undefined;
+    /**
+     * Check if the session is currently active.
+     *
+     * @returns True if the session is active
+     */
+    isSessionActive(): boolean;
+    private _onWebSocketOpen;
+    private _onWebSocketMessage;
+    private _onWebSocketClose;
+    private _handleMessage;
+    private _getWssUrl;
+    private _getSignedUrl;
+}

package/api/resources/conversationalAi/conversation/Conversation.js

@@ -0,0 +1,280 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Conversation = void 0;
+const events_1 = require("events");
+const ws_1 = __importDefault(require("ws"));
+const Client_1 = require("../../../../Client");
+const ClientTools_1 = require("./ClientTools");
+const events_2 = require("./events");
+const WebSocketInterface_1 = require("./interfaces/WebSocketInterface");
+/**
+ * Conversational AI session for Node.js.
+ *
+ * BETA: This API is subject to change without regard to backwards compatibility.
+ */
+class Conversation extends events_1.EventEmitter {
+    constructor(options) {
+        super();
+        this.shouldStop = false;
+        this.lastInterruptId = 0;
+        this.client = options.client || new Client_1.ElevenLabsClient();
+        this.agentId = options.agentId;
+        this.requiresAuth = options.requiresAuth;
+        this.audioInterface = options.audioInterface;
+        this.clientTools = options.clientTools || new ClientTools_1.ClientTools();
+        this.config = options.config || { extraBody: {}, conversationConfigOverride: {}, dynamicVariables: {} };
+        this.conversationClient = options.conversationClient || this.client;
+        this.webSocketFactory = options.webSocketFactory || new WebSocketInterface_1.DefaultWebSocketFactory();
+        this.callbackAgentResponse = options.callbackAgentResponse;
+        this.callbackAgentResponseCorrection = options.callbackAgentResponseCorrection;
+        this.callbackUserTranscript = options.callbackUserTranscript;
+        this.callbackLatencyMeasurement = options.callbackLatencyMeasurement;
+    }
+    /**
+     * Starts the conversation session.
+     *
+     * Will run until `endSession` is called.
+     */
+    startSession() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.ws) {
+                throw new Error("Session already started");
+            }
+            const wsUrl = this.requiresAuth ? yield this._getSignedUrl() : this._getWssUrl();
+            return new Promise((resolve, reject) => {
+                this.ws = this.webSocketFactory.create(wsUrl, {
+                    perMessageDeflate: false,
+                    maxPayload: 16 * 1024 * 1024, // 16MB max message size
+                });
+                this.ws.on("open", () => {
+                    this._onWebSocketOpen();
+                    resolve();
+                });
+                this.ws.on("message", (data) => {
+                    this._onWebSocketMessage(data);
+                });
+                this.ws.on("error", (error) => {
+                    this.emit("error", error);
+                    reject(error);
+                });
+                this.ws.on("close", (code, reason) => {
+                    this._onWebSocketClose(code, reason);
+                });
+            });
+        });
+    }
+    /**
+     * Ends the conversation session and cleans up resources.
+     */
+    endSession() {
+        this.shouldStop = true;
+        if (this.audioInterface) {
+            this.audioInterface.stop();
+        }
+        if (this.ws) {
+            this.ws.close();
+            this.ws = undefined;
+        }
+        this.emit("session_ended", this.conversationId);
+    }
+    /**
+     * Send a text message from the user to the agent.
+     *
+     * @param text The text message to send to the agent
+     */
+    sendUserMessage(text) {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.USER_MESSAGE,
+            text,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Register user activity to prevent session timeout.
+     *
+     * This sends a ping to the orchestrator to reset the timeout timer.
+     */
+    registerUserActivity() {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.USER_ACTIVITY,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Send a contextual update to the conversation.
+     *
+     * Contextual updates are non-interrupting content that is sent to the server
+     * to update the conversation state without directly prompting the agent.
+     *
+     * @param text The contextual information to send to the conversation
+     */
+    sendContextualUpdate(text) {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            throw new Error("Session not started or websocket not connected");
+        }
+        const event = {
+            type: events_2.ClientToOrchestratorEvent.CONTEXTUAL_UPDATE,
+            text,
+        };
+        this.ws.send(JSON.stringify(event));
+    }
+    /**
+     * Get the conversation ID if available.
+     *
+     * @returns The conversation ID or undefined if not yet available
+     */
+    getConversationId() {
+        return this.conversationId;
+    }
+    /**
+     * Check if the session is currently active.
+     *
+     * @returns True if the session is active
+     */
+    isSessionActive() {
+        return !!this.ws && this.ws.readyState === ws_1.default.OPEN && !this.shouldStop;
+    }
+    _onWebSocketOpen() {
+        // Send conversation initiation data
+        const initEvent = {
+            type: events_2.ClientToOrchestratorEvent.CONVERSATION_INITIATION_CLIENT_DATA,
+            custom_llm_extra_body: this.config.extraBody,
+            conversation_config_override: this.config.conversationConfigOverride,
+            dynamic_variables: this.config.dynamicVariables,
+        };
+        this.ws.send(JSON.stringify(initEvent));
+        // Set up audio input callback
+        this.inputCallback = (audio) => {
+            if (this.shouldStop || !this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+                return;
+            }
+            try {
+                const audioEvent = {
+                    user_audio_chunk: audio.toString("base64"),
+                };
+                this.ws.send(JSON.stringify(audioEvent));
+            }
+            catch (error) {
+                console.error("Error sending user audio chunk:", error);
+                this.endSession();
+            }
+        };
+        // Start audio interface
+        this.audioInterface.start(this.inputCallback);
+        this.emit("session_started");
+    }
+    _onWebSocketMessage(data) {
+        if (this.shouldStop) {
+            return;
+        }
+        try {
+            const message = JSON.parse(data.toString());
+            this._handleMessage(message);
+        }
+        catch (error) {
+            console.error("Error parsing WebSocket message:", error);
+        }
+    }
+    _onWebSocketClose(code, reason) {
+        this.emit("session_ended", this.conversationId, code, reason.toString());
+    }
+    _handleMessage(message) {
+        const messageType = message.type;
+        switch (messageType) {
+            case "conversation_initiation_metadata":
+                const event = message.conversation_initiation_metadata_event;
+                if (!this.conversationId) {
+                    this.conversationId = event.conversation_id;
+                    this.emit("conversation_started", this.conversationId);
+                }
+                break;
+            case "audio":
+                const audioEvent = message.audio_event;
+                if (parseInt(audioEvent.event_id) <= this.lastInterruptId) {
+                    return;
+                }
+                const audio = Buffer.from(audioEvent.audio_base_64, "base64");
+                this.audioInterface.output(audio);
+                break;
+            case "agent_response":
+                if (this.callbackAgentResponse) {
+                    const responseEvent = message.agent_response_event;
+                    this.callbackAgentResponse(responseEvent.agent_response.trim());
+                }
+                break;
+            case "agent_response_correction":
+                if (this.callbackAgentResponseCorrection) {
+                    const correctionEvent = message.agent_response_correction_event;
+                    this.callbackAgentResponseCorrection(correctionEvent.original_agent_response.trim(), correctionEvent.corrected_agent_response.trim());
+                }
+                break;
+            case "user_transcript":
+                if (this.callbackUserTranscript) {
+                    const transcriptEvent = message.user_transcription_event;
+                    this.callbackUserTranscript(transcriptEvent.user_transcript.trim());
+                }
+                break;
+            case "interruption":
+                const interruptionEvent = message.interruption_event;
+                this.lastInterruptId = parseInt(interruptionEvent.event_id);
+                this.audioInterface.interrupt();
+                break;
+            case "ping":
+                const pingEvent = message.ping_event;
+                const pongEvent = {
+                    type: events_2.ClientToOrchestratorEvent.PONG,
+                    event_id: pingEvent.event_id,
+                };
+                this.ws.send(JSON.stringify(pongEvent));
+                if (this.callbackLatencyMeasurement && pingEvent.ping_ms) {
+                    this.callbackLatencyMeasurement(parseInt(pingEvent.ping_ms));
+                }
+                break;
+            case "client_tool_call":
+                const toolCall = message.client_tool_call;
+                const toolName = toolCall.tool_name;
+                const parameters = Object.assign({ tool_call_id: toolCall.tool_call_id }, toolCall.parameters);
+                this.clientTools.executeToolAsync(toolName, parameters, (response) => {
+                    if (!this.shouldStop && this.ws && this.ws.readyState === ws_1.default.OPEN) {
+                        this.ws.send(JSON.stringify(response));
+                    }
+                });
+                break;
+            default:
+                // Ignore unknown message types
+                break;
+        }
+    }
+    _getWssUrl() {
+        // Default to production environment WebSocket URL
+        const baseWsUrl = "wss://api.elevenlabs.io";
+        return `${baseWsUrl}/v1/convai/conversation?agent_id=${this.agentId}`;
+    }
+    _getSignedUrl() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.conversationClient.conversationalAi.conversations.getSignedUrl({
+                agentId: this.agentId,
+            });
+            return response.signedUrl;
+        });
+    }
+}
+exports.Conversation = Conversation;

package/api/resources/conversationalAi/conversation/ConversationConfig.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Configuration options for the Conversation.
+ */
+export interface ConversationInitiationData {
+    /** Extra body parameters to include in the conversation initiation request */
+    extraBody?: Record<string, any>;
+    /** Configuration overrides for the conversation */
+    conversationConfigOverride?: Record<string, any>;
+    /** Dynamic variables to use in the conversation */
+    dynamicVariables?: Record<string, any>;
+}
+/**
+ * Creates a default conversation initiation data object.
+ *
+ * @param options Optional partial configuration to override defaults
+ * @returns Complete conversation initiation data object
+ */
+export declare function createConversationInitiationData(options?: Partial<ConversationInitiationData>): ConversationInitiationData;

package/api/resources/conversationalAi/conversation/ConversationConfig.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createConversationInitiationData = createConversationInitiationData;
+/**
+ * Creates a default conversation initiation data object.
+ *
+ * @param options Optional partial configuration to override defaults
+ * @returns Complete conversation initiation data object
+ */
+function createConversationInitiationData(options = {}) {
+    return {
+        extraBody: options.extraBody || {},
+        conversationConfigOverride: options.conversationConfigOverride || {},
+        dynamicVariables: options.dynamicVariables || {},
+    };
+}

package/api/resources/conversationalAi/conversation/DefaultAudioInterface.d.ts

@@ -0,0 +1,63 @@
+import { AudioInterface } from "./AudioInterface";
+/**
+ * Default audio interface implementation for Node.js using basic audio processing.
+ *
+ * Note: This is a basic implementation. For production use, consider using
+ * professional audio libraries like 'naudiodon', 'mic'/'speaker', or similar.
+ *
+ * This implementation provides a foundation that can be extended with actual
+ * audio capture and playback capabilities.
+ */
+export declare class DefaultAudioInterface extends AudioInterface {
+    private static readonly INPUT_FRAMES_PER_BUFFER;
+    private static readonly OUTPUT_FRAMES_PER_BUFFER;
+    private static readonly SAMPLE_RATE;
+    private static readonly CHANNELS;
+    private inputCallback?;
+    private outputQueue;
+    private shouldStop;
+    private inputInterval?;
+    private outputInterval?;
+    /**
+     * Starts the audio interface.
+     *
+     * @param inputCallback Function to call with audio chunks from the microphone
+     */
+    start(inputCallback: (audio: Buffer) => void): void;
+    /**
+     * Stops the audio interface and cleans up resources.
+     */
+    stop(): void;
+    /**
+     * Output audio to the user.
+     *
+     * @param audio Audio data to output to the speaker
+     */
+    output(audio: Buffer): void;
+    /**
+     * Interruption signal to stop any audio output.
+     */
+    interrupt(): void;
+    /**
+     * Starts audio input processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'mic', 'naudiodon', or 'node-record-lpcm16' to capture
+     * actual microphone input.
+     */
+    private _startAudioInput;
+    /**
+     * Starts audio output processing.
+     *
+     * Note: This is a placeholder implementation. In a real scenario, you would
+     * use libraries like 'speaker', 'naudiodon', or similar to play audio
+     * through the system speakers.
+     */
+    private _startAudioOutput;
+}
+/**
+ * Creates a new DefaultAudioInterface instance.
+ *
+ * @returns A new DefaultAudioInterface instance
+ */
+export declare function createDefaultAudioInterface(): DefaultAudioInterface;