@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/voice/voiceSession.ts

@@ -0,0 +1,309 @@
+import { WebSocket } from 'ws';
+import { Agent } from '../agent';
+import * as mindedConnection from '../platform/mindedConnection';
+import { mindedConnectionSocketMessageType } from '../platform/mindedConnectionTypes';
+import { getConfig } from '../platform/config';
+import { ConversationInitiationClientData, ElevenLabsWebSocketEvent } from './elevenLabsUtils';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { logger } from '../utils/logger';
+import { SessionType } from '../types/Agent.types';
+import { QUEUE_INTERRUPT_DETECTED_BEFORE_SPEECH } from '../interrupts/BaseInterruptSessionManager';
+
+/**
+ * Voice Conversation class for managing individual ElevenLabs voice conversations
+ */
+export class VoiceSession {
+  private agent: Agent;
+  private sessionId: string;
+  private firstMessage: string;
+  private voiceId?: string;
+  private elevenLabsSocket?: WebSocket;
+
+  private onAudioCallback?: (data: string) => void;
+  private onInterruptionCallback?: () => void;
+  private onMessageCallback?: (text: string, message: BaseMessage) => void;
+  private onDisconnectCallback?: () => void;
+
+  constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
+    logger.debug({ msg: '[Voice] Starting voice session', sessionId, firstMessage, voiceId });
+    this.agent = agent;
+    this.sessionId = sessionId;
+    this.firstMessage = firstMessage;
+    this.voiceId = voiceId;
+  }
+
+  /**
+   * Initialize the voice conversation connection
+   */
+  public async init(): Promise<void> {
+    const { elevenLabsKey, elevenLabsAgentId } = getConfig();
+
+    if (!elevenLabsKey) {
+      throw new Error('Missing ElevenLabs key - set ELEVEN_LABS_API_KEY env var');
+    }
+
+    if (!elevenLabsAgentId) {
+      throw new Error('Missing ElevenLabs agent id - set ELEVEN_LABS_AGENT_ID env var');
+    }
+
+    const signedUrl = await this.getSignedUrl({
+      agentId: elevenLabsAgentId,
+      apiKey: elevenLabsKey,
+    });
+
+    this.elevenLabsSocket = new WebSocket(signedUrl);
+    this.setupSocketHandlers();
+  }
+
+  private async sendToElevenLabs(message: string): Promise<void> {
+    if (!this.elevenLabsSocket) {
+      throw new Error('Socket not initialized');
+    }
+    await this.waitForSocketOpen(this.elevenLabsSocket);
+    this.elevenLabsSocket.send(message);
+  }
+
+  private async waitForSocketOpen(socket: WebSocket): Promise<void> {
+    if (socket.readyState === WebSocket.OPEN) {
+      return;
+    }
+    return new Promise((resolve, reject) => {
+      const timeoutMs = 10000;
+      const intervalMs = 100;
+      let tries = 0;
+      const interval = setInterval(() => {
+        if (socket.readyState === WebSocket.OPEN) {
+          clearInterval(interval);
+          resolve();
+        }
+        tries++;
+        if (tries >= timeoutMs) {
+          clearInterval(interval);
+          reject(new Error('Socket not open'));
+        }
+      }, intervalMs);
+    });
+  }
+
+  private setupSocketHandlers(): void {
+    const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
+
+    socket.onopen = () => {
+      logger.debug({ msg: '[Voice] Connected to ElevenLabs', sessionId: this.sessionId });
+      const initiationData: ConversationInitiationClientData = {
+        type: 'conversation_initiation_client_data',
+        conversation_config_override: {
+          agent: {
+            first_message: this.firstMessage,
+            language: 'en',
+          },
+          ...(this.voiceId
+            ? {
+                tts: {
+                  voice_id: this.voiceId,
+                },
+              }
+            : {}),
+        } as ConversationInitiationClientData['conversation_config_override'],
+      };
+
+      this.sendToElevenLabs(JSON.stringify(initiationData));
+      this.sendToElevenLabs(
+        JSON.stringify({ type: 'contextual_update', text: JSON.stringify({ sessionId: this.sessionId, mindedToken: getConfig().token! }) }),
+      );
+      this.addFirstMessageToState();
+    };
+
+    socket.onclose = () => {
+      logger.debug({ msg: '[Voice] Disconnected from ElevenLabs, removing voice session', sessionId: this.sessionId });
+      this.onDisconnectCallback?.();
+      this.agent.voiceSessions.delete(this.sessionId);
+    };
+
+    socket.onerror = (err: unknown) => {
+      logger.error({ msg: '[Voice] socket error', sessionId: this.sessionId, error: err });
+    };
+
+    socket.onmessage = async (event: any) => {
+      const data: ElevenLabsWebSocketEvent = JSON.parse(event.data.toString());
+
+      switch (data.type) {
+        case 'ping':
+          setTimeout(() => {
+            this.sendToElevenLabs(JSON.stringify({ type: 'pong', event_id: data.ping_event.event_id }));
+          }, data.ping_event.ping_ms);
+          break;
+        case 'user_transcript':
+          logger.debug({
+            msg: '[Voice] Received user transcript',
+            sessionId: this.sessionId,
+            data: data.user_transcription_event.user_transcript,
+          });
+          if (this.onMessageCallback) {
+            this.onMessageCallback(
+              data.user_transcription_event.user_transcript,
+              new HumanMessage({ content: data.user_transcription_event.user_transcript }),
+            );
+          }
+          break;
+        case 'agent_response':
+          logger.debug({ msg: '[Voice] Received agent response' });
+          if (this.onMessageCallback) {
+            this.onMessageCallback(
+              data.agent_response_event.agent_response,
+              new AIMessage({ content: data.agent_response_event.agent_response }),
+            );
+          }
+          break;
+        case 'interruption':
+          logger.debug({ msg: '[Voice] Received interruption' });
+          if (await this.agent.interruptSessionManager.isProcessed(this.sessionId)) {
+            await this.agent.interruptSessionManager.enqueueMessage(this.sessionId, {
+              triggerBody: {},
+              triggerName: QUEUE_INTERRUPT_DETECTED_BEFORE_SPEECH,
+            });
+          }
+          this.onInterruptionCallback?.();
+          // Send interruption event to dashboard if connected
+          if (getConfig().dashboardConnected) {
+            try {
+              mindedConnection.emit(mindedConnectionSocketMessageType.VOICE_INTERRUPTION, {
+                type: mindedConnectionSocketMessageType.VOICE_INTERRUPTION,
+                sessionId: this.sessionId,
+                timestamp: Date.now(),
+              });
+            } catch (err) {
+              logger.error({ message: '[Voice] Error sending interruption to dashboard', sessionId: this.sessionId, err });
+            }
+          }
+          break;
+        case 'audio':
+          logger.debug({ msg: '[Voice] Received audio response to user', sessionId: this.sessionId });
+          if (this.onAudioCallback) {
+            this.onAudioCallback(data.audio_event.audio_base_64);
+          }
+          if (getConfig().dashboardConnected) {
+            try {
+              mindedConnection.emit(mindedConnectionSocketMessageType.VOICE_AGENT_AUDIO, {
+                sessionId: this.sessionId,
+                audioData: data.audio_event.audio_base_64,
+                timestamp: Date.now(),
+                type: mindedConnectionSocketMessageType.VOICE_AGENT_AUDIO,
+              });
+            } catch (err) {
+              logger.error({ message: '[Voice] Error sending audio to dashboard', sessionId: this.sessionId, err });
+            }
+          }
+          break;
+        case 'conversation_initiation_metadata':
+          logger.trace({ msg: '[Voice] ElevenLabs conversation initiation metadata', sessionId: this.sessionId, data });
+          break;
+        case 'agent_response_correction':
+          try {
+            logger.debug({
+              msg: '[Voice] Received agent response correction',
+              sessionId: this.sessionId,
+              data: data.agent_response_correction_event,
+            });
+            await this.updateAgentResponse(
+              data.agent_response_correction_event.original_agent_response,
+              data.agent_response_correction_event.corrected_agent_response,
+            );
+          } catch (err) {
+            logger.error({ message: '[Voice] Error updating agent response', sessionId: this.sessionId, err });
+          }
+          break;
+        default:
+          logger.debug({ msg: '[Voice] Received unknown message from ElevenLabs', sessionId: this.sessionId, data });
+          break;
+      }
+    };
+  }
+
+  private async addFirstMessageToState(): Promise<void> {
+    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    if (!graphState.values.messages?.length) {
+      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
+        messages: [new AIMessage({ content: this.firstMessage, id: uuidv4() })],
+        sessionType: SessionType.VOICE,
+      });
+    }
+  }
+
+  private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
+    const cleanedOriginalAgentResponse = originalAgentResponse.replace('... ', '');
+    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    const agentMessage: AIMessage | null = graphState.values.messages.find(
+      (message: BaseMessage) => message.content === cleanedOriginalAgentResponse && message instanceof AIMessage,
+    );
+    if (agentMessage) {
+      agentMessage.content = correctedAgentResponse;
+      agentMessage.additional_kwargs = agentMessage.additional_kwargs || {};
+      agentMessage.additional_kwargs.update = true;
+      const currentNodeId = graphState.tasks[graphState.tasks.length - 1].name;
+      await this.agent.compiledGraph.updateState(graphState.config, {
+        messages: [agentMessage],
+        goto: currentNodeId,
+      });
+    } else {
+      logger.warn({ msg: '[Voice] Agent message not found for correction', sessionId: this.sessionId, originalAgentResponse });
+    }
+  }
+
+  /*
+   * Hangup the voice session
+   */
+  public hangup(): void {
+    logger.info({ msg: '[Voice] Hanging up voice session', sessionId: this.sessionId });
+    this.onDisconnectCallback?.();
+    this.elevenLabsSocket?.close();
+  }
+
+  /*
+   * Set callback for audio data in base64 format
+   * @param callback - The callback to handle the audio data
+   */
+  public onAudio(callback: (data: string) => void): void {
+    this.onAudioCallback = callback;
+  }
+
+  /*
+   * Set callback for interruption events
+   * @param callback - The callback to handle the interruption event
+   */
+  onInterruption(callback: () => void): void {
+    this.onInterruptionCallback = callback;
+  }
+
+  /*
+   * Set callback for disconnect events
+   * @param callback - The callback to handle the disconnect event
+   */
+  onDisconnect(callback: () => void): void {
+    this.onDisconnectCallback = callback;
+  }
+
+  /*
+   * Send audio to ElevenLabs
+   * @param audioData - The audio data to send to ElevenLabs
+   */
+  sendAudio(audioData: string): void {
+    this.sendToElevenLabs(JSON.stringify({ user_audio_chunk: audioData }));
+  }
+
+  /* Fetch signed URL from ElevenLabs API */
+  private async getSignedUrl({ agentId, apiKey }: { agentId: string; apiKey: string }): Promise<string> {
+    const response = await fetch(`https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${agentId}`, {
+      method: 'GET',
+      headers: {
+        'xi-api-key': apiKey,
+      },
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to fetch signed url - status ${response.status}`);
+    }
+    const body = await response.json();
+    return body.signed_url;
+  }
+}