@minded-ai/mindedjs 1.0.53-patch6 → 1.0.53

package/src/voice/voiceSession.ts

@@ -1,257 +0,0 @@
-import { WebSocket } from 'ws';
-import { Agent } from '../agent';
-import {
-  MindedConnectionSocketMessageType,
-} from '../platform/mindedConnectionTypes';
-import { getConfig } from '../platform/config';
-import {
-  ElevenLabsWebSocketEvent,
-  getSignedUrl,
-} from './elevenLabsUtils';
-import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
-
-/**
- * Voice Conversation class for managing individual ElevenLabs voice conversations
- */
-export class VoiceSession {
-  private agent: Agent;
-  private sessionId: string;
-  private firstMessage: string;
-  private voiceId?: string;
-  private elevenLabsSocket?: WebSocket;
-
-  private onAudioCallback?: (data: string) => void;
-  private onInterruptionCallback?: () => void;
-  private onMessageCallback?: (text: string, message: BaseMessage) => void;
-  private onDisconnectCallback?: () => void;
-
-  constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
-    console.debug('Starting voice session', { sessionId, firstMessage });
-    this.agent = agent;
-    this.sessionId = sessionId;
-    this.firstMessage = firstMessage;
-    this.voiceId = voiceId;
-  }
-
-  /**
-   * Initialize the voice conversation connection
-   */
-  public async init(): Promise<void> {
-    const { elevenLabsKey, elevenLabsAgentId } = getConfig();
-
-    if (!elevenLabsKey) {
-      throw new Error('Missing ElevenLabs key - set ELEVEN_LABS_KEY env var');
-    }
-
-    if (!elevenLabsAgentId) {
-      throw new Error('Missing ElevenLabs agent id - set ELEVEN_LABS_AGENT_ID env var');
-    }
-
-    const signedUrl = await getSignedUrl({
-      agentId: elevenLabsAgentId,
-      apiKey: elevenLabsKey
-    });
-
-    this.elevenLabsSocket = new WebSocket(signedUrl);
-    this.setupSocketHandlers();
-  }
-
-  private async sendToElevenLabs(message: string): Promise<void> {
-    if (!this.elevenLabsSocket) {
-      throw new Error('Socket not initialized');
-    }
-    await this.waitForSocketOpen(this.elevenLabsSocket);
-    this.elevenLabsSocket.send(message);
-  }
-
-  private async waitForSocketOpen(socket: WebSocket): Promise<void> {
-    if (socket.readyState === WebSocket.OPEN) {
-      return;
-    }
-    return new Promise((resolve, reject) => {
-      const timeoutMs = 10000;
-      const intervalMs = 100;
-      let tries = 0;
-      const interval = setInterval(() => {
-        if (socket.readyState === WebSocket.OPEN) {
-          clearInterval(interval);
-          resolve();
-        }
-        tries++;
-        if (tries >= timeoutMs) {
-          clearInterval(interval);
-          reject(new Error('Socket not open'));
-        }
-      }, intervalMs);
-    });
-  }
-
-  private setupSocketHandlers(): void {
-    const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
-
-    socket.onopen = () => {
-      console.debug('Connected to voice provider');
-      const initiationData: ConversationInitiationClientData = {
-        type: 'conversation_initiation_client_data',
-        conversation_config_override: {
-          agent: {
-            first_message: this.firstMessage,
-            language: 'en',
-          },
-          ...(this.voiceId
-            ? {
-              tts: {
-                voice_id: this.voiceId,
-              },
-            }
-            : {}),
-        } as ConversationInitiationClientData['conversation_config_override'],
-      };
-
-      this.sendToElevenLabs(JSON.stringify(initiationData));
-      this.sendToElevenLabs(JSON.stringify({ type: 'contextual_update', text: JSON.stringify({ sessionId: this.sessionId, mindedToken: getConfig().token! }) }));
-    };
-
-    socket.onclose = () => {
-      console.debug('Disconnected from voice provider');
-      this.onDisconnectCallback?.();
-      this.agent.voiceSessions.delete(this.sessionId);
-    };
-
-    socket.onerror = (err: unknown) => {
-      console.error('[ElevenLabsVoice] WebSocket error', err);
-    };
-
-    socket.onmessage = async (event: any) => {
-      const data: ElevenLabsWebSocketEvent = JSON.parse(event.data.toString());
-
-      switch (data.type) {
-        case 'ping':
-          setTimeout(() => {
-            this.sendToElevenLabs(JSON.stringify({ type: 'pong', event_id: data.ping_event.event_id }));
-          }, data.ping_event.ping_ms);
-          break;
-        case 'user_transcript':
-          console.debug('User transcript received', data.user_transcription_event.user_transcript);
-          if (this.onMessageCallback) {
-            this.onMessageCallback(data.user_transcription_event.user_transcript, new HumanMessage(data.user_transcription_event.user_transcript));
-          }
-          break;
-        case 'agent_response':
-          if (this.onMessageCallback) {
-            this.onMessageCallback(data.agent_response_event.agent_response, new AIMessage(data.agent_response_event.agent_response));
-          }
-          break;
-        case 'interruption':
-          console.debug('Interruption received');
-          this.onInterruptionCallback?.();
-          // Send interruption event to dashboard if connected
-          if (getConfig().dashboardConnected) {
-            try {
-              this.agent.mindedConnection?.emit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION, {
-                type: MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION,
-                sessionId: this.sessionId,
-                timestamp: Date.now(),
-              });
-            } catch (error) {
-              console.error('[ElevenLabsVoice] Error sending interruption to dashboard', error);
-            }
-          }
-          break;
-        case 'audio':
-          if (this.onAudioCallback) {
-            this.onAudioCallback(data.audio_event.audio_base_64);
-          }
-          if (getConfig().dashboardConnected) {
-            try {
-              await this.agent.mindedConnection?.awaitEmit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO, {
-                sessionId: this.sessionId,
-                audioData: data.audio_event.audio_base_64,
-              });
-            } catch (error) {
-              console.error('[ElevenLabsVoice] Error sending audio to dashboard', error);
-            }
-          }
-          break;
-        case 'conversation_initiation_metadata':
-          console.debug('ElevenLabs conversation initiation metadata', data);
-          break;
-        case 'agent_response_correction':
-          try {
-            console.debug('Agent response correction received', data.agent_response_correction_event);
-            await this.updateAgentResponse(data.agent_response_correction_event.original_agent_response, data.agent_response_correction_event.corrected_agent_response);
-          } catch (error) {
-            console.error('[ElevenLabsVoice] Error updating agent response', error);
-          }
-          break;
-        default:
-          console.debug('Received unknown message from ElevenLabs', data);
-          break;
-      }
-    };
-  }
-
-  private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
-    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
-    const agentMessage = graphState.values.messages.find((message: BaseMessage) => message.content === originalAgentResponse && message instanceof AIMessage);
-    if (agentMessage) {
-      agentMessage.content = correctedAgentResponse;
-      const currentNodeId = graphState.tasks[graphState.tasks.length - 1].name;
-      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
-        messages: graphState.values.messages,
-      }, currentNodeId);
-    }
-    else {
-      console.warn('Agent message not found for correction', originalAgentResponse);
-    }
-  }
-
-  public hangup(): void {
-    this.onDisconnectCallback?.();
-    this.elevenLabsSocket?.close();
-  }
-
-  /**
-   * Set callback for audio data in base64 format
-   */
-  onAudio(callback: (data: string) => void): void {
-    this.onAudioCallback = callback;
-  }
-
-  /**
-   * Set callback for interruption events
-   */
-  onInterruption(callback: () => void): void {
-    this.onInterruptionCallback = callback;
-  }
-
-  /**
-   * Set callback for disconnect events
-   */
-  onDisconnect(callback: () => void): void {
-    this.onDisconnectCallback = callback;
-  }
-
-  /**
-   * User audio in base64 format
-   */
-  sendAudio(audioData: string): void {
-    this.sendToElevenLabs(JSON.stringify({ user_audio_chunk: audioData }));
-  }
-}
-
-// ----------------- Helper Types -----------------
-
-/** Shape of the message sent to ElevenLabs when starting a conversation */
-interface ConversationInitiationClientData {
-  type: 'conversation_initiation_client_data';
-  conversation_config_override: {
-    agent: {
-      first_message: string;
-      language: string;
-    };
-    tts?: {
-      voice_id: string;
-    };
-  };
-}