@minded-ai/mindedjs 1.0.108 → 1.0.109-beta-1

package/src/voice/voiceSession.ts

@@ -1,294 +0,0 @@
-import { WebSocket } from 'ws';
-import { Agent } from '../agent';
-import * as mindedConnection from '../platform/mindedConnection';
-import { mindedConnectionSocketMessageType } from '../platform/mindedConnectionTypes';
-import { getConfig } from '../platform/config';
-import { ConversationInitiationClientData, ElevenLabsWebSocketEvent } from './elevenLabsUtils';
-import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
-import { v4 as uuidv4 } from 'uuid';
-import { logger } from '../utils/logger';
-import { SessionType } from '../types/Agent.types';
-
-/**
- * Voice Conversation class for managing individual ElevenLabs voice conversations
- */
-export class VoiceSession {
-  private agent: Agent;
-  private sessionId: string;
-  private firstMessage: string;
-  private voiceId?: string;
-  private elevenLabsSocket?: WebSocket;
-
-  private onAudioCallback?: (data: string) => void;
-  private onInterruptionCallback?: () => void;
-  private onMessageCallback?: (text: string, message: BaseMessage) => void;
-  private onDisconnectCallback?: () => void;
-
-  constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
-    logger.debug({ msg: '[Voice] Starting voice session', sessionId, firstMessage, voiceId });
-    this.agent = agent;
-    this.sessionId = sessionId;
-    this.firstMessage = firstMessage;
-    this.voiceId = voiceId;
-  }
-
-  /**
-   * Initialize the voice conversation connection
-   */
-  public async init(): Promise<void> {
-    const { elevenLabsKey, elevenLabsAgentId } = getConfig();
-
-    if (!elevenLabsKey) {
-      throw new Error('Missing ElevenLabs key - set ELEVEN_LABS_API_KEY env var');
-    }
-
-    if (!elevenLabsAgentId) {
-      throw new Error('Missing ElevenLabs agent id - set ELEVEN_LABS_AGENT_ID env var');
-    }
-
-    const signedUrl = await this.getSignedUrl({
-      agentId: elevenLabsAgentId,
-      apiKey: elevenLabsKey,
-    });
-
-    this.elevenLabsSocket = new WebSocket(signedUrl);
-    this.setupSocketHandlers();
-  }
-
-  private async sendToElevenLabs(message: string): Promise<void> {
-    if (!this.elevenLabsSocket) {
-      throw new Error('Socket not initialized');
-    }
-    await this.waitForSocketOpen(this.elevenLabsSocket);
-    this.elevenLabsSocket.send(message);
-  }
-
-  private async waitForSocketOpen(socket: WebSocket): Promise<void> {
-    if (socket.readyState === WebSocket.OPEN) {
-      return;
-    }
-    return new Promise((resolve, reject) => {
-      const timeoutMs = 10000;
-      const intervalMs = 100;
-      let tries = 0;
-      const interval = setInterval(() => {
-        if (socket.readyState === WebSocket.OPEN) {
-          clearInterval(interval);
-          resolve();
-        }
-        tries++;
-        if (tries >= timeoutMs) {
-          clearInterval(interval);
-          reject(new Error('Socket not open'));
-        }
-      }, intervalMs);
-    });
-  }
-
-  private setupSocketHandlers(): void {
-    const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
-
-    socket.onopen = () => {
-      logger.debug({ msg: '[Voice] Connected to ElevenLabs', sessionId: this.sessionId });
-      const initiationData: ConversationInitiationClientData = {
-        type: 'conversation_initiation_client_data',
-        conversation_config_override: {
-          agent: {
-            first_message: this.firstMessage,
-            language: 'en',
-          },
-          ...(this.voiceId
-            ? {
-                tts: {
-                  voice_id: this.voiceId,
-                },
-              }
-            : {}),
-        } as ConversationInitiationClientData['conversation_config_override'],
-      };
-
-      this.sendToElevenLabs(JSON.stringify(initiationData));
-      this.sendToElevenLabs(
-        JSON.stringify({ type: 'contextual_update', text: JSON.stringify({ sessionId: this.sessionId, mindedToken: getConfig().token! }) }),
-      );
-      this.addFirstMessageToState();
-    };
-
-    socket.onclose = () => {
-      logger.debug({ msg: '[Voice] Disconnected from ElevenLabs, removing voice session', sessionId: this.sessionId });
-      this.onDisconnectCallback?.();
-      this.agent.voiceSessions.delete(this.sessionId);
-    };
-
-    socket.onerror = (err: unknown) => {
-      logger.error({ msg: '[Voice] socket error', sessionId: this.sessionId, error: err });
-    };
-
-    socket.onmessage = async (event: any) => {
-      const data: ElevenLabsWebSocketEvent = JSON.parse(event.data.toString());
-
-      switch (data.type) {
-        case 'ping':
-          setTimeout(() => {
-            this.sendToElevenLabs(JSON.stringify({ type: 'pong', event_id: data.ping_event.event_id }));
-          }, data.ping_event.ping_ms);
-          break;
-        case 'user_transcript':
-          logger.debug({
-            msg: '[Voice] Received user transcript',
-            sessionId: this.sessionId,
-            data: data.user_transcription_event.user_transcript,
-          });
-          if (this.onMessageCallback) {
-            this.onMessageCallback(
-              data.user_transcription_event.user_transcript,
-              new HumanMessage({ content: data.user_transcription_event.user_transcript }),
-            );
-          }
-          break;
-        case 'agent_response':
-          logger.debug({ msg: '[Voice] Received agent response' });
-          if (this.onMessageCallback) {
-            this.onMessageCallback(
-              data.agent_response_event.agent_response,
-              new AIMessage({ content: data.agent_response_event.agent_response }),
-            );
-          }
-          break;
-        case 'interruption':
-          logger.debug({ msg: '[Voice] Received interruption' });
-          this.onInterruptionCallback?.();
-          // Send interruption event to dashboard if connected
-          if (getConfig().dashboardConnected) {
-            try {
-              mindedConnection.emit(mindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION, {
-                type: mindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION,
-                sessionId: this.sessionId,
-                timestamp: Date.now(),
-              });
-            } catch (error) {
-              logger.error({ msg: '[Voice] Error sending interruption to dashboard', sessionId: this.sessionId, error });
-            }
-          }
-          break;
-        case 'audio':
-          logger.debug({ msg: '[Voice] Received audio response to user', sessionId: this.sessionId });
-          if (this.onAudioCallback) {
-            this.onAudioCallback(data.audio_event.audio_base_64);
-          }
-          if (getConfig().dashboardConnected) {
-            try {
-              mindedConnection.emit(mindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO, {
-                sessionId: this.sessionId,
-                audioData: data.audio_event.audio_base_64,
-                timestamp: Date.now(),
-                type: mindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO,
-              });
-            } catch (error) {
-              logger.error({ msg: '[Voice] Error sending audio to dashboard', sessionId: this.sessionId, error });
-            }
-          }
-          break;
-        case 'conversation_initiation_metadata':
-          logger.trace({ msg: '[Voice] ElevenLabs conversation initiation metadata', sessionId: this.sessionId, data });
-          break;
-        case 'agent_response_correction':
-          try {
-            logger.debug({
-              msg: '[Voice] Received agent response correction',
-              sessionId: this.sessionId,
-              data: data.agent_response_correction_event,
-            });
-            await this.updateAgentResponse(
-              data.agent_response_correction_event.original_agent_response,
-              data.agent_response_correction_event.corrected_agent_response,
-            );
-          } catch (error) {
-            logger.error({ msg: '[Voice] Error updating agent response', sessionId: this.sessionId, error });
-          }
-          break;
-        default:
-          logger.debug({ msg: '[Voice] Received unknown message from ElevenLabs', sessionId: this.sessionId, data });
-          break;
-      }
-    };
-  }
-
-  private async addFirstMessageToState(): Promise<void> {
-    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
-    if (!graphState.values.messages?.length) {
-      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
-        messages: [new AIMessage({ content: this.firstMessage, id: uuidv4() })],
-        sessionType: SessionType.VOICE,
-      });
-    }
-  }
-
-  private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
-    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
-    const agentMessage: AIMessage | null = graphState.values.messages.find(
-      (message: BaseMessage) => message.content === originalAgentResponse && message instanceof AIMessage,
-    );
-    if (agentMessage) {
-      agentMessage.content = correctedAgentResponse;
-      agentMessage.additional_kwargs = agentMessage.additional_kwargs || {};
-      agentMessage.additional_kwargs.update = true;
-      const currentNodeId = graphState.tasks[graphState.tasks.length - 1].name;
-      await this.agent.compiledGraph.updateState(graphState.config, {
-        messages: [agentMessage],
-        overrideStartFromNodeId: currentNodeId,
-      });
-    } else {
-      logger.warn({ msg: '[Voice] Agent message not found for correction', sessionId: this.sessionId, originalAgentResponse });
-    }
-  }
-
-  public hangup(): void {
-    logger.info({ msg: '[Voice] Hanging up voice session', sessionId: this.sessionId });
-    this.onDisconnectCallback?.();
-    this.elevenLabsSocket?.close();
-  }
-
-  /**
-   * Set callback for audio data in base64 format
-   */
-  onAudio(callback: (data: string) => void): void {
-    this.onAudioCallback = callback;
-  }
-
-  /**
-   * Set callback for interruption events
-   */
-  onInterruption(callback: () => void): void {
-    this.onInterruptionCallback = callback;
-  }
-
-  /**
-   * Set callback for disconnect events
-   */
-  onDisconnect(callback: () => void): void {
-    this.onDisconnectCallback = callback;
-  }
-
-  /**
-   * User audio in base64 format
-   */
-  sendAudio(audioData: string): void {
-    this.sendToElevenLabs(JSON.stringify({ user_audio_chunk: audioData }));
-  }
-
-  /** Fetch signed URL from ElevenLabs API */
-  private async getSignedUrl({ agentId, apiKey }: { agentId: string; apiKey: string }): Promise<string> {
-    const response = await fetch(`https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${agentId}`, {
-      method: 'GET',
-      headers: {
-        'xi-api-key': apiKey,
-      },
-    });
-    if (!response.ok) {
-      throw new Error(`Failed to fetch signed url - status ${response.status}`);
-    }
-    const body = await response.json();
-    return body.signed_url;
-  }
-}