@minded-ai/mindedjs 1.0.57 → 1.0.59

package/src/voice/elevenLabsUtils.ts

@@ -0,0 +1,81 @@
+// ----------------- ElevenLabs WebSocket event types -----------------
+
+interface BaseEvent {
+  type: string;
+}
+
+interface UserTranscriptEvent extends BaseEvent {
+  type: 'user_transcript';
+  user_transcription_event: {
+    user_transcript: string;
+  };
+}
+
+interface AgentResponseEvent extends BaseEvent {
+  type: 'agent_response';
+  agent_response_event: {
+    agent_response: string;
+  };
+}
+
+interface AudioResponseEvent extends BaseEvent {
+  type: 'audio';
+  audio_event: {
+    audio_base_64: string;
+    event_id: number;
+  };
+}
+
+interface InterruptionEvent extends BaseEvent {
+  type: 'interruption';
+  interruption_event: {
+    event_id: number;
+  };
+}
+
+interface PingEvent extends BaseEvent {
+  type: 'ping';
+  ping_event: {
+    event_id: number;
+    ping_ms: number;
+  };
+}
+
+interface ConversationInitiationMetadataEvent extends BaseEvent {
+  type: 'conversation_initiation_metadata';
+  conversation_initiation_metadata_event: {
+    conversation_id: string;
+  };
+}
+
+export interface AgentResponseCorrectionEvent extends BaseEvent {
+  type: 'agent_response_correction';
+  agent_response_correction_event: {
+    corrected_agent_response: string;
+    original_agent_response: string;
+  };
+}
+
+/** Union of all possible WebSocket events we care about */
+export type ElevenLabsWebSocketEvent =
+  | UserTranscriptEvent
+  | AgentResponseEvent
+  | AudioResponseEvent
+  | InterruptionEvent
+  | PingEvent
+  | ConversationInitiationMetadataEvent
+  | AgentResponseCorrectionEvent;
+
+/** Shape of the message sent to ElevenLabs when starting a conversation */
+export interface ConversationInitiationClientData {
+  type: 'conversation_initiation_client_data';
+  conversation_config_override: {
+    agent: {
+      first_message: string;
+      language: string;
+    };
+    tts?: {
+      voice_id: string;
+    };
+  };
+} 

package/src/voice/voiceSession.ts

@@ -0,0 +1,277 @@
+import { WebSocket } from 'ws';
+import { Agent } from '../agent';
+import {
+  MindedConnectionSocketMessageType,
+} from '../platform/mindedConnectionTypes';
+import { getConfig } from '../platform/config';
+import {
+  ConversationInitiationClientData,
+  ElevenLabsWebSocketEvent,
+} from './elevenLabsUtils';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { logger } from '../utils/logger';
+
+/**
+ * Voice Conversation class for managing individual ElevenLabs voice conversations
+ */
+export class VoiceSession {
+  private agent: Agent;
+  private sessionId: string;
+  private firstMessage: string;
+  private voiceId?: string;
+  private elevenLabsSocket?: WebSocket;
+
+  private onAudioCallback?: (data: string) => void;
+  private onInterruptionCallback?: () => void;
+  private onMessageCallback?: (text: string, message: BaseMessage) => void;
+  private onDisconnectCallback?: () => void;
+
+  constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
+    logger.debug('Starting voice session', { sessionId, firstMessage, voiceId });
+    this.agent = agent;
+    this.sessionId = sessionId;
+    this.firstMessage = firstMessage;
+    this.voiceId = voiceId;
+  }
+
+  /**
+   * Initialize the voice conversation connection
+   */
+  public async init(): Promise<void> {
+    const { elevenLabsKey, elevenLabsAgentId } = getConfig();
+
+    if (!elevenLabsKey) {
+      throw new Error('Missing ElevenLabs key - set ELEVEN_LABS_KEY env var');
+    }
+
+    if (!elevenLabsAgentId) {
+      throw new Error('Missing ElevenLabs agent id - set ELEVEN_LABS_AGENT_ID env var');
+    }
+
+    const signedUrl = await this.getSignedUrl({
+      agentId: elevenLabsAgentId,
+      apiKey: elevenLabsKey
+    });
+
+    this.elevenLabsSocket = new WebSocket(signedUrl);
+    this.setupSocketHandlers();
+  }
+
+  private async sendToElevenLabs(message: string): Promise<void> {
+    if (!this.elevenLabsSocket) {
+      throw new Error('Socket not initialized');
+    }
+    await this.waitForSocketOpen(this.elevenLabsSocket);
+    this.elevenLabsSocket.send(message);
+  }
+
+  private async waitForSocketOpen(socket: WebSocket): Promise<void> {
+    if (socket.readyState === WebSocket.OPEN) {
+      return;
+    }
+    return new Promise((resolve, reject) => {
+      const timeoutMs = 10000;
+      const intervalMs = 100;
+      let tries = 0;
+      const interval = setInterval(() => {
+        if (socket.readyState === WebSocket.OPEN) {
+          clearInterval(interval);
+          resolve();
+        }
+        tries++;
+        if (tries >= timeoutMs) {
+          clearInterval(interval);
+          reject(new Error('Socket not open'));
+        }
+      }, intervalMs);
+    });
+  }
+
+  private setupSocketHandlers(): void {
+    const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
+
+    socket.onopen = () => {
+      logger.debug('Connected to ElevenLabs');
+      const initiationData: ConversationInitiationClientData = {
+        type: 'conversation_initiation_client_data',
+        conversation_config_override: {
+          agent: {
+            first_message: this.firstMessage,
+            language: 'en',
+          },
+          ...(this.voiceId
+            ? {
+              tts: {
+                voice_id: this.voiceId,
+              },
+            }
+            : {}),
+        } as ConversationInitiationClientData['conversation_config_override'],
+      };
+
+      this.sendToElevenLabs(JSON.stringify(initiationData));
+      this.sendToElevenLabs(JSON.stringify({ type: 'contextual_update', text: JSON.stringify({ sessionId: this.sessionId, mindedToken: getConfig().token! }) }));
+      this.addFirstMessageToState();
+    };
+
+    socket.onclose = () => {
+      logger.debug('Disconnected from ElevenLabs');
+      this.onDisconnectCallback?.();
+      this.agent.voiceSessions.delete(this.sessionId);
+    };
+
+    socket.onerror = (err: unknown) => {
+      logger.error('[ElevenLabsVoice] socket error', err);
+    };
+
+    socket.onmessage = async (event: any) => {
+      const data: ElevenLabsWebSocketEvent = JSON.parse(event.data.toString());
+
+      switch (data.type) {
+        case 'ping':
+          setTimeout(() => {
+            this.sendToElevenLabs(JSON.stringify({ type: 'pong', event_id: data.ping_event.event_id }));
+          }, data.ping_event.ping_ms);
+          break;
+        case 'user_transcript':
+          logger.debug('User transcript received:', data.user_transcription_event.user_transcript);
+          if (this.onMessageCallback) {
+            this.onMessageCallback(data.user_transcription_event.user_transcript, new HumanMessage({ content: data.user_transcription_event.user_transcript }));
+          }
+          break;
+        case 'agent_response':
+          if (this.onMessageCallback) {
+            this.onMessageCallback(data.agent_response_event.agent_response, new AIMessage({ content: data.agent_response_event.agent_response }));
+          }
+          break;
+        case 'interruption':
+          logger.debug('Interruption received');
+          this.onInterruptionCallback?.();
+          // Send interruption event to dashboard if connected
+          if (getConfig().dashboardConnected) {
+            try {
+              this.agent.mindedConnection?.emit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION, {
+                type: MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION,
+                sessionId: this.sessionId,
+                timestamp: Date.now(),
+              });
+            } catch (error) {
+              logger.error('[ElevenLabsVoice] Error sending interruption to dashboard', error);
+            }
+          }
+          break;
+        case 'audio':
+          if (this.onAudioCallback) {
+            this.onAudioCallback(data.audio_event.audio_base_64);
+          }
+          if (getConfig().dashboardConnected) {
+            try {
+              this.agent.mindedConnection?.emit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO, {
+                sessionId: this.sessionId,
+                audioData: data.audio_event.audio_base_64,
+                type: MindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO,
+                timestamp: Date.now(),
+              });
+            } catch (error) {
+              logger.error('[ElevenLabsVoice] Error sending audio to dashboard', error);
+            }
+          }
+          break;
+        case 'conversation_initiation_metadata':
+          logger.debug('ElevenLabs conversation initiation metadata', data);
+          break;
+        case 'agent_response_correction':
+          try {
+            logger.debug('Agent response correction received', data.agent_response_correction_event);
+            await this.updateAgentResponse(data.agent_response_correction_event.original_agent_response, data.agent_response_correction_event.corrected_agent_response);
+          } catch (error) {
+            logger.error('[ElevenLabsVoice] Error updating agent response', error);
+          }
+          break;
+        default:
+          logger.debug('Received unknown message from ElevenLabs', data);
+          break;
+      }
+    };
+  }
+
+  private async addFirstMessageToState(): Promise<void> {
+    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    if (graphState.values.messages.length === 0) {
+      logger.info('Adding first message to state', this.firstMessage);
+      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
+        messages: [new AIMessage({ content: this.firstMessage, id: uuidv4() })],
+      });
+    }
+  }
+
+  private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
+    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    const agentMessage: AIMessage | null = graphState.values.messages.find((message: BaseMessage) => message.content === originalAgentResponse && message instanceof AIMessage);
+    if (agentMessage) {
+      agentMessage.content = correctedAgentResponse;
+      agentMessage.additional_kwargs = agentMessage.additional_kwargs || {};
+      agentMessage.additional_kwargs.update = true;
+      const currentNodeId = graphState.tasks[graphState.tasks.length - 1].name;
+      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
+        messages: [agentMessage],
+      }, currentNodeId);
+    }
+    else {
+      logger.warn('Agent message not found for correction', originalAgentResponse);
+    }
+  }
+
+  public hangup(): void {
+    logger.info('Hanging up voice session', this.sessionId);
+    this.onDisconnectCallback?.();
+    this.elevenLabsSocket?.close();
+  }
+
+  /**
+   * Set callback for audio data in base64 format
+   */
+  onAudio(callback: (data: string) => void): void {
+    this.onAudioCallback = callback;
+  }
+
+  /**
+   * Set callback for interruption events
+   */
+  onInterruption(callback: () => void): void {
+    this.onInterruptionCallback = callback;
+  }
+
+  /**
+   * Set callback for disconnect events
+   */
+  onDisconnect(callback: () => void): void {
+    this.onDisconnectCallback = callback;
+  }
+
+  /**
+   * User audio in base64 format
+   */
+  sendAudio(audioData: string): void {
+    this.sendToElevenLabs(JSON.stringify({ user_audio_chunk: audioData }));
+  }
+
+  /** Fetch signed URL from ElevenLabs API */
+  private async getSignedUrl({ agentId, apiKey }: { agentId: string; apiKey: string }): Promise<string> {
+    const response = await fetch(
+      `https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${agentId}`,
+      {
+        method: 'GET',
+        headers: {
+          'xi-api-key': apiKey,
+        },
+      },
+    );
+    if (!response.ok) {
+      throw new Error(`Failed to fetch signed url - status ${response.status}`);
+    }
+    const body = await response.json();
+    return body.signed_url;
+  }
+}