@minded-ai/mindedjs 1.0.53-alpha-latest → 1.0.53-patch-alpha

package/src/voice/elevenLabsUtils.ts

@@ -0,0 +1,101 @@
+/** ElevenLabs LLM proxy request body shape */
+export interface LLMRequestBody {
+  messages: Array<{
+    content: string;
+    role: string;
+    tool_call_id?: string;
+  }>;
+  model: string;
+  max_tokens: number;
+  stream: boolean;
+  temperature: number;
+  tools: null;
+}
+
+// ----------------- ElevenLabs WebSocket event types -----------------
+
+interface BaseEvent {
+  type: string;
+}
+
+interface UserTranscriptEvent extends BaseEvent {
+  type: 'user_transcript';
+  user_transcription_event: {
+    user_transcript: string;
+  };
+}
+
+interface AgentResponseEvent extends BaseEvent {
+  type: 'agent_response';
+  agent_response_event: {
+    agent_response: string;
+  };
+}
+
+interface AudioResponseEvent extends BaseEvent {
+  type: 'audio';
+  audio_event: {
+    audio_base_64: string;
+    event_id: number;
+  };
+}
+
+interface InterruptionEvent extends BaseEvent {
+  type: 'interruption';
+  interruption_event: {
+    event_id: number;
+  };
+}
+
+interface PingEvent extends BaseEvent {
+  type: 'ping';
+  ping_event: {
+    event_id: number;
+    ping_ms: number;
+  };
+}
+
+interface ConversationInitiationMetadataEvent extends BaseEvent {
+  type: 'conversation_initiation_metadata';
+  conversation_initiation_metadata_event: {
+    conversation_id: string;
+  };
+}
+
+export interface AgentResponseCorrectionEvent extends BaseEvent {
+  type: 'agent_response_correction';
+  agent_response_correction_event: {
+    corrected_agent_response: string;
+    original_agent_response: string;
+  };
+}
+
+/** Union of all possible WebSocket events we care about */
+export type ElevenLabsWebSocketEvent =
+  | UserTranscriptEvent
+  | AgentResponseEvent
+  | AudioResponseEvent
+  | InterruptionEvent
+  | PingEvent
+  | ConversationInitiationMetadataEvent
+  | AgentResponseCorrectionEvent;
+
+// ----------------- ElevenLabs Utility Functions -----------------
+
+/** Fetch signed URL from ElevenLabs API */
+export const getSignedUrl = async ({ agentId, apiKey }: { agentId: string; apiKey: string }): Promise<string> => {
+  const response = await fetch(
+    `https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${agentId}`,
+    {
+      method: 'GET',
+      headers: {
+        'xi-api-key': apiKey,
+      },
+    },
+  );
+  if (!response.ok) {
+    throw new Error(`Failed to fetch signed url - status ${response.status}`);
+  }
+  const body = await response.json();
+  return body.signed_url;
+};

package/src/voice/voiceSession.ts

@@ -0,0 +1,251 @@
+import { WebSocket } from 'ws';
+import { Agent } from '../agent';
+import {
+  MindedConnectionSocketMessageType,
+} from '../platform/mindedConnectionTypes';
+import { getConfig } from '../platform/config';
+import {
+  ElevenLabsWebSocketEvent,
+  getSignedUrl,
+} from './elevenLabsUtils';
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages';
+
+/**
+ * Voice Conversation class for managing individual ElevenLabs voice conversations
+ */
+export class VoiceSession {
+  private agent: Agent;
+  private sessionId: string;
+  private firstMessage: string;
+  private voiceId?: string;
+  private elevenLabsSocket?: WebSocket;
+
+  private onAudioCallback?: (data: string) => void;
+  private onInterruptionCallback?: () => void;
+  private onMessageCallback?: (text: string, message: BaseMessage) => void;
+  private onDisconnectCallback?: () => void;
+
+  constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
+    console.debug('Starting voice session', { sessionId, firstMessage, voiceId });
+    this.agent = agent;
+    this.sessionId = sessionId;
+    this.firstMessage = firstMessage;
+    this.voiceId = voiceId;
+  }
+
+  /**
+   * Initialize the voice conversation connection
+   */
+  public async init(): Promise<void> {
+    const { elevenLabsKey, elevenLabsAgentId } = getConfig();
+
+    if (!elevenLabsKey) {
+      throw new Error('Missing ElevenLabs key - set ELEVEN_LABS_KEY env var');
+    }
+
+    if (!elevenLabsAgentId) {
+      throw new Error('Missing ElevenLabs agent id - set ELEVEN_LABS_AGENT_ID env var');
+    }
+
+    const signedUrl = await getSignedUrl({
+      agentId: elevenLabsAgentId,
+      apiKey: elevenLabsKey
+    });
+
+    this.elevenLabsSocket = new WebSocket(signedUrl);
+    this.setupSocketHandlers();
+  }
+
+  private async sendToElevenLabs(message: string): Promise<void> {
+    if (!this.elevenLabsSocket) {
+      throw new Error('Socket not initialized');
+    }
+    await this.waitForSocketOpen(this.elevenLabsSocket);
+    this.elevenLabsSocket.send(message);
+  }
+
+  private async waitForSocketOpen(socket: WebSocket): Promise<void> {
+    if (socket.readyState === WebSocket.OPEN) {
+      return;
+    }
+    return new Promise((resolve, reject) => {
+      const timeoutMs = 10000;
+      const intervalMs = 100;
+      let tries = 0;
+      const interval = setInterval(() => {
+        if (socket.readyState === WebSocket.OPEN) {
+          clearInterval(interval);
+          resolve();
+        }
+        tries++;
+        if (tries >= timeoutMs) {
+          clearInterval(interval);
+          reject(new Error('Socket not open'));
+        }
+      }, intervalMs);
+    });
+  }
+
+  private setupSocketHandlers(): void {
+    const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
+
+    socket.onopen = () => {
+      console.debug('Connected to voice provider');
+      const initiationData: ConversationInitiationClientData = {
+        type: 'conversation_initiation_client_data',
+        conversation_config_override: {
+          agent: {
+            first_message: this.firstMessage,
+            language: 'en',
+          },
+          ...(this.voiceId ? { tts: { voice_id: this.voiceId } } : {}),
+        } as ConversationInitiationClientData['conversation_config_override'],
+      };
+
+      this.sendToElevenLabs(JSON.stringify(initiationData));
+      this.sendToElevenLabs(JSON.stringify({ type: 'contextual_update', text: JSON.stringify({ sessionId: this.sessionId, mindedToken: getConfig().token! }) }));
+    };
+
+    socket.onclose = () => {
+      console.debug('Disconnected from voice provider');
+      this.onDisconnectCallback?.();
+      this.agent.voiceSessions.delete(this.sessionId);
+    };
+
+    socket.onerror = (err: unknown) => {
+      console.error('[ElevenLabsVoice] WebSocket error', err);
+    };
+
+    socket.onmessage = async (event: any) => {
+      const data: ElevenLabsWebSocketEvent = JSON.parse(event.data.toString());
+
+      switch (data.type) {
+        case 'ping':
+          setTimeout(() => {
+            this.sendToElevenLabs(JSON.stringify({ type: 'pong', event_id: data.ping_event.event_id }));
+          }, data.ping_event.ping_ms);
+          break;
+        case 'user_transcript':
+          console.debug('User transcript received', data.user_transcription_event.user_transcript);
+          if (this.onMessageCallback) {
+            this.onMessageCallback(data.user_transcription_event.user_transcript, new HumanMessage(data.user_transcription_event.user_transcript));
+          }
+          break;
+        case 'agent_response':
+          if (this.onMessageCallback) {
+            this.onMessageCallback(data.agent_response_event.agent_response, new AIMessage(data.agent_response_event.agent_response));
+          }
+          break;
+        case 'interruption':
+          console.debug('Interruption received');
+          this.onInterruptionCallback?.();
+          // Send interruption event to dashboard if connected
+          if (getConfig().dashboardConnected) {
+            try {
+              this.agent.mindedConnection?.emit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION, {
+                type: MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION,
+                sessionId: this.sessionId,
+                timestamp: Date.now(),
+              });
+            } catch (error) {
+              console.error('[ElevenLabsVoice] Error sending interruption to dashboard', error);
+            }
+          }
+          break;
+        case 'audio':
+          if (this.onAudioCallback) {
+            this.onAudioCallback(data.audio_event.audio_base_64);
+          }
+          if (getConfig().dashboardConnected) {
+            try {
+              await this.agent.mindedConnection?.awaitEmit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO, {
+                sessionId: this.sessionId,
+                audioData: data.audio_event.audio_base_64,
+              });
+            } catch (error) {
+              console.error('[ElevenLabsVoice] Error sending audio to dashboard', error);
+            }
+          }
+          break;
+        case 'conversation_initiation_metadata':
+          console.debug('ElevenLabs conversation initiation metadata', data);
+          break;
+        case 'agent_response_correction':
+          try {
+            console.debug('Agent response correction received', data.agent_response_correction_event);
+            await this.updateAgentResponse(data.agent_response_correction_event.original_agent_response, data.agent_response_correction_event.corrected_agent_response);
+          } catch (error) {
+            console.error('[ElevenLabsVoice] Error updating agent response', error);
+          }
+          break;
+        default:
+          console.debug('Received unknown message from ElevenLabs', data);
+          break;
+      }
+    };
+  }
+
+  private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
+    const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    const agentMessage = graphState.values.messages.find((message: BaseMessage) => message.content === originalAgentResponse && message instanceof AIMessage);
+    if (agentMessage) {
+      agentMessage.content = correctedAgentResponse;
+      const currentNodeId = graphState.tasks[graphState.tasks.length - 1].name;
+      await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
+        messages: graphState.values.messages,
+      }, currentNodeId);
+    }
+    else {
+      console.warn('Agent message not found for correction', originalAgentResponse);
+    }
+  }
+
+  public hangup(): void {
+    this.onDisconnectCallback?.();
+    this.elevenLabsSocket?.close();
+  }
+
+  /**
+   * Set callback for audio data in base64 format
+   */
+  onAudio(callback: (data: string) => void): void {
+    this.onAudioCallback = callback;
+  }
+
+  /**
+   * Set callback for interruption events
+   */
+  onInterruption(callback: () => void): void {
+    this.onInterruptionCallback = callback;
+  }
+
+  /**
+   * Set callback for disconnect events
+   */
+  onDisconnect(callback: () => void): void {
+    this.onDisconnectCallback = callback;
+  }
+
+  /**
+   * User audio in base64 format
+   */
+  sendAudio(audioData: string): void {
+    this.sendToElevenLabs(JSON.stringify({ user_audio_chunk: audioData }));
+  }
+}
+
+// ----------------- Helper Types -----------------
+
+/** Shape of the message sent to ElevenLabs when starting a conversation */
+interface ConversationInitiationClientData {
+  type: 'conversation_initiation_client_data';
+  conversation_config_override: {
+    agent: {
+      first_message: string;
+      language: string;
+    };
+    tts?: {
+      voice_id: string;
+    };
+  };
+}