@yrpri/api 9.0.101 → 9.0.103

package/dist/agents/assistants/voiceAssistant.js

@@ -0,0 +1,619 @@
+import { YpBaseChatBot } from "../../active-citizen/llms/baseChatBot.js";
+import WebSocket from "ws";
+// Extend the base class with voice capabilities
+export class YpBaseChatBotWithVoice extends YpBaseChatBot {
+    constructor(wsClientId, wsClients, redisKey, redisConnection, voiceEnabled = false, parentAssistant) {
+        super(wsClientId, wsClients, redisConnection, redisKey);
+        this.voiceEnabled = false;
+        this.VAD_TIMEOUT = 1000; // 1 second of silence for VAD
+        this.sendTranscriptsToClient = false;
+        this.isWaitingOnCancelResponseCompleted = false;
+        this.lastNumberOfChatHistoryForInstructions = 15;
+        this.DEBUG = false;
+        this.parentAssistant = parentAssistant;
+        this.voiceEnabled = voiceEnabled;
+        this.voiceState = {
+            speaking: false,
+            listening: false,
+            processingAudio: false,
+        };
+        // Default voice configuration
+        this.voiceConfig = {
+            model: process.env.OPENAI_VOICE_MODEL_NAME ||
+                "gpt-4o-realtime-preview-2024-12-17",
+            voice: "echo",
+            modalities: ["text", "audio"],
+        };
+        this.parentAssistant.on("update-ai-model-session", this.updateAiModelSession.bind(this));
+    }
+    async updateAiModelSession(message) {
+        console.log(`voiceAssistant: updateAiModelSession: ${message}`);
+        console.log(`--------------------> Logged in memory user: ${this.parentAssistant.memory.currentUser
+            ?.name}`);
+        await new Promise((resolve) => setTimeout(resolve, 150));
+        await this.initializeVoiceSession(message);
+    }
+    async initializeMainAssistantVoiceConnection() {
+        if (!this.voiceEnabled)
+            return;
+        console.log("initializeVoiceConnection");
+        const url = "wss://api.openai.com/v1/realtime";
+        const wsConfig = {
+            headers: {
+                Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+                "OpenAI-Beta": "realtime=v1",
+            },
+        };
+        try {
+            const ws = new WebSocket(`${url}?model=${this.voiceConfig.model}`, wsConfig);
+            ws.on("open", () => {
+                console.log("Voice connection established");
+                this.assistantVoiceConnection = {
+                    ws,
+                    connected: true,
+                    model: this.voiceConfig.model,
+                    voice: this.voiceConfig.voice,
+                };
+                this.initializeVoiceSession();
+            });
+            ws.on("close", () => {
+                console.log("Voice connection to OpenAI closed");
+                this.assistantVoiceConnection = undefined;
+            });
+            ws.on("error", (error) => {
+                console.error("Voice connection error:", error);
+                this.assistantVoiceConnection = undefined;
+            });
+            this.setupVoiceMessageHandlers(ws, true);
+        }
+        catch (error) {
+            console.error("Failed to initialize voice connection:", error);
+            throw error;
+        }
+    }
+    async initializeDirectAgentVoiceConnection() {
+        if (!this.voiceEnabled)
+            return;
+        //TODO: Wait because of Redis update or something figure it out
+        //await new Promise((resolve) => setTimeout(resolve, 150));
+        await this.sendCancelResponse();
+        const subscriptionPlan = await this.parentAssistant.getCurrentSubscriptionPlan();
+        if (!subscriptionPlan) {
+            throw new Error("No subscription plan found");
+        }
+        if (subscriptionPlan?.AgentProduct.name) {
+            this.exitMessageFromDirectAgentConversation = `Welcome the user back from their conversation with the ${subscriptionPlan.AgentProduct.name}. (it happened on a seperate channel). Now help the user with agent selection`;
+        }
+        this.sendToClient("assistant", "", "clear_audio_buffer");
+        if (this.directAgentVoiceConnection) {
+            console.log("Direct agent voice connection already initialized, closing");
+            this.destroyDirectAgentVoiceConnection();
+        }
+        console.log("initializeDirectAgentVoiceConnection");
+        const url = "wss://api.openai.com/v1/realtime";
+        const wsConfig = {
+            headers: {
+                Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+                "OpenAI-Beta": "realtime=v1",
+            },
+        };
+        try {
+            const ws = new WebSocket(`${url}?model=${this.voiceConfig.model}`, wsConfig);
+            ws.on("open", () => {
+                console.log("Agent voice connection established");
+                this.directAgentVoiceConnection = {
+                    ws,
+                    connected: true,
+                    model: this.voiceConfig.model,
+                    voice: this.voiceConfig.voice,
+                };
+                this.initializeVoiceSession("Say hi and welcome the user, offer to explain and show the user a workflow overview with the 'show_agent_workflow_overview_widget' function/tool you have access to call");
+            });
+            ws.on("close", () => {
+                console.log("Agent voice connection to OpenAI closed");
+                this.directAgentVoiceConnection = undefined;
+            });
+            ws.on("error", (error) => {
+                console.error("Agent voice connection error:", error);
+                this.directAgentVoiceConnection = undefined;
+            });
+            this.setupVoiceMessageHandlers(ws, false);
+        }
+        catch (error) {
+            console.error("Failed to initialize voice connection:", error);
+            throw error;
+        }
+    }
+    async destroyDirectAgentVoiceConnection() {
+        if (this.directAgentVoiceConnection) {
+            await this.parentAssistant.loadMemoryAsync();
+            this.directAgentVoiceConnection.ws.close();
+            this.directAgentVoiceConnection.connected = false;
+            this.directAgentVoiceConnection = undefined;
+            this.parentAssistant.memory.chatLog.push({
+                sender: "user",
+                message: "Thank you for the information, I would now like to speak to the main assistant about selecting agents",
+            });
+            await this.parentAssistant.saveMemory();
+        }
+    }
+    destroyAssistantVoiceConnection() {
+        if (this.vadTimeout) {
+            clearTimeout(this.vadTimeout);
+            this.vadTimeout = undefined;
+        }
+        // Reset voice state
+        this.voiceState = {
+            speaking: false,
+            listening: false,
+            processingAudio: false,
+        };
+        // Close WebSocket connection
+        if (this.assistantVoiceConnection?.ws) {
+            this.assistantVoiceConnection.ws.close();
+            this.assistantVoiceConnection.connected = false;
+            this.assistantVoiceConnection = undefined;
+        }
+    }
+    setupVoiceMessageHandlers(ws, disableWhenAgentIsSpeaking) {
+        // Instead of an inline callback, keep a reference
+        const handler = async (data) => {
+            try {
+                const event = JSON.parse(data.toString());
+                if (disableWhenAgentIsSpeaking && this.directAgentVoiceConnection) {
+                    console.log("Voice message received but agent is speaking, ignoring", event.type);
+                    return;
+                }
+                if (this.DEBUG) {
+                    console.log("voiceMessage: ", event.type);
+                }
+                switch (event.type) {
+                    case "session.created":
+                        await this.handleVoiceSessionCreated(event);
+                        break;
+                    case "session.error":
+                        await this.handleVoiceSessionError(event);
+                        break;
+                    case "response.generating.started":
+                        this.voiceState.speaking = true;
+                        await this.handleVoiceResponseStatus(event);
+                        break;
+                    case "response.generating.completed":
+                        this.voiceState.speaking = false;
+                        await this.handleVoiceResponseStatus(event);
+                        break;
+                    case "input_audio_buffer.speech_started":
+                        this.handleSpeechStarted();
+                        break;
+                    case "input_audio_buffer.speech_stopped":
+                        await this.handleSpeechStopped();
+                        break;
+                    case "response.cancelled":
+                        console.log("response.cancelled");
+                        this.isWaitingOnCancelResponseCompleted = false;
+                        break;
+                    case "conversation.item.input_audio_transcription.completed":
+                        await this.handleAudioTranscriptDone(event);
+                        break;
+                    case "text":
+                        await this.handleTextOutput(event);
+                        break;
+                    case "input_audio_buffer.committed":
+                        await this.handleAudioBufferCommitted(event);
+                        break;
+                    case "response.audio.delta":
+                        await this.handleAudioDelta(event);
+                        break;
+                    case "response.output_item.done":
+                        await this.handleResponseDone(event);
+                        break;
+                    case "response.function_call_arguments.done":
+                        await this.callFunctionHandler(event);
+                        break;
+                    default:
+                        if (this.DEBUG) {
+                            console.log("Unhandled voice event type:", event.type);
+                        }
+                        if (event.type === "error") {
+                            console.log(JSON.stringify(event, null, 2));
+                        }
+                        break;
+                }
+            }
+            catch (error) {
+                console.error("Error handling voice message:", error);
+            }
+        };
+        // Attach it
+        ws.on("message", handler);
+        // Store the reference so we can remove it in destroy()
+        if (disableWhenAgentIsSpeaking) {
+            this.voiceMainMessageHandler = handler;
+        }
+        else {
+            this.voiceDirectMessageHandler = handler;
+        }
+    }
+    destroy() {
+        // 1) Remove voice message handlers from assistantVoiceConnection
+        if (this.assistantVoiceConnection?.ws && this.voiceMainMessageHandler) {
+            this.assistantVoiceConnection.ws.removeListener("message", this.voiceMainMessageHandler);
+            this.voiceMainMessageHandler = undefined;
+        }
+        // 2) Remove voice message handlers from directAgentVoiceConnection
+        if (this.directAgentVoiceConnection?.ws && this.voiceDirectMessageHandler) {
+            this.directAgentVoiceConnection.ws.removeListener("message", this.voiceDirectMessageHandler);
+            this.voiceDirectMessageHandler = undefined;
+        }
+        // 3) Close any open connections
+        this.destroyAssistantVoiceConnection();
+        this.destroyDirectAgentVoiceConnection();
+        // 4) Call super destroy so it can remove *its* WebSocket listeners, etc.
+        super.destroy();
+    }
+    async handleResponseDone(event) {
+        if (event.item?.content && event.item.content.length > 0) {
+            if (this.sendTranscriptsToClient) {
+                this.sendToClient("assistant", event.item.content[0].transcript, "message");
+            }
+            this.parentAssistant.addAssistantMessage(event.item.content[0].transcript);
+        }
+        else {
+            console.error("No text in response.done event");
+        }
+    }
+    async handleAudioTranscriptDone(event) {
+        if (event.transcript) {
+            if (this.sendTranscriptsToClient) {
+                this.sendToClient("user", event.transcript, "message");
+            }
+            this.parentAssistant.addUserMessage(event.transcript);
+        }
+    }
+    //TODO: Don't have this duplicated in the voice assistant and main assistant
+    async callFunctionHandler(event) {
+        const tools = this.parentAssistant.getCurrentModeFunctions();
+        if (!tools)
+            return;
+        //console.log("callFunctionHandler event: ", JSON.stringify(event, null, 2));
+        const tool = tools.find((t) => t.name === event.name);
+        if (!tool) {
+            console.log("Tool not found: ", event.name);
+            return;
+        }
+        const toolName = tool.name;
+        console.log("Calling tool handler: ", toolName);
+        const result = await tool.handler(event.arguments);
+        // Store the result in memory for context
+        if (result.success && result.data) {
+            await this.parentAssistant.setModeData(`${toolName}_result`, result.data);
+        }
+        let resultData = result.data || result.error;
+        // Generate a user-friendly message based on the tool result
+        const resultMessage = `<contextFromRetrievedData>${JSON.stringify(resultData, null, 2)}</contextFromRetrievedData>`;
+        if (resultData) {
+            this.sendToClient("assistant", resultMessage, "hiddenContextMessage", result.uniqueToken, true);
+            if (toolName === "switch_mode") {
+                const createResponse = {
+                    type: "response.create",
+                    event_id: `switchMode_${this.getRandomStringAscii(10)}`,
+                    response: {
+                        modalities: this.voiceConfig.modalities,
+                        instructions: "Inform the user about the mode change",
+                        tools: this.parentAssistant.getCurrentModeFunctions(),
+                    },
+                };
+                this.assistantVoiceConnection?.ws.send(JSON.stringify(createResponse));
+            }
+            /*this.parentAssistant.memory.chatLog!.push({
+              sender: "assistant",
+              hiddenContextMessage: true,
+              message: resultMessage,
+            });
+            await this.saveMemoryIfNeeded();*/
+        }
+        else {
+            console.error(`No data returned from tool execution: ${event.name}`);
+        }
+        if (result.html) {
+            console.log("--------------------------------============================>  handleResponseDone result.html with token: ", result.uniqueToken);
+            this.sendToClient("assistant", result.html, "html", result.uniqueToken, true);
+            this.parentAssistant.addAssistantHtmlMessage(result.html, result.uniqueToken);
+        }
+        if (result.clientEvents) {
+            console.log(`clientEvents: ${JSON.stringify(result.clientEvents, null, 2)}`);
+            for (const clientEvent of result.clientEvents) {
+                this.sendToClient("assistant", clientEvent.details, clientEvent.name);
+            }
+        }
+        if (!result.success) {
+            console.error(`Tool execution failed: ${event.name} ${result.error}`);
+        }
+        const responseEvent = {
+            type: "conversation.item.create",
+            event_id: `callFunctionHandler_${this.getRandomStringAscii(10)}`,
+            item: {
+                type: "function_call_output",
+                call_id: event.call_id,
+                output: JSON.stringify(resultData, null, 2),
+            },
+        };
+        this.sendToVoiceConnection(responseEvent);
+        if (result.error) {
+            const createResponse = {
+                type: "response.create",
+                event_id: `callFunctionHandlerError_${this.getRandomStringAscii(10)}`,
+                response: {
+                    modalities: this.voiceConfig.modalities,
+                    instructions: "Inform the user about the tool execution error",
+                    tools: this.parentAssistant.getCurrentModeFunctions(),
+                },
+            };
+            this.sendToVoiceConnection(createResponse);
+        }
+    }
+    getRandomStringAscii(length = 10) {
+        return Array.from({ length }, () => Math.floor(Math.random() * 128))
+            .map((n) => String.fromCharCode(n))
+            .join("");
+    }
+    async proxyToClient(event) {
+        console.log("proxyToClient: ", event.type);
+        console.log(JSON.stringify(event, null, 2));
+        const proxyMessage = {
+            sender: "assistant",
+            type: event.type,
+            data: event,
+        };
+        this.wsClientSocket.send(JSON.stringify(proxyMessage));
+    }
+    // Handle incoming audio from client
+    async handleIncomingAudio(audioData) {
+        //console.log(`handleIncomingAudio: ${audioData.length} ${this.voiceEnabled} ${this.assistantVoiceConnection?.ws}`);
+        if (!this.assistantVoiceConnection?.ws || !this.voiceEnabled)
+            return;
+        this.voiceState.lastAudioTimestamp = Date.now();
+        // Reset VAD timeout
+        if (this.vadTimeout) {
+            clearTimeout(this.vadTimeout);
+        }
+        // Set new VAD timeout
+        this.vadTimeout = setTimeout(() => {
+            this.handleVADSilence();
+        }, this.VAD_TIMEOUT);
+        // Send audio to server
+        const audioMessage = {
+            type: "input_audio_buffer.append",
+            event_id: `bufferAppend_${this.getRandomStringAscii(10)}`,
+            audio: Buffer.from(audioData).toString("base64"),
+        };
+        //console.log("Sending audio message to server:");
+        this.sendToVoiceConnection(audioMessage);
+    }
+    // Handle Voice Activity Detection silence
+    async handleVADSilence() {
+        if (!this.assistantVoiceConnection?.ws || !this.voiceState.listening)
+            return;
+        // Commit the audio buffer
+        const commitMessage = {
+            type: "input_audio_buffer.commit",
+            event_id: `handleVADSilence_${this.getRandomStringAscii(10)}`,
+        };
+        this.sendToVoiceConnection(commitMessage);
+        this.voiceState.processingAudio = true;
+    }
+    // Handle speech started event
+    handleSpeechStarted() {
+        this.voiceState.listening = true;
+        this.sendToClient("assistant", "", "listening_start");
+    }
+    // Handle speech stopped event
+    async handleSpeechStopped() {
+        this.voiceState.listening = false;
+        this.sendToClient("assistant", "", "listening_stop");
+        // If we're not already processing, commit the buffer
+        if (!this.voiceState.processingAudio) {
+            await this.handleVADSilence();
+        }
+    }
+    async handleAudioDelta(event) {
+        if (!event.delta)
+            return;
+        const audioMessage = {
+            sender: "assistant",
+            type: "audio",
+            base64Audio: event.delta, // base64 encoded audio
+        };
+        this.wsClientSocket.send(JSON.stringify(audioMessage));
+    }
+    // Handle text output from server
+    async handleTextOutput(event) {
+        if (!event.content)
+            return;
+        // Add to chat log
+        this.parentAssistant.memory.chatLog.push({
+            sender: "assistant",
+            message: event.content,
+        });
+        // Send to client
+        this.sendToClient("assistant", event.content);
+    }
+    sendToVoiceConnection(message) {
+        if (this.directAgentVoiceConnection &&
+            this.directAgentVoiceConnection.connected) {
+            this.directAgentVoiceConnection.ws.send(JSON.stringify(message));
+        }
+        else {
+            this.assistantVoiceConnection?.ws.send(JSON.stringify(message));
+        }
+    }
+    // Handle audio buffer committed event
+    async handleAudioBufferCommitted(event) {
+        this.voiceState.processingAudio = false;
+    }
+    async waitForCancelResponseCompleted() {
+        this.isWaitingOnCancelResponseCompleted = true;
+        await Promise.race([
+            new Promise((resolve) => {
+                const checkFlag = () => {
+                    if (!this.isWaitingOnCancelResponseCompleted) {
+                        console.log("Cancel response completed from event");
+                        resolve(true);
+                    }
+                    else {
+                        setTimeout(checkFlag, 10); // Check every 10ms
+                    }
+                };
+                checkFlag();
+            }),
+            new Promise((resolve) => setTimeout(resolve, 75)),
+        ]);
+        console.log("Cancel response completed from timeout");
+    }
+    async sendCancelResponse() {
+        const cancelResponse = {
+            type: "response.cancel",
+            event_id: `sendCancelResponse_${this.getRandomStringAscii(10)}`,
+        };
+        this.sendToVoiceConnection(cancelResponse);
+        console.log("Have sent cancel response");
+        await this.waitForCancelResponseCompleted();
+    }
+    async initializeVoiceSession(customResponseMessage) {
+        if (!this.assistantVoiceConnection?.ws) {
+            console.error("No voice connection");
+            return;
+        }
+        await this.parentAssistant.initializeModes();
+        console.log("======================> initializeVoiceSession current mode", this.parentAssistant.memory.currentMode);
+        console.log("======================> initializeVoiceSession system prompt", this.parentAssistant.getCurrentSystemPrompt());
+        console.log("======================> initializeVoiceSession functions", this.parentAssistant.getCurrentModeFunctions());
+        let chatHistory;
+        if (this.parentAssistant.memory &&
+            this.parentAssistant.memory.chatLog &&
+            this.parentAssistant.memory.chatLog.length > 0) {
+            chatHistory = JSON.stringify(this.parentAssistant.memory.chatLog
+                .filter((message) => message.message != "")
+                .slice(-this.lastNumberOfChatHistoryForInstructions)
+                .map((message) => ({
+                role: message.sender,
+                content: message.message,
+            })));
+        }
+        let instructions = this.parentAssistant.getCurrentSystemPrompt() || "";
+        //console.log("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< initializeVoiceSession current mode", JSON.stringify(this.parentAssistant.memory, null, 2));
+        let voiceName = this.voiceConfig.voice;
+        const subscriptionPlan = await this.parentAssistant.getCurrentSubscriptionPlan();
+        if (this.parentAssistant.memory.currentMode ===
+            "agent_direct_connection_mode" &&
+            subscriptionPlan?.AgentProduct.configuration.avatar?.voiceName) {
+            voiceName = subscriptionPlan.AgentProduct.configuration.avatar.voiceName;
+            instructions = `${subscriptionPlan.AgentProduct.configuration.avatar.systemPrompt}\n${instructions}`;
+            this.parentAssistant.sendAvatarUrlChange(subscriptionPlan.AgentProduct.configuration.avatar.imageUrl, subscriptionPlan.AgentProduct.name);
+        }
+        else {
+            this.parentAssistant.sendAvatarUrlChange(null, null);
+        }
+        if (chatHistory) {
+            instructions += `\n\n<ImportantPreviousChatHistory>\n${chatHistory}\n</ImportantPreviousChatHistory>`;
+        }
+        if (this.DEBUG) {
+            console.log("initializeVoiceSession final instructions", instructions);
+        }
+        // Then update the session with full configuration
+        const sessionConfig = {
+            type: "session.update",
+            event_id: `initializeVoiceSession_${this.getRandomStringAscii(10)}`,
+            session: {
+                ...this.voiceConfig,
+                voice: voiceName,
+                instructions: instructions,
+                //@ts-ignore
+                tools: this.parentAssistant.getCurrentModeFunctions(),
+                tool_choice: "auto",
+                temperature: 0.72,
+                input_audio_transcription: {
+                    model: "whisper-1",
+                },
+                turn_detection: {
+                    type: "server_vad",
+                    threshold: 0.5,
+                    prefix_padding_ms: 300,
+                    silence_duration_ms: 500,
+                },
+            },
+        };
+        if (this.DEBUG) {
+            console.log("Sending session config to server:", JSON.stringify(sessionConfig, null, 2));
+        }
+        this.sendToVoiceConnection(sessionConfig);
+        setTimeout(() => {
+            if (this.exitMessageFromDirectAgentConversation &&
+                !this.directAgentVoiceConnection) {
+                this.sendCancelResponse();
+                setTimeout(() => {
+                    if (this.exitMessageFromDirectAgentConversation) {
+                        this.triggerResponse(this.exitMessageFromDirectAgentConversation, true);
+                        this.exitMessageFromDirectAgentConversation = undefined;
+                    }
+                }, 250);
+            }
+            else if (!customResponseMessage) {
+                this.triggerResponse("Say hi and offer to show the agents", false);
+            }
+            else {
+                this.triggerResponse(customResponseMessage, false);
+            }
+        }, 200);
+    }
+    async triggerResponse(message, cancelResponse = true) {
+        console.log("triggerResponse: ", message);
+        const createResponse = {
+            type: "response.create",
+            event_id: `triggerResponse_${this.getRandomStringAscii(10)}`,
+            response: {
+                modalities: this.voiceConfig.modalities,
+                instructions: message,
+            },
+        };
+        if (cancelResponse) {
+            await this.sendCancelResponse();
+        }
+        this.sendToVoiceConnection(createResponse);
+    }
+    // Handle voice-specific events
+    async handleVoiceSessionCreated(event) {
+        console.log("Voice session created:", event.session.id);
+        // Additional session initialization if needed
+    }
+    async handleVoiceSessionError(event) {
+        console.error("Voice session error:", event.error);
+        this.sendToClient("system", "Voice processing error occurred", "error");
+    }
+    async handleVoiceResponseStatus(event) {
+        switch (event.type) {
+            case "response.generating.started":
+                this.sendToClient("assistant", "", "start");
+                break;
+            case "response.generating.completed":
+                this.sendToClient("assistant", "", "end");
+                break;
+        }
+    }
+    // Override the streamWebSocketResponses method to handle both modes
+    async streamWebSocketResponses(stream) {
+        if (this.voiceEnabled) {
+            // For voice mode, responses are handled through the WebSocket event handlers
+            return;
+        }
+        // Otherwise, use the original implementation
+        return super.streamWebSocketResponses(stream);
+    }
+    // Method to update voice configuration
+    async updateVoiceConfig(config) {
+        this.voiceConfig = {
+            ...this.voiceConfig,
+            ...config,
+        };
+    }
+}