@yrpri/api 9.0.101 → 9.0.103

package/dist/agents/assistants/baseAssistantWithVoice.js

@@ -0,0 +1,150 @@
+import { YpBaseAssistant } from "./baseAssistant.js";
+import { YpBaseChatBotWithVoice } from "./voiceAssistant.js";
+export class YpBaseAssistantWithVoice extends YpBaseAssistant {
+    constructor(wsClientId, wsClients, redis, voiceEnabled, redisKey, domainId) {
+        super(wsClientId, wsClients, redis, redisKey, domainId);
+        if (!domainId) {
+            throw new Error("Domain ID is required");
+        }
+        this.voiceEnabled = voiceEnabled;
+    }
+    destroy() {
+        // 2) Remove the WebSocket listener we set up in createVoiceBot()
+        const ws = this.wsClients.get(this.wsClientId);
+        if (ws && this.mainBotWsHandler) {
+            ws.removeListener("message", this.mainBotWsHandler);
+            this.mainBotWsHandler = undefined;
+        }
+        // 3) Remove the forward event handler on the voiceBot socket
+        if (this.voiceBot?.wsClientSocket && this.forwardEventHandler) {
+            this.voiceBot.wsClientSocket.removeListener("message", this.forwardEventHandler);
+            this.forwardEventHandler = undefined;
+        }
+        if (this.voiceBot) {
+            this.voiceBot.destroy();
+            this.voiceBot = undefined;
+        }
+        // 4) Finally call super.destroy()
+        super.destroy();
+    }
+    async initialize() {
+        await this.initializeModes();
+        if (this.voiceEnabled) {
+            this.createVoiceBot();
+        }
+    }
+    async createVoiceBot() {
+        this.voiceBot = new YpBaseChatBotWithVoice(this.wsClientId, this.wsClients, this.redisKey, this.redis, true, this);
+        await this.voiceBot.initializeMainAssistantVoiceConnection();
+        this.setupVoiceEventForwarder();
+        await this.voiceBot.updateVoiceConfig({
+            instructions: this.getCurrentSystemPrompt(),
+            tools: this.getCurrentModeFunctions(),
+            modalities: ["text", "audio"],
+        });
+        const ws = this.wsClients.get(this.wsClientId);
+        if (ws) {
+            this.mainBotWsHandler = async (data) => {
+                try {
+                    const message = JSON.parse(data.toString());
+                    switch (message.type) {
+                        case "voice_mode":
+                            await this.setVoiceMode(message.enabled);
+                            break;
+                        case "voice_input":
+                            if (this.voiceEnabled && message.audio) {
+                                await this.voiceBot?.handleIncomingAudio(Buffer.from(message.audio, "base64"));
+                            }
+                            break;
+                    }
+                }
+                catch (error) {
+                    console.error("Error processing message:", error);
+                }
+            };
+            ws.on("message", this.mainBotWsHandler);
+        }
+        else {
+            console.error("No WebSocket found for client: ", this.wsClientId);
+        }
+    }
+    destroyVoiceBot() {
+        if (this.voiceBot?.wsClientSocket && this.forwardEventHandler) {
+            this.voiceBot.wsClientSocket.removeListener("message", this.forwardEventHandler);
+            this.forwardEventHandler = undefined;
+        }
+        if (this.voiceBot) {
+            this.voiceBot.destroy();
+            this.voiceBot = undefined;
+        }
+    }
+    setupVoiceEventForwarder() {
+        // Forward all voice bot events to client
+        this.forwardEventHandler = (data) => {
+            try {
+                const event = JSON.parse(data.toString());
+                // Forward all voice-related events to client
+                if ([
+                    "session.created",
+                    "session.error",
+                    "response.generating.started",
+                    "response.generating.completed",
+                    "speech.started",
+                    "speech.stopped",
+                    "audio",
+                    "text",
+                    "input_audio_buffer.committed",
+                ].includes(event.type)) {
+                    this.wsClientSocket.send(data);
+                    return;
+                }
+                // Handle message events through parent class
+                this.sendToClient(event.sender || "bot", event.message || "", event.type);
+            }
+            catch (error) {
+                console.error("Error forwarding voice event:", error);
+            }
+        };
+        this.voiceBot?.wsClientSocket?.on("message", this.forwardEventHandler);
+    }
+    async setVoiceMode(enabled) {
+        this.voiceEnabled = enabled;
+        if (!this.voiceEnabled) {
+            this.destroyVoiceBot();
+        }
+        this.wsClientSocket.send(JSON.stringify({
+            type: "voice_mode_changed",
+            enabled: this.voiceEnabled,
+        }));
+    }
+    async handleModeSwitch(newMode, reason, params) {
+        await super.handleModeSwitch(newMode, reason, params);
+        if (this.voiceEnabled) {
+            await this.voiceBot?.updateVoiceConfig({
+                instructions: this.getCurrentSystemPrompt(),
+                tools: this.getCurrentModeFunctions(),
+                modalities: ["text", "audio"]
+            });
+            console.log("handleModeSwitch", newMode, params);
+            if (newMode === "agent_direct_connection_mode") {
+                this.voiceBot?.initializeDirectAgentVoiceConnection();
+            }
+            else {
+                await this.voiceBot?.destroyDirectAgentVoiceConnection();
+                this.voiceBot?.initializeVoiceSession();
+            }
+        }
+    }
+    async conversation(chatLog) {
+        if (this.voiceEnabled) {
+            console.log("voiceEnabled: Updating voice config");
+            await this.voiceBot?.updateVoiceConfig({
+                instructions: this.getCurrentSystemPrompt(),
+                tools: this.getCurrentModeFunctions(),
+                modalities: ["text", "audio"],
+            });
+            return this.voiceBot?.conversation(chatLog);
+        }
+        return super.conversation(chatLog);
+    }
+}

package/dist/agents/assistants/modes/agentDirectConnection.js

@@ -0,0 +1,84 @@
+// DirectConversationMode.ts
+import { BaseAssistantMode } from "./baseAssistantMode.js";
+import { SubscriptionTools } from "./tools/subscriptionTools.js";
+import { NavigationTools } from "./tools/navigationTools.js";
+import { LoginAssistantTools } from "./tools/loginTools.js";
+import { AgentTools } from "./tools/agentTools.js";
+export class DirectConversationMode extends BaseAssistantMode {
+    constructor(assistant) {
+        super(assistant);
+        this.subscriptionTools = new SubscriptionTools(this.assistant);
+        this.navigationTools = new NavigationTools(this.assistant);
+        this.loginTools = new LoginAssistantTools(this.assistant);
+        this.agentTools = new AgentTools(this.assistant);
+    }
+    async getCurrentModeSystemPrompt() {
+        let systemPrompt = ``; // Is filled in dynamically
+        systemPrompt += await this.renderCommon();
+        return systemPrompt;
+    }
+    async getCurrentModeTools() {
+        const tools = [];
+        try {
+            if (this.assistant.isLoggedIn) {
+                // User logged in
+                console.log("Mode: agent_direct_connection_mode, User logged in");
+                tools.push(this.loginTools.logout);
+                tools.push(this.agentTools.showAgentWorkflowOverviewWidget);
+                if (this.assistant.isSubscribedToCurrentAgentProduct) {
+                    if (this.assistant.hasConfiguredcurrentAgentProduct) {
+                        // User has configured the current agent
+                        console.log("Mode: agent_direct_connection_mode, User has configured the current agent");
+                        if (await this.assistant.isCurrentAgentRunning()) {
+                            tools.push(this.agentTools.stopCurrentAgentWorkflow);
+                            tools.push(this.agentTools.deactivateAgent);
+                        }
+                        else {
+                            if (!(await this.assistant.isCurrentAgentActive())) {
+                                tools.push(this.agentTools.createNewAgentRunReadyToRunFirstWorkflowStep);
+                            }
+                            else {
+                                const nextWorkflowStep = await this.agentTools.agentModels.getNextWorkflowStep();
+                                if (nextWorkflowStep) {
+                                    tools.push(await this.agentTools.startCurrentRunAgentNextWorkflowStep());
+                                }
+                                //TODO: Do we need this if the user looses the widget in their chat history?
+                                //tools.push(this.agentTools.showAgentRunWidget);
+                            }
+                        }
+                    }
+                    else {
+                        // User has not configured the current agent
+                        console.log("Mode: agent_direct_connection_mode, User has not configured the current agent");
+                        tools.push(this.agentTools.showConfigurationWidget);
+                        if (this.assistant.haveShownConfigurationWidget) {
+                            console.log("Mode: agent_direct_connection_mode, User has shown the configuration widget");
+                            tools.push(this.agentTools.submitConfiguration);
+                        }
+                    }
+                }
+                else {
+                    // User is not subscribed to the current agent
+                    console.error("Mode: agent_direct_connection_mode, User is not subscribed to the current agent should not happen");
+                }
+            }
+            return tools;
+        }
+        catch (error) {
+            console.error(error);
+            return [];
+        }
+    }
+    async getMode() {
+        console.log("---------------------> getMode DirectConversationMode");
+        const systemPrompt = await this.getCurrentModeSystemPrompt();
+        const tools = await this.getCurrentModeTools();
+        return {
+            name: "agent_direct_connection_mode",
+            description: "Direct connection to an agent the user is subscribed to or available for purchase",
+            systemPrompt: systemPrompt,
+            tools: tools,
+            allowedTransitions: ["agent_direct_connection_mode"],
+        };
+    }
+}

package/dist/agents/assistants/modes/agentSelectionMode.js

@@ -0,0 +1,44 @@
+import { BaseAssistantMode } from "./baseAssistantMode.js";
+import { NavigationTools } from "./tools/navigationTools.js";
+import { LoginAssistantTools } from "./tools/loginTools.js";
+export class AgentSelectionMode extends BaseAssistantMode {
+    constructor(assistant) {
+        super(assistant);
+        this.navigationTools = new NavigationTools(this.assistant);
+        this.loginTools = new LoginAssistantTools(this.assistant);
+    }
+    async getCurrentModeSystemPrompt() {
+        let systemPrompt = `You are the main AI agent assistant. Help users select AI agent workflows to connect to. Use your tools when needed.`;
+        systemPrompt += await this.renderCommon();
+        return systemPrompt;
+    }
+    async getCurrentModeTools() {
+        const tools = [
+            this.navigationTools.listAllAgentsAvailableForConnection,
+        ];
+        if (this.assistant.isLoggedIn) {
+            tools.push(this.navigationTools.connectDirectlyToAgent);
+            tools.push(this.loginTools.logout);
+        }
+        else {
+            tools.push(this.loginTools.showLogin("Show login widget to the user when needed. You do need to log in to connect to an agent. When you have logged in the connect directly to agent tool will be available."));
+            if (this.assistant.haveShownLoginWidget) {
+                tools.push(this.loginTools.clickMainLoginButton);
+                tools.push(this.loginTools.clickGoogleLoginButton);
+            }
+        }
+        return tools;
+    }
+    async getMode() {
+        console.log("---------------------> getMode AgentSelectionMode");
+        const systemPrompt = await this.getCurrentModeSystemPrompt();
+        const tools = await this.getCurrentModeTools();
+        return {
+            name: "agent_selection_mode",
+            description: "List available agents the user can connect to then connect the user to the agent selected by the user",
+            systemPrompt: systemPrompt,
+            tools: tools,
+            allowedTransitions: ["agent_direct_connection_mode"],
+        };
+    }
+}

package/dist/agents/assistants/modes/baseAssistantMode.js

@@ -0,0 +1,54 @@
+// baseAssistantMode.ts
+export class BaseAssistantMode {
+    constructor(assistant) {
+        this.assistant = assistant;
+    }
+    get memory() {
+        return this.assistant.memory;
+    }
+    renderSimplifiedWorkflowStep(step) {
+        if (!step) {
+            return "";
+        }
+        return `<${step.type}>${step.name}: ${step.description}</${step.type}>`;
+    }
+    renderSimplifiedWorkflow(workflow) {
+        let workflowInfo = "";
+        for (let step of workflow.steps) {
+            workflowInfo += this.renderSimplifiedWorkflowStep(step);
+        }
+        workflowInfo += `<currentStepIndex>${workflow.currentStepIndex}</currentStepIndex>`;
+        return workflowInfo;
+    }
+    renderSimplifiedAgentRun(agentRun) {
+        if (!agentRun) {
+            return "";
+        }
+        return `Status: ${agentRun.status}, Started: ${agentRun.start_time}, Workflow: ${this.renderSimplifiedWorkflow(agentRun.workflow)}`;
+    }
+    async renderCommon() {
+        if (!this.memory.currentMode) {
+            return "";
+        }
+        console.log(`renderCommon: currentConversationMode ${this.memory.currentMode}`);
+        let modeInfo = `<currentConversationMode>${this.memory.currentMode}</currentConversationMode>\n`;
+        const currentAgentProduct = await this.assistant.getCurrentAgentProduct();
+        const currentAgentWorkflow = await this.assistant.getCurrentAgentWorkflow();
+        if (currentAgentProduct) {
+            modeInfo += `<currentAgent>${currentAgentProduct.name}</currentAgent>\n`;
+        }
+        if (await this.assistant.isCurrentAgentActive()) {
+            modeInfo += `<currentAgentRunStatus>${this.renderSimplifiedAgentRun(await this.assistant.getCurrentAgentRun())}</currentAgentRunStatus>\n`;
+            modeInfo += `<currentWorkflowStep>${this.renderSimplifiedWorkflowStep(currentAgentWorkflow?.steps[currentAgentWorkflow?.currentStepIndex])}</currentWorkflowStep>\n`;
+            if (currentAgentWorkflow?.currentStepIndex != undefined &&
+                currentAgentWorkflow?.currentStepIndex + 1 <
+                    currentAgentWorkflow?.steps.length) {
+                modeInfo += `<nextWorkflowStep>${JSON.stringify(currentAgentWorkflow?.steps[currentAgentWorkflow?.currentStepIndex + 1], null, 2)}</nextWorkflowStep>`;
+            }
+            else {
+                modeInfo += `<nextWorkflowStep>none</nextWorkflowStep>\n`;
+            }
+        }
+        console.log(`modeInfo: ${modeInfo}`);
+    }
+}