@minded-ai/mindedjs 1.0.89 → 1.0.91-beta-1

package/src/agent.ts

@@ -25,7 +25,6 @@ import {
   AgentInvokeParams,
   AppTriggerHistoryStep,
   TriggerHistoryStep,
-  HistoryStepType,
   MindedSDKConfig,
   SessionType,
   HistoryStep,
@@ -378,12 +377,12 @@ export class Agent {
     return appName
       ? createHistoryStep<AppTriggerHistoryStep>(currentHistory, {
           ...baseStep,
-          type: HistoryStepType.APP_TRIGGER_NODE,
+          type: NodeType.TRIGGER,
           appName,
         })
       : createHistoryStep<TriggerHistoryStep>(currentHistory, {
           ...baseStep,
-          type: HistoryStepType.TRIGGER_NODE,
+          type: NodeType.TRIGGER,
         });
   }
 
@@ -456,7 +455,7 @@ export class Agent {
         }
       }
 
-      logger.info(`Invoking trigger ${triggerName} with session ${sessionId}`);
+      logger.info({ message: 'Invoking trigger', triggerName, sessionId, triggerBody });
       const langraphConfig = this.getLangraphConfig(sessionId || uuidv4());
       const state = await this.compiledGraph.getState(langraphConfig);
       const suffixes = Object.values(internalNodesSuffix);
@@ -487,8 +486,8 @@ export class Agent {
         );
       }
       return res;
-    } catch (error) {
-      logger.error({ message: 'Invoke error', error, sessionId });
+    } catch (error: any) {
+      logger.error({ message: 'Invoke error', errorMessage: error.message, sessionId });
       const state = await this.compiledGraph.getState(this.getLangraphConfig(sessionId));
       this.emit(AgentEvents.ERROR, { error: error instanceof Error ? error : new Error(JSON.stringify(error)), state: state.values });
       throw error;
@@ -813,23 +812,45 @@ export class Agent {
         if (voiceSession) {
           voiceSession.sendAudio(audioMessage.audioData);
         } else {
-          logger.warn({ message: 'Audio received; voice session not found for sessionId', sessionId: audioMessage.sessionId });
+          logger.trace({
+            message: 'Audio received; voice session not found for sessionId',
+            sessionId: audioMessage.sessionId,
+            activeSessions: Array.from(this.voiceSessions.keys()),
+          });
         }
       });
 
       // Hangup / end session handler
       connection.on(MindedConnectionSocketMessageType.DASHBOARD_VOICE_SESSION_END, (message) => {
         const hangup = message as BaseVoiceMessage;
+        logger.debug({ message: 'Dashboard eneded voice session', sessionId: hangup.sessionId });
         const voiceSession = this.voiceSessions.get(hangup.sessionId);
         if (voiceSession) {
           voiceSession.hangup();
         } else {
-          logger.warn({ message: 'Session ended; voice session not found for sessionId', sessionId: hangup.sessionId });
+          logger.trace({
+            message: 'Session ended; voice session not found for sessionId',
+            sessionId: hangup.sessionId,
+            activeSessions: this.voiceSessions.keys(),
+          });
         }
       });
     }
   }
 
+  public async sendPlaceholderMessage({ sessionId, message }: { sessionId: string; message: string }): Promise<void> {
+    if (!this.mindedConnection) {
+      throw new Error('Minded connection is not established when trying to send placeholder message');
+    }
+    const connection = this.mindedConnection;
+    connection.emit(MindedConnectionSocketMessageType.DASHBOARD_VOICE_PLACEHOLDER_MESSAGES, {
+      sessionId,
+      message,
+      timestamp: Date.now(),
+      type: MindedConnectionSocketMessageType.DASHBOARD_VOICE_PLACEHOLDER_MESSAGES,
+    });
+  }
+
   public async startVoiceSession(params: { sessionId: string }): Promise<VoiceSession> {
     await this.waitForInitialization();
     const voiceTrigger = this.flows
@@ -845,7 +866,14 @@ export class Agent {
       voiceId: voiceTrigger.voiceId,
     });
     await voiceSession.init();
+    logger.debug({ message: 'Voice session initialized', sessionId: params.sessionId });
     this.voiceSessions.set(params.sessionId, voiceSession);
+
+    // Emit voice session start event
+    await this.emit(AgentEvents.VOICE_SESSION_START, {
+      sessionId: params.sessionId,
+    });
+
     return voiceSession;
   }
 

package/src/events/AgentEvents.ts

@@ -6,6 +6,7 @@ export enum AgentEvents {
   INIT = 'INIT',
   AI_MESSAGE = 'AI_MESSAGE',
   TRIGGER_EVENT = 'TRIGGER_EVENT',
+  VOICE_SESSION_START = 'VOICE_SESSION_START',
   ERROR = 'ERROR',
   TIMER_TRIGGER = 'TIMER_TRIGGER',
 }
@@ -23,6 +24,9 @@ export type AgentEventRequestPayloads<Memory> = {
     triggerBody: any;
     sessionId?: string;
   };
+  [AgentEvents.VOICE_SESSION_START]: {
+    sessionId: string;
+  };
   [AgentEvents.ERROR]: {
     error: Error;
     state?: State<Memory>;
@@ -44,6 +48,7 @@ export type AgentEventResponsePayloads<Memory> = {
     history?: HistoryStep[];
     sessionId?: string;
   };
+  [AgentEvents.VOICE_SESSION_START]: void;
   [AgentEvents.ERROR]: void;
   [AgentEvents.TIMER_TRIGGER]: void;
 };

package/src/index.ts

@@ -36,7 +36,6 @@ export {
 } from './types/Flows.types';
 export type { Tool, ToolExecuteInput } from './types/Tools.types';
 export {
-  HistoryStepType,
   HistoryStep,
   TriggerHistoryStep,
   AppActionInvocationHistoryStep,

package/src/nodes/addAppToolNode.ts

@@ -1,4 +1,4 @@
-import { AppToolNode } from '../types/Flows.types';
+import { AppToolNode, NodeType } from '../types/Flows.types';
 import { tool as langchainTool } from '@langchain/core/tools';
 import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
 import { SystemMessage } from '@langchain/core/messages';
@@ -6,7 +6,7 @@ import { RunnableLike } from '@langchain/core/runnables';
 import { z } from 'zod';
 import { LLMProviders } from '../types/LLM.types';
 import { getAppActionRunnerTool } from '../internalTools/appActionRunnerTool';
-import { AppActionInvocationHistoryStep, HistoryStepType } from '../types/Agent.types';
+import { AppActionInvocationHistoryStep } from '../types/Agent.types';
 import { Agent } from '../agent';
 import { logger } from '../utils/logger';
 import { compilePlaybooks } from '../playbooks/playbooks';
@@ -82,7 +82,7 @@ export const addAppToolNode = async ({
     return {
       messages: [AIToolCallMessage, toolCallMessage],
       history: createHistoryStep<AppActionInvocationHistoryStep>(state.history, {
-        type: HistoryStepType.APP_ACTION_NODE,
+        type: NodeType.APP_TOOL,
         nodeId: node.name,
         nodeDisplayName: node.displayName!,
         raw: AIToolCallMessage.tool_calls[0],

package/src/nodes/addJumpToNode.ts

@@ -1,8 +1,8 @@
 import { RunnableLike } from '@langchain/core/runnables';
 import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
-import { JumpToNode } from '../types/Flows.types';
+import { JumpToNode, NodeType } from '../types/Flows.types';
 import { logger } from '../utils/logger';
-import { HistoryStepType, HistoryStep } from '../types/Agent.types';
+import { HistoryStep } from '../types/Agent.types';
 import { createHistoryStep } from '../utils/history';
 
 export const addJumpToNode = async ({ graph, node }: { graph: PreCompiledGraph; node: JumpToNode }) => {
@@ -11,7 +11,7 @@ export const addJumpToNode = async ({ graph, node }: { graph: PreCompiledGraph;
     // No state modifications are necessary; control flow is handled via edges.
     return {
       history: createHistoryStep<HistoryStep>(state.history, {
-        type: HistoryStepType.JUMP_TO_NODE,
+        type: NodeType.JUMP_TO_NODE,
         nodeId: node.name,
         nodeDisplayName: node.displayName,
         raw: '',

package/src/nodes/addJunctionNode.ts

@@ -0,0 +1,19 @@
+import { JunctionNode, NodeType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { createHistoryStep } from '../utils/history';
+import { HistoryStep } from '../types/Agent.types';
+
+export const addJunctionNode = ({ graph, node }: { graph: PreCompiledGraph; node: JunctionNode }) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    return {
+      history: createHistoryStep<HistoryStep>(state.history, {
+        type: NodeType.JUNCTION,
+        nodeId: node.name,
+        nodeDisplayName: node.displayName,
+        raw: '',
+      }),
+    };
+  };
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addPromptNode.ts

@@ -1,5 +1,5 @@
 import { RunnableLike } from '@langchain/core/runnables';
-import { PromptNode } from '../types/Flows.types';
+import { NodeType, PromptNode } from '../types/Flows.types';
 import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
 import { z } from 'zod';
 import { BaseLanguageModel } from '@langchain/core/language_models/base';
@@ -7,7 +7,7 @@ import { SystemMessage, AIMessage, ToolMessage } from '@langchain/core/messages'
 import { Tool } from '../types/Tools.types';
 import { tool as langchainTool } from '@langchain/core/tools';
 import { AgentEventRequestPayloads, AgentEvents } from '../events/AgentEvents';
-import { EmitSignature, HistoryStepType, HistoryStep } from '../types/Agent.types';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
 import { createLlmInstance } from '../llm/createLlmInstance';
 import extractToolStateResponse from '../utils/extractStateMemoryResponse';
 import { Agent } from '../agent';
@@ -104,7 +104,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
 
     return {
       history: createHistoryStep<HistoryStep>(state.history, {
-        type: HistoryStepType.PROMPT_NODE,
+        type: NodeType.PROMPT_NODE,
         nodeId: node.name,
         nodeDisplayName: node.displayName,
         raw: result.content,

package/src/nodes/addToolNode.ts

@@ -1,4 +1,4 @@
-import { ToolNode } from '../types/Flows.types';
+import { NodeType, ToolNode } from '../types/Flows.types';
 import { Tool } from '../types/Tools.types';
 import { tool as langchainTool } from '@langchain/core/tools';
 import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
@@ -10,7 +10,6 @@ import { Agent } from '../agent';
 import { compilePlaybooks } from '../playbooks/playbooks';
 import { createHistoryStep } from '../utils/history';
 import { HistoryStep } from '../types/Agent.types';
-import { HistoryStepType } from '../types/Agent.types';
 
 export const addToolNode = async ({
   graph,
@@ -60,7 +59,7 @@ export const addToolNode = async ({
     return {
       messages: [AIToolCallMessage],
       history: createHistoryStep<HistoryStep>(state.history, {
-        type: HistoryStepType.CUSTOM_ACTION_NODE,
+        type: NodeType.TOOL,
         nodeId: node.name,
         nodeDisplayName: node.displayName,
         raw: AIToolCallMessage.tool_calls?.[0] || '',

package/src/nodes/addToolRunNode.ts

@@ -2,7 +2,7 @@ import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
 import { RunnableLike } from '@langchain/core/runnables';
 import { Tool } from '../types/Tools.types';
 import { LLMProviders } from '../types/LLM.types';
-import { internalNodesSuffix, ToolNode } from '../types/Flows.types';
+import { internalNodesSuffix, NodeType, ToolNode } from '../types/Flows.types';
 import { tool as langchainTool } from '@langchain/core/tools';
 import { ToolMessage } from '@langchain/core/messages';
 import { z } from 'zod';
@@ -11,7 +11,6 @@ import { Agent } from '../agent';
 import { logger } from '../utils/logger';
 import { createHistoryStep } from '../utils/history';
 import { HistoryStep } from '../types/Agent.types';
-import { HistoryStepType } from '../types/Agent.types';
 
 type AddToolRunNodeParams = {
   graph: PreCompiledGraph;
@@ -54,14 +53,14 @@ export const addToolRunNode = async ({ graph, tools, toolNode, attachedToNodeNam
     const toolStateUpdate = extractToolStateResponse(toolCallMessage as ToolMessage);
 
     // Push the toolCallMessage into the messages array from toolStateUpdate
-    const updatedMessages = [...((toolStateUpdate as any).messages || []), toolCallMessage];
+    const updatedMessages = [toolCallMessage, ...((toolStateUpdate as any).messages || [])];
 
     // Return the full state update from the tool with the updated messages
     return {
       ...toolStateUpdate,
       messages: updatedMessages,
       history: createHistoryStep<HistoryStep>(state.history, {
-        type: HistoryStepType.CUSTOM_ACTION_NODE,
+        type: NodeType.TOOL,
         nodeId: toolNode.name,
         nodeDisplayName: toolNode.displayName,
         raw: toolCallMessage,

package/src/nodes/addTriggerNode.ts

@@ -1,8 +1,8 @@
 import { RunnableLike } from '@langchain/core/runnables';
-import { TriggerNode, TriggerType } from '../types/Flows.types';
+import { NodeType, TriggerNode, TriggerType } from '../types/Flows.types';
 import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
 import { logger } from '../utils/logger';
-import { HistoryStep, HistoryStepType } from '../types/Agent.types';
+import { HistoryStep } from '../types/Agent.types';
 import { createHistoryStep } from '../utils/history';
 
 export const addTriggerNode = async ({ graph, node }: { graph: PreCompiledGraph; node: TriggerNode }) => {
@@ -11,7 +11,7 @@ export const addTriggerNode = async ({ graph, node }: { graph: PreCompiledGraph;
     if (node.triggerType === TriggerType.MANUAL) {
       return {
         history: createHistoryStep<HistoryStep>(state.history, {
-          type: HistoryStepType.TRIGGER_NODE,
+          type: NodeType.TRIGGER,
           nodeId: node.name,
           nodeDisplayName: node.displayName,
           raw: '',

package/src/nodes/nodeFactory.ts

@@ -11,11 +11,7 @@ import { addTriggerNode } from './addTriggerNode';
 import { LLMProviders } from '../types/LLM.types';
 import { Agent } from '../agent';
 import { addJumpToNode } from './addJumpToNode';
-
-const addJunctionNode = ({ graph, node }: { graph: PreCompiledGraph; node: JunctionNode }) => {
-  const callback: RunnableLike = async () => { };
-  graph.addNode(node.name, callback);
-};
+import { addJunctionNode } from './addJunctionNode';
 
 export const nodeFactory = ({
   graph,

package/src/platform/mindedConnectionTypes.ts

@@ -27,6 +27,7 @@ export enum MindedConnectionSocketMessageType {
   DASHBOARD_VOICE_SESSION_START = 'dashboard-voice-session-start',
   DASHBOARD_VOICE_SESSION_END = 'dashboard-voice-session-end',
   DASHBOARD_VOICE_INTERRUPTION = 'dashboard-voice-interruption',
+  DASHBOARD_VOICE_PLACEHOLDER_MESSAGES = 'dashboard-voice-placeholder-messages',
   INTERFACE_FUNCTION_CALL = 'interface-function-call',
   TIMER_RESET = 'timer-reset',
   TIMER_CANCEL = 'timer-cancel',
@@ -48,6 +49,7 @@ export type MindedConnectionSocketMessageTypeMap = {
   [MindedConnectionSocketMessageType.DASHBOARD_VOICE_SESSION_START]: BaseVoiceMessage;
   [MindedConnectionSocketMessageType.DASHBOARD_VOICE_SESSION_END]: OnVoiceSessionEnd;
   [MindedConnectionSocketMessageType.DASHBOARD_VOICE_INTERRUPTION]: OnVoiceInterruption;
+  [MindedConnectionSocketMessageType.DASHBOARD_VOICE_PLACEHOLDER_MESSAGES]: VoicePlaceholderMessage;
   [MindedConnectionSocketMessageType.PII_HTTP_REQUEST]: OnPiiHttpRequest;
   [MindedConnectionSocketMessageType.INTERFACE_FUNCTION_CALL]: InterfaceFunctionCall;
   [MindedConnectionSocketMessageType.TIMER_RESET]: TimerResetRequest;
@@ -110,6 +112,10 @@ export interface OnVoiceAudioOut extends BaseVoiceMessage {
   audioData: string; // base64 encoded audio data
 }
 
+export interface VoicePlaceholderMessage extends BaseVoiceMessage {
+  message: string;
+}
+
 export interface OnVoiceInterruption extends BaseMindedConnectionSocketMessage {
   sessionId: string;
   timestamp: number;

package/src/types/Agent.types.ts

@@ -1,5 +1,6 @@
 import { AgentEventRequestPayloads, AgentEventResponsePayloads } from '../events/AgentEvents';
 import { LLMConfig } from './LLM.types';
+import { NodeType } from './Flows.types';
 
 export type EmitSignature<Memory, E extends keyof AgentEventRequestPayloads<Memory>> = (
   event: E,
@@ -37,10 +38,11 @@ export enum HistoryStepType {
   CUSTOM_ACTION_NODE = 'customActionNode',
   PROMPT_NODE = 'promptNode',
   JUMP_TO_NODE = 'jumpToNode',
+  JUNCTION_NODE = 'junctionNode',
 }
 
 export interface HistoryStep {
-  type: HistoryStepType;
+  type: NodeType;
   step: number;
   raw: any;
   nodeId: string;
@@ -49,21 +51,25 @@ export interface HistoryStep {
 }
 
 export interface AppTriggerHistoryStep extends HistoryStep {
-  type: HistoryStepType.APP_TRIGGER_NODE;
+  type: NodeType.TRIGGER;
   appName: string;
 }
 
 export interface TriggerHistoryStep extends HistoryStep {
-  type: HistoryStepType.TRIGGER_NODE;
+  type: NodeType.TRIGGER;
 }
 
 export interface AppActionInvocationHistoryStep extends HistoryStep {
-  type: HistoryStepType.APP_ACTION_NODE;
+  type: NodeType.APP_TOOL;
   appName: string;
 }
 
 export interface CustomActionInvocationHistoryStep extends HistoryStep {
-  type: HistoryStepType.CUSTOM_ACTION_NODE;
+  type: NodeType.TOOL;
+}
+
+export interface JunctionHistoryStep extends HistoryStep {
+  type: NodeType.JUNCTION;
 }
 
 export enum SessionType {