@minded-ai/mindedjs 1.0.103-beta-1 → 1.0.103-beta-4

package/src/platform/mindedConnection.ts

@@ -87,43 +87,43 @@ const connect = async (token: string): Promise<void> => {
 
     const checkReady = () => {
       if (connected && ready) {
-        logger.info({ message: '\x1b[32mConnection with Minded platform is ready!\x1b[0m' });
-        logger.info({ message: '\x1b[32mPress Ctrl+C to exit...' });
+        logger.info({ msg: '\x1b[32mConnection with Minded platform is ready!\x1b[0m' });
+        logger.info({ msg: '\x1b[32mPress Ctrl+C to exit...' });
         resolve();
       }
     };
 
     // Connection event handlers
     socket.on('connect', () => {
-      logger.info({ message: 'Socket connected, waiting for server setup...' });
+      logger.info({ msg: 'Socket connected, waiting for server setup...' });
       connected = true;
       checkReady();
     });
 
     // Listen for ready event from server
     socket.on('sdk-socket-ready', (data: { agentId: string; orgName: string }) => {
-      logger.info({ message: 'Server ready signal received', data });
+      logger.info({ msg: 'Server ready signal received', data });
       ready = true;
       checkReady();
     });
 
     socket.on('connect_error', () => {
-      logger.error({ message: 'Failed to connect to minded platform' });
+      logger.error({ msg: 'Failed to connect to minded platform' });
       reject(new Error('Failed to connect to minded platform'));
     });
 
     socket.on('disconnect', () => {
-      logger.info({ message: 'Disconnected from local debugging socket' });
+      logger.info({ msg: 'Disconnected from local debugging socket' });
       connected = false;
       ready = false;
     });
 
     // Listen for error messages from the server
     socket.on('error', async (error: { message: string }) => {
-      logger.error({ message: 'Server error:', error });
+      logger.error({ msg: 'Server error:', error });
 
       if (error.message.includes('Invalid token')) {
-        logger.info({ message: 'Invalid token' });
+        logger.info({ msg: 'Invalid token' });
 
         // Disconnect current socket
         if (socket?.connected) {
@@ -142,14 +142,14 @@ const connect = async (token: string): Promise<void> => {
           listener(message, callback);
         });
       } else {
-        console.warn({ message: 'No listeners found for event', event });
+        console.warn({ msg: 'No listeners found for event', event });
       }
     });
 
     // Handle process termination
     process.on('SIGINT', () => {
       if (socket?.connected) {
-        logger.info({ message: '\nDisconnecting...' });
+        logger.info({ msg: '\nDisconnecting...' });
         socket.disconnect();
       }
       process.exit(0);
@@ -167,17 +167,17 @@ export const start = async (): Promise<void> => {
 
 export const disconnect = () => {
   if (!socket) {
-    logger.warn({ message: 'No socket connection to disconnect' });
+    logger.warn({ msg: 'No socket connection to disconnect' });
     return;
   }
 
   if (socket.connected) {
-    logger.info({ message: 'Disconnecting from Minded platform...' });
+    logger.info({ msg: 'Disconnecting from Minded platform...' });
     socket.disconnect();
     return;
   }
 
-  logger.warn({ message: 'Socket is already disconnected' });
+  logger.warn({ msg: 'Socket is already disconnected' });
 };
 
 export const mindedConnection = {

package/src/platform/mindedConnectionTypes.ts

@@ -42,6 +42,8 @@ export enum mindedConnectionSocketMessageType {
   INTERRUPT_SESSION_DEQUEUE = 'interrupt-session-dequeue',
   INTERRUPT_SESSION_HAS_MESSAGES = 'interrupt-session-has-messages',
   INTERRUPT_SESSION_GET_MESSAGES = 'interrupt-session-get-messages',
+  INTERRUPT_SESSION_INTERRUPT = 'interrupt-session-interrupt',
+  INTERRUPT_SESSION_IS_INTERRUPTED = 'interrupt-session-is-interrupted',
 }
 
 export type mindedConnectionSocketMessageTypeMap = {

package/src/platform/models/mindedChatOpenAI.ts

@@ -0,0 +1,49 @@
+import { ChatOpenAI, ChatOpenAIFields } from '@langchain/openai';
+import { getConfig } from '../config';
+
+export interface BaseParallelChatFields {
+  /**
+   * Number of parallel requests to make. Defaults to 1 (no parallelization).
+   * When > 1, multiple identical requests are sent and the fastest response is used.
+   */
+  numParallelRequests?: number;
+
+  /**
+   * Whether to log timing information for parallel requests
+   */
+  logTimings?: boolean;
+}
+
+export interface MindedChatOpenAIFields extends ChatOpenAIFields, BaseParallelChatFields {}
+
+export class MindedChatOpenAI extends ChatOpenAI {
+  private numParallelRequests: number;
+  private logTimings: boolean;
+
+  constructor(fields?: MindedChatOpenAIFields) {
+    const { token, baseUrl } = getConfig();
+    const mindedBaseUrl = `${baseUrl}/sdk/llmGateway/chatOpenAI`;
+    if (!token) {
+      throw new Error('Minded token not found');
+    }
+
+    // Store parallel configuration
+    const numParallelRequests = fields?.numParallelRequests || 1;
+    const logTimings = fields?.logTimings || false;
+
+    super({
+      ...fields,
+      apiKey: token,
+      configuration: {
+        baseURL: mindedBaseUrl,
+        defaultHeaders: {
+          'X-Parallel-Requests': numParallelRequests.toString(),
+          'X-Log-Timings': logTimings.toString(),
+        },
+      },
+    });
+
+    this.numParallelRequests = numParallelRequests;
+    this.logTimings = logTimings;
+  }
+}

package/src/platform/models/parallelWrapper.ts

@@ -0,0 +1,141 @@
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { BaseMessage } from '@langchain/core/messages';
+import { ChatGenerationChunk, ChatResult } from '@langchain/core/outputs';
+import { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import { logger } from '../../utils/logger';
+
+export interface BaseParallelChatFields {
+  /**
+   * Number of parallel requests to make. Defaults to 1 (no parallelization).
+   * When > 1, multiple identical requests are sent and the fastest response is used.
+   */
+  numParallelRequests?: number;
+
+  /**
+   * Whether to log timing information for parallel requests
+   */
+  logTimings?: boolean;
+}
+
+/**
+ * Creates a wrapped version of a chat model with parallel request capabilities
+ */
+export function createParallelWrapper<T extends BaseChatModel>(model: T, fields?: BaseParallelChatFields): T {
+  const numParallelRequests = fields?.numParallelRequests || 1;
+  const logTimings = fields?.logTimings || false;
+
+  // If parallel requests are disabled, return the original model
+  if (numParallelRequests <= 1) {
+    return model;
+  }
+
+  // Helper function that returns a promise that resolves with the second fulfilled promise
+  const promiseSecondFulfillment = <T>(promises: Promise<T>[]): Promise<T> => {
+    return new Promise((resolve) => {
+      let fulfillCount = 0;
+
+      for (const p of promises) {
+        Promise.resolve(p)
+          .then((value) => {
+            fulfillCount++;
+            if (fulfillCount === 2) {
+              resolve(value);
+            }
+          })
+          .catch(() => {
+            // Ignoring rejections for second fulfillment tracking
+          });
+      }
+    });
+  };
+
+  // Store original methods
+  const originalGenerate = model._generate.bind(model);
+  const originalStream = model._streamResponseChunks.bind(model);
+
+  // Override _generate
+  model._generate = async function (messages: BaseMessage[], options?: any, runManager?: CallbackManagerForLLMRun): Promise<ChatResult> {
+    const startTime = Date.now();
+
+    // Create array of identical requests
+    const requests = Array.from({ length: numParallelRequests }, () => originalGenerate(messages, options, runManager));
+
+    let fastestRequestTime = 0;
+    let secondFastestRequestTime = 0;
+
+    // Race all requests and return the fastest
+    const racePromise = Promise.race(requests).then((result) => {
+      fastestRequestTime = (Date.now() - startTime) / 1000.0;
+
+      if (logTimings) {
+        logger.debug({
+          msg: '[Model] Fastest request completed',
+          requestTime: fastestRequestTime,
+          numParallelRequests: numParallelRequests,
+        });
+      }
+
+      return result;
+    });
+
+    // Track second fastest if we have 3+ requests
+    if (requests.length >= 3) {
+      promiseSecondFulfillment(requests)
+        .then(() => {
+          secondFastestRequestTime = (Date.now() - startTime) / 1000.0;
+        })
+        .catch(() => {
+          // Ignore errors in timing tracking
+        });
+    }
+
+    // Track all completion times if we have 2+ requests
+    if (requests.length >= 2 && logTimings) {
+      Promise.all(requests)
+        .then(() => {
+          const allFinishTime = (Date.now() - startTime) / 1000.0;
+          const timeSaved = allFinishTime - fastestRequestTime;
+          const timeSavedFromSecond = secondFastestRequestTime > 0 ? secondFastestRequestTime - fastestRequestTime : 0;
+
+          logger.debug({
+            msg: '[Model] Time saved using parallel requests',
+            fastestRequestTime,
+            secondFastestRequestTime: secondFastestRequestTime || 'N/A',
+            allFinishTime,
+            timeSaved,
+            timeSavedFromSecond,
+            numParallelRequests: numParallelRequests,
+          });
+        })
+        .catch(() => {
+          // Ignore errors in timing tracking
+        });
+    }
+
+    return racePromise;
+  };
+
+  // Override _streamResponseChunks
+  model._streamResponseChunks = async function* (
+    messages: BaseMessage[],
+    options?: any,
+    runManager?: CallbackManagerForLLMRun,
+  ): AsyncGenerator<ChatGenerationChunk> {
+    // For streaming, we'll use the original implementation
+    // Parallel streaming is more complex and may not provide the same benefits
+    if (numParallelRequests > 1 && logTimings) {
+      logger.debug({
+        msg: '[Model] Streaming mode - using single request',
+        reason: 'Parallel streaming not implemented',
+      });
+    }
+
+    yield* originalStream(messages, options, runManager);
+  };
+
+  // Add properties for testing
+  (model as any).numParallelRequests = numParallelRequests;
+  (model as any).logTimings = logTimings;
+
+  return model;
+}

package/src/playbooks/playbooks.ts

@@ -71,7 +71,7 @@ function editorJsToMarkdown(blocks: OutputBlockData[]): string {
           }
         }
       } catch (error) {
-        logger.error({ message: 'Error converting EditorJS blocks to markdown', error, block });
+        logger.error({ msg: 'Error converting EditorJS blocks to markdown', error, block });
         throw error;
       }
     })
@@ -113,7 +113,7 @@ function loadPlaybooksFromDirectories(directories: string[]): Playbook[] {
 
   for (const directory of directories) {
     if (!fs.existsSync(directory)) {
-      logger.info({ message: `Playbooks directory does not exist: ${directory}` });
+      logger.info({ msg: `Playbooks directory does not exist: ${directory}` });
       continue;
     }
 
@@ -126,12 +126,12 @@ function loadPlaybooksFromDirectories(directories: string[]): Playbook[] {
 
         if (playbook && playbook.name && playbook.blocks) {
           playbooks.push(playbook);
-          logger.info({ message: `Loaded playbook: ${playbook.name} from ${file}` });
+          logger.info({ msg: `Loaded playbook: ${playbook.name} from ${file}` });
         } else {
-          logger.warn({ message: `Invalid playbook structure in ${file}` });
+          logger.warn({ msg: `Invalid playbook structure in ${file}` });
         }
       } catch (error) {
-        logger.error({ message: `Failed to load playbook file ${file}: ${error}` });
+        logger.error({ msg: `Failed to load playbook file ${file}: ${error}` });
       }
     }
   }
@@ -183,7 +183,7 @@ export function compilePlaybooks(playbooks: Playbook[], params: Record<string, a
 
     return compiledPlaybooks;
   } catch (error) {
-    logger.error({ message: 'Error compiling playbooks', error });
+    logger.error({ msg: 'Error compiling playbooks', error });
     return combinedPlaybooks; // Return uncompiled if there's an error
   }
 }

package/src/types/Flows.types.ts

@@ -11,6 +11,7 @@ export enum NodeType {
   APP_TOOL = 'appTool',
   PROMPT_NODE = 'promptNode',
   JUMP_TO_NODE = 'jumpToNode',
+  BROWSER_TASK = 'browserTask',
 }
 
 export enum EdgeType {
@@ -58,6 +59,8 @@ export interface AppTriggerNode extends BaseTriggerNode, BaseAppNode {
 
 export interface WebhookTriggerNode extends BaseTriggerNode {
   triggerType: TriggerType.WEBHOOK;
+  autoTrigger?: boolean;
+  defaultPayload?: string;
 }
 
 export interface VoiceTriggerNode extends BaseTriggerNode {
@@ -87,6 +90,18 @@ export interface JumpToNode extends BaseNode {
   targetNodeId: string;
 }
 
+export interface BrowserTaskNode extends BaseNode {
+  type: NodeType.BROWSER_TASK;
+  prompt: string;
+  model?: string;
+  inputSchema?: {
+    name: string;
+    type: 'string' | 'number';
+    description?: string;
+    required?: boolean;
+  }[];
+}
+
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode;
 
 export interface JunctionNode extends BaseNode {
@@ -109,7 +124,7 @@ export interface AppToolNode extends BaseNode, BaseAppNode {
   actionKey: string;
 }
 
-export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode;
+export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode;
 
 export interface BaseEdge {
   source: string;
@@ -181,4 +196,5 @@ export enum KnownTriggerNames {
 export enum internalNodesSuffix {
   HUMAN_IN_THE_LOOP = '_humanInTheLoop',
   TOOL_RUN = '_toolRun',
+  BROWSER_TASK_RUN = '_browserTaskRun',
 }

package/src/types/LLM.types.ts

@@ -1,10 +1,10 @@
 import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
-import { MindedChatOpenAI } from '../platform/mindedChatOpenAI';
+import { MindedChatOpenAI } from '../platform/models/mindedChatOpenAI';
 
 export type LLMConfig = {
-  name: string,
-  properties: Record<string, any>
-}
+  name: string;
+  properties: Record<string, any>;
+};
 
 export type LLMProvider = 'ChatOpenAI' | 'AzureChatOpenAI' | 'MindedChatOpenAI';
 
@@ -12,4 +12,4 @@ export const LLMProviders: Record<LLMProvider, any> = {
   ChatOpenAI: ChatOpenAI,
   AzureChatOpenAI: AzureChatOpenAI,
   MindedChatOpenAI,
-}
+};

package/src/types/LangGraph.types.ts

@@ -13,6 +13,7 @@ export const createStateAnnotation = <Memory = any>() =>
             const index = res.findIndex((m) => m.id && m.id === message.id);
             if (index !== -1) {
               res[index].content = message.content;
+              res[index].additional_kwargs = { ...res[index].additional_kwargs, ...message.additional_kwargs };
             }
           } else {
             res.push(message);
@@ -32,6 +33,10 @@ export const createStateAnnotation = <Memory = any>() =>
     sessionId: Annotation<string>({
       reducer: (a, b) => b || a,
     }),
+    goto: Annotation<string | null | undefined>({
+      default: () => null,
+      reducer: (a, b) => b,
+    }),
     sessionType: Annotation<SessionType>({
       default: () => SessionType.TEXT,
       reducer: (a, b) => b || a,

package/src/utils/logger.ts

@@ -4,7 +4,7 @@ dotenv.config();
 
 // Create a Pino logger with a custom log format
 export const logger = pino({
-  level: process.env.LOG_LEVEL || 'info',
+  level: process.env.LOG_LEVEL || 'trace',
   base: null,
   transport: {
     target: 'pino-pretty',
@@ -12,7 +12,6 @@ export const logger = pino({
       colorize: true,
       crlf: true,
       translateTime: 'SYS:standard',
-      messageKey: 'message', // Use 'message' instead of default 'msg'
     },
   },
   formatters: {

package/src/voice/voiceSession.ts

@@ -25,7 +25,7 @@ export class VoiceSession {
   private onDisconnectCallback?: () => void;
 
   constructor({ agent, sessionId, firstMessage, voiceId }: { agent: Agent; sessionId: string; firstMessage: string; voiceId?: string }) {
-    logger.debug({ message: 'Starting voice session', sessionId, firstMessage, voiceId });
+    logger.debug({ msg: '[Voice] Starting voice session', sessionId, firstMessage, voiceId });
     this.agent = agent;
     this.sessionId = sessionId;
     this.firstMessage = firstMessage;
@@ -51,7 +51,6 @@ export class VoiceSession {
       apiKey: elevenLabsKey,
     });
 
-    logger.debug({ message: 'Connecting to ElevenLabs', sessionId: this.sessionId });
     this.elevenLabsSocket = new WebSocket(signedUrl);
     this.setupSocketHandlers();
   }
@@ -90,7 +89,7 @@ export class VoiceSession {
     const socket = this.elevenLabsSocket!; // non-null assertion once, we ensured it's assigned in init()
 
     socket.onopen = () => {
-      logger.debug({ message: 'Connected to ElevenLabs', sessionId: this.sessionId });
+      logger.debug({ msg: '[Voice] Connected to ElevenLabs', sessionId: this.sessionId });
       const initiationData: ConversationInitiationClientData = {
         type: 'conversation_initiation_client_data',
         conversation_config_override: {
@@ -116,13 +115,13 @@ export class VoiceSession {
     };
 
     socket.onclose = () => {
-      logger.debug({ message: 'Disconnected from ElevenLabs, removing voice session', sessionId: this.sessionId });
+      logger.debug({ msg: '[Voice] Disconnected from ElevenLabs, removing voice session', sessionId: this.sessionId });
       this.onDisconnectCallback?.();
       this.agent.voiceSessions.delete(this.sessionId);
     };
 
     socket.onerror = (err: unknown) => {
-      logger.error({ message: '[ElevenLabsVoice] socket error', sessionId: this.sessionId, error: err });
+      logger.error({ msg: '[Voice] socket error', sessionId: this.sessionId, error: err });
     };
 
     socket.onmessage = async (event: any) => {
@@ -136,7 +135,7 @@ export class VoiceSession {
           break;
         case 'user_transcript':
           logger.debug({
-            message: 'User transcript received',
+            msg: '[Voice] Received user transcript',
             sessionId: this.sessionId,
             data: data.user_transcription_event.user_transcript,
           });
@@ -148,6 +147,7 @@ export class VoiceSession {
           }
           break;
         case 'agent_response':
+          logger.debug({ msg: '[Voice] Received agent response' });
           if (this.onMessageCallback) {
             this.onMessageCallback(
               data.agent_response_event.agent_response,
@@ -156,8 +156,9 @@ export class VoiceSession {
           }
           break;
         case 'interruption':
-          logger.debug('Interruption received');
+          logger.debug({ msg: '[Voice] Received interruption' });
           this.onInterruptionCallback?.();
+          this.agent.interruptSessionManager.interrupt(this.sessionId);
           // Send interruption event to dashboard if connected
           if (getConfig().dashboardConnected) {
             try {
@@ -167,12 +168,12 @@ export class VoiceSession {
                 timestamp: Date.now(),
               });
             } catch (error) {
-              logger.error({ message: '[ElevenLabsVoice] Error sending interruption to dashboard', sessionId: this.sessionId, error });
+              logger.error({ msg: '[Voice] Error sending interruption to dashboard', sessionId: this.sessionId, error });
             }
           }
           break;
         case 'audio':
-          logger.debug({ message: 'Agent audio received', sessionId: this.sessionId });
+          logger.debug({ msg: '[Voice] Received audio response to user', sessionId: this.sessionId });
           if (this.onAudioCallback) {
             this.onAudioCallback(data.audio_event.audio_base_64);
           }
@@ -185,17 +186,17 @@ export class VoiceSession {
                 type: mindedConnectionSocketMessageType.DASHBOARD_VOICE_AGENT_AUDIO,
               });
             } catch (error) {
-              logger.error(`[ElevenLabsVoice] Error sending audio to dashboard: ${error}`);
+              logger.error({ msg: '[Voice] Error sending audio to dashboard', sessionId: this.sessionId, error });
             }
           }
           break;
         case 'conversation_initiation_metadata':
-          logger.debug({ message: 'ElevenLabs conversation initiation metadata', sessionId: this.sessionId, data });
+          logger.trace({ msg: '[Voice] ElevenLabs conversation initiation metadata', sessionId: this.sessionId, data });
           break;
         case 'agent_response_correction':
           try {
             logger.debug({
-              message: 'Agent response correction received',
+              msg: '[Voice] Received agent response correction',
               sessionId: this.sessionId,
               data: data.agent_response_correction_event,
             });
@@ -203,19 +204,18 @@ export class VoiceSession {
               data.agent_response_correction_event.original_agent_response,
               data.agent_response_correction_event.corrected_agent_response,
             );
-          } catch (error) {
-            logger.error({ message: '[ElevenLabsVoice] Error updating agent response', sessionId: this.sessionId, error });
+          } catch (error: any) {
+            logger.error({ msg: '[Voice] Error updating agent response', sessionId: this.sessionId, error: error.message });
           }
           break;
         default:
-          logger.debug({ message: 'Received unknown message from ElevenLabs', sessionId: this.sessionId, data });
+          logger.debug({ msg: '[Voice] Received unknown message from ElevenLabs', sessionId: this.sessionId, data });
           break;
       }
     };
   }
 
   private async addFirstMessageToState(): Promise<void> {
-    logger.debug({ message: 'Adding first message to state', sessionId: this.sessionId, firstMessage: this.firstMessage });
     const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
     if (!graphState.values.messages?.length) {
       await this.agent.compiledGraph.updateState(this.agent.getLangraphConfig(this.sessionId), {
@@ -227,8 +227,9 @@ export class VoiceSession {
 
   private async updateAgentResponse(originalAgentResponse: string, correctedAgentResponse: string): Promise<void> {
     const graphState = await this.agent.compiledGraph.getState(this.agent.getLangraphConfig(this.sessionId));
+    const cleanedOriginalAgentResponse = originalAgentResponse.replace('... ', '');
     const agentMessage: AIMessage | null = graphState.values.messages.find(
-      (message: BaseMessage) => message.content === originalAgentResponse && message instanceof AIMessage,
+      (message: BaseMessage) => message.content === cleanedOriginalAgentResponse && message instanceof AIMessage,
     );
     if (agentMessage) {
       agentMessage.content = correctedAgentResponse;
@@ -240,12 +241,12 @@ export class VoiceSession {
         overrideStartFromNodeId: currentNodeId,
       });
     } else {
-      logger.warn({ message: 'Agent message not found for correction', sessionId: this.sessionId, originalAgentResponse });
+      logger.warn({ msg: '[Voice] Agent message not found for correction', sessionId: this.sessionId, originalAgentResponse });
     }
   }
 
   public hangup(): void {
-    logger.info({ message: 'Hanging up voice session', sessionId: this.sessionId });
+    logger.info({ msg: '[Voice] Hanging up voice session', sessionId: this.sessionId });
     this.onDisconnectCallback?.();
     this.elevenLabsSocket?.close();
   }

package/src/platform/mindedChatOpenAI.ts

@@ -1,19 +0,0 @@
-import { ChatOpenAI, ChatOpenAIFields } from "@langchain/openai";
-import { getConfig } from "./config";
-
-export class MindedChatOpenAI extends ChatOpenAI {
-  constructor(fields?: ChatOpenAIFields) {
-    const { token, baseUrl } = getConfig();
-    const mindedBaseUrl = `${baseUrl}/sdk/llmGateway/chatOpenAI`;
-    if (!token) {
-      throw new Error('Minded token not found');
-    }
-    super({
-      ...fields,
-      apiKey: token,
-      configuration: {
-        baseURL: mindedBaseUrl,
-      },
-    });
-  }
-}