@minded-ai/mindedjs 1.0.117 → 1.0.118-beta-2

package/src/browserTask/executeBrowserTask.ts

@@ -1,4 +1,11 @@
 import { logger } from '../utils/logger';
+import { mindedConnection } from '../platform/mindedConnection';
+import {
+  mindedConnectionSocketMessageType,
+  CreateBrowserSessionResponse,
+  InvokeBrowserTaskResponse,
+  CreateBrowserSessionRequest,
+} from '../platform/mindedConnectionTypes';
 
 // Browser Use Cloud API configuration
 const BROWSER_USE_API_BASE_URL = 'https://api.browser-use.com/api/v1';
@@ -211,3 +218,83 @@ export const waitForCompletion = async (taskId: string, maxWaitTime: number = 30
   });
   throw new Error('Timeout waiting for task completion');
 };
+
+// Socket-based browser task functions
+export const createBrowserSession = async (proxy?: string): Promise<CreateBrowserSessionResponse> => {
+  logger.debug({ msg: 'Creating browser session via socket', proxy });
+
+  try {
+    const response = await mindedConnection.awaitEmit<CreateBrowserSessionRequest, CreateBrowserSessionResponse>(
+      mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION,
+      {
+        type: mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION,
+        proxy,
+      },
+      30000, // 30 seconds timeout
+    );
+
+    if (response.error) {
+      logger.error({ msg: 'Failed to create browser session', error: response.error });
+      throw new Error(response.error);
+    }
+
+    logger.debug({
+      msg: 'Browser session created successfully',
+      sessionId: response.sessionId,
+      hasLiveUrl: !!response.liveViewUrl,
+    });
+
+    return response;
+  } catch (error) {
+    logger.error({ msg: 'Error creating browser session', error });
+    throw error;
+  }
+};
+
+export const invokeBrowserTask = async (
+  sessionId: string,
+  cdpUrl: string,
+  task: string,
+  keepAlive?: boolean,
+  hooks?: { name: string }[],
+): Promise<InvokeBrowserTaskResponse> => {
+  logger.debug({
+    msg: 'Invoking browser task via socket',
+    sessionId,
+    taskLength: task.length,
+    keepAlive,
+    hooksCount: hooks?.length || 0,
+  });
+
+  try {
+    const response = await mindedConnection.awaitEmit<any, InvokeBrowserTaskResponse>(
+      mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
+      {
+        sessionId,
+        cdpUrl,
+        task,
+        keepAlive,
+        hooks,
+      },
+      120000, // 2 minutes timeout
+    );
+
+    if (response.error) {
+      logger.error({ msg: 'Failed to invoke browser task', error: response.error });
+      throw new Error(response.error);
+    }
+
+    logger.debug({
+      msg: 'Browser task completed successfully',
+      sessionId,
+      hasResult: !!response.result,
+      stepCount: response.steps?.length || 0,
+      recordingCount: response.recordings?.length || 0,
+    });
+
+    return response;
+  } catch (error) {
+    logger.error({ msg: 'Error invoking browser task', error });
+    throw error;
+  }
+};

package/src/nodes/addBrowserTaskNode.ts

@@ -7,7 +7,7 @@ import { createHistoryStep } from '../utils/history';
 import { AIMessage, SystemMessage } from '@langchain/core/messages';
 import { v4 as uuidv4 } from 'uuid';
 import { Agent } from '../agent';
-import { createCloudTask, waitForLiveUrl } from '../browserTask/executeBrowserTask';
+import { createBrowserSession } from '../browserTask/executeBrowserTask';
 import { tool as langchainTool } from '@langchain/core/tools';
 import { z } from 'zod';
 import { LLMProviders } from '../types/LLM.types';
@@ -146,20 +146,29 @@ ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(i
 ${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
 `;
 
-      // Create the browser task in the cloud
-      const taskId = await createCloudTask(fullPrompt, node.model);
-      logger.debug({ msg: 'Browser task created', taskId });
+      // Create browser session using socket
+      const session = await createBrowserSession(node.proxy);
 
-      // Wait for live_url to become available
-      const taskDetails = await waitForLiveUrl(taskId);
-      logger.debug({ msg: 'Live URL available', taskId, liveUrl: taskDetails.live_url });
+      if (!session.sessionId || !session.cdpUrl) {
+        throw new Error('Failed to create browser session: missing session details');
+      }
+
+      // Determine keepAlive based on NODE_ENV
+      const keepAlive = process.env.NODE_ENV !== 'production';
+
+      logger.debug({
+        msg: 'Browser session created',
+        sessionId: session.sessionId,
+        hasLiveUrl: !!session.liveViewUrl,
+        keepAlive,
+        nodeEnv: process.env.NODE_ENV,
+      });
 
-      // Update the tool call with taskId and other metadata
+      // Update the tool call with session info and prompt
       const updatedToolCall = {
         ...toolCall,
         args: {
           ...inputParams,
-          taskId,
           prompt: fullPrompt,
         },
       };
@@ -172,8 +181,11 @@ ${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
           mindedMetadata: {
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: node.displayName,
-            taskId,
-            liveUrl: taskDetails.live_url,
+            sessionId: session.sessionId,
+            cdpUrl: session.cdpUrl,
+            liveUrl: session.liveViewUrl,
+            keepAlive,
+            hooks: node.hooks || [],
           },
         },
       });
@@ -183,7 +195,7 @@ ${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
           type: NodeType.BROWSER_TASK,
           nodeId: node.name,
           nodeDisplayName: node.displayName,
-          raw: { taskId, liveUrl: taskDetails.live_url, inputParams },
+          raw: { taskId: session.sessionId, liveUrl: session.liveViewUrl, inputParams },
           messageIds: [toolCallingMessage.id!],
         }),
         messages: [toolCallingMessage],
@@ -211,6 +223,7 @@ ${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: node.displayName,
             error: error.message,
+            hooks: node.hooks || [],
           },
         },
       });

package/src/nodes/addBrowserTaskRunNode.ts

@@ -7,7 +7,7 @@ import { logger } from '../utils/logger';
 import { createHistoryStep } from '../utils/history';
 import { HistoryStep } from '../types/Agent.types';
 import { v4 as uuidv4 } from 'uuid';
-import { waitForCompletion } from '../browserTask/executeBrowserTask';
+import { invokeBrowserTask } from '../browserTask/executeBrowserTask';
 
 type AddBrowserTaskRunNodeParams = {
   graph: PreCompiledGraph;
@@ -26,23 +26,33 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
     if (!toolCallObj.tool_calls) {
       throw new Error('Tool call not found');
     }
+    const toolCallMindedMetadata = toolCallObj.additional_kwargs?.mindedMetadata;
     const toolCall = toolCallObj.tool_calls[0];
-    const { taskId, ...inputParams } = toolCall.args;
+    const { prompt, ...inputParams } = toolCall.args;
+    const { sessionId, cdpUrl, keepAlive, hooks } = toolCallMindedMetadata;
     try {
-      if (!taskId) {
-        throw new Error('Task ID not found in tool call arguments');
+      if (!sessionId || !cdpUrl || !prompt) {
+        throw new Error('Missing required parameters: sessionId, cdpUrl, or prompt');
       }
 
-      // Wait for task completion
-      const completedTask = await waitForCompletion(taskId);
-      logger.debug({ msg: 'Browser task completed', taskId, output: completedTask.output });
+      // Invoke browser task via socket
+      const result = await invokeBrowserTask(sessionId, cdpUrl, prompt, keepAlive, hooks);
+
+      logger.debug({
+        msg: 'Browser task completed',
+        sessionId,
+        hasResult: !!result.result,
+        stepCount: result.steps?.length || 0,
+        recordingCount: result.recordings?.length || 0,
+      });
 
       // Create tool message with the result
       const toolMessage = new ToolMessage({
         id: uuidv4(),
         content: JSON.stringify({
-          result: completedTask.output || 'Task completed successfully',
-          steps: completedTask.steps || [],
+          result: result.result || 'Task completed successfully',
+          steps: result.steps || [],
+          recordings: result.recordings || [],
           inputParams: inputParams,
         }),
         name: 'browser-task',
@@ -52,8 +62,9 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: browserTaskNode.displayName,
-            taskId,
-            steps: completedTask.steps,
+            sessionId,
+            steps: result.steps,
+            recordings: result.recordings,
             inputParams: inputParams,
           },
         },
@@ -69,7 +80,8 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             ...toolCallObj.additional_kwargs?.mindedMetadata,
             nodeDisplayName: browserTaskNode.displayName,
-            steps: completedTask.steps,
+            steps: result.steps,
+            recordings: result.recordings,
             status: 'completed',
           },
           update: true, // This triggers the message reducer to update the existing message
@@ -82,7 +94,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           type: NodeType.BROWSER_TASK,
           nodeId: browserTaskNode.name,
           nodeDisplayName: browserTaskNode.displayName,
-          raw: completedTask,
+          raw: result,
           messageIds: [],
         }),
       };
@@ -102,7 +114,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: browserTaskNode.displayName,
-            taskId,
+            sessionId,
             error: error.message,
             inputParams: inputParams,
           },

package/src/platform/mindedConnectionTypes.ts

@@ -39,7 +39,9 @@ export enum mindedConnectionSocketMessageType {
   TIMER_TRIGGER = 'timer-trigger',
   RETELL_CALL = 'retell-call',
   RETELL_GET_CALL = 'retell-get-call',
-  SDK_VERSION_MISMATCH = 'sdk-version-mismatch',
+  SDK_VERSION_MISMATCH = 'sdk-version-mismatch', // Browser Task
+  CREATE_BROWSER_SESSION = 'create-browser-session',
+  INVOKE_BROWSER_TASK = 'invoke-browser-task',
 }
 
 export type mindedConnectionSocketMessageTypeMap = {
@@ -69,6 +71,8 @@ export type mindedConnectionSocketMessageTypeMap = {
   [mindedConnectionSocketMessageType.RETELL_GET_CALL]: RetellGetCall;
   [mindedConnectionSocketMessageType.SDK_VERSION_MISMATCH]: SdkVersionMismatchMessage;
   [mindedConnectionSocketMessageType.UPDATE_STATE]: UpdateStateRequest;
+  [mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION]: CreateBrowserSessionRequest;
+  [mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK]: InvokeBrowserTaskRequest;
 };
 
 export interface BasemindedConnectionSocketMessage {
@@ -244,3 +248,30 @@ export interface UpdateStateRequest extends BasemindedConnectionSocketMessage {
   sessionId: string;
   state: Partial<typeof stateAnnotation.State>;
 }
+
+// Browser Task Messages
+export interface CreateBrowserSessionRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION;
+  proxy?: string; // 2-digit country code like 'IL'
+}
+
+export interface CreateBrowserSessionResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  sessionId?: string;
+  cdpUrl?: string;
+  liveViewUrl?: string;
+}
+
+export interface InvokeBrowserTaskRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK;
+  cdpUrl: string;
+  task: string;
+  sessionId: string;
+  keepAlive?: boolean;
+  hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
+}
+
+export interface InvokeBrowserTaskResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  result?: any;
+  steps?: any[];
+  recordings?: any[];
+}

package/src/types/Flows.types.ts

@@ -112,6 +112,8 @@ export interface BrowserTaskNode extends BaseNode {
   }[];
   autoTrigger?: boolean;
   defaultPayload?: string;
+  proxy?: string; // 2-digit country code like 'IL'
+  hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
 }
 
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;