@minded-ai/mindedjs 1.0.128 → 1.0.130

package/src/browserTask/executeBrowserTask.ts

@@ -5,222 +5,11 @@ import {
   CreateBrowserSessionResponse,
   InvokeBrowserTaskResponse,
   CreateBrowserSessionRequest,
+  InvokeBrowserTaskRequest,
 } from '../platform/mindedConnectionTypes';
 
-// Browser Use Cloud API configuration
-const BROWSER_USE_API_BASE_URL = 'https://api.browser-use.com/api/v1';
-
-// Types for Browser Use Cloud API
-export interface CloudTaskResponse {
-  id: string;
-  status: string;
-  live_url?: string;
-  steps?: any[];
-  output?: string;
-}
-
-// Browser Use Cloud API methods
-export const createCloudTask = async (prompt: string, model?: string): Promise<string> => {
-  const apiKey = process.env.BROWSER_USE_API_KEY;
-  if (!apiKey) {
-    throw new Error('BROWSER_USE_API_KEY environment variable is required');
-  }
-
-  logger.debug({ msg: 'Creating cloud browser task', prompt: prompt.substring(0, 100) + '...' });
-
-  const response = await fetch(`${BROWSER_USE_API_BASE_URL}/run-task`, {
-    method: 'POST',
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({
-      task: prompt,
-      use_proxy: false,
-      llm_model: model || 'gpt-4o',
-    }),
-  });
-
-  if (!response.ok) {
-    logger.error({ msg: 'Failed to create cloud browser task', status: response.status, statusText: response.statusText });
-    throw new Error(`Failed to create browser task: ${response.statusText}`);
-  }
-
-  const data = await response.json();
-  logger.debug({ msg: 'Cloud browser task created', taskId: data.id });
-  return data.id;
-};
-
-export const getTaskDetails = async (taskId: string): Promise<CloudTaskResponse> => {
-  const apiKey = process.env.BROWSER_USE_API_KEY;
-  if (!apiKey) {
-    throw new Error('BROWSER_USE_API_KEY environment variable is required');
-  }
-
-  const response = await fetch(`${BROWSER_USE_API_BASE_URL}/task/${taskId}`, {
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-    },
-  });
-
-  if (!response.ok) {
-    logger.error({ msg: 'Failed to get task details', taskId, status: response.status, statusText: response.statusText });
-    throw new Error(`Failed to get task details: ${response.statusText}`);
-  }
-
-  return response.json();
-};
-
-export const waitForLiveUrl = async (taskId: string, maxWaitTime: number = 30000): Promise<CloudTaskResponse> => {
-  const startTime = Date.now();
-  const pollInterval = 2000; // 2 seconds
-  let pollCount = 0;
-
-  logger.debug({ msg: 'Starting to poll for live_url', taskId, maxWaitTime, pollInterval });
-
-  while (Date.now() - startTime < maxWaitTime) {
-    pollCount++;
-    const elapsedTime = Date.now() - startTime;
-
-    logger.trace({
-      msg: 'Polling for live_url',
-      taskId,
-      pollCount,
-      elapsedTime,
-      remainingTime: maxWaitTime - elapsedTime,
-    });
-
-    const taskDetails = await getTaskDetails(taskId);
-
-    logger.trace({
-      msg: 'Task details received',
-      taskId,
-      status: taskDetails.status,
-      hasLiveUrl: !!taskDetails.live_url,
-      pollCount,
-    });
-
-    if (taskDetails.live_url) {
-      logger.debug({
-        msg: 'Live URL available',
-        taskId,
-        liveUrl: taskDetails.live_url,
-        pollCount,
-        totalTime: elapsedTime,
-      });
-      return taskDetails;
-    }
-
-    if (taskDetails.status === 'failed' || taskDetails.status === 'stopped') {
-      logger.error({
-        msg: 'Task failed while waiting for live_url',
-        taskId,
-        status: taskDetails.status,
-        pollCount,
-        elapsedTime,
-      });
-      throw new Error(`Task failed with status: ${taskDetails.status}`);
-    }
-
-    logger.trace({
-      msg: 'Live URL not yet available, continuing to poll',
-      taskId,
-      status: taskDetails.status,
-      pollCount,
-      nextPollIn: pollInterval,
-    });
-
-    await new Promise((resolve) => setTimeout(resolve, pollInterval));
-  }
-
-  logger.error({
-    msg: 'Timeout waiting for live_url',
-    taskId,
-    pollCount,
-    totalTime: Date.now() - startTime,
-    maxWaitTime,
-  });
-  throw new Error('Timeout waiting for live_url to become available');
-};
-
-export const waitForCompletion = async (taskId: string, maxWaitTime: number = 300000): Promise<CloudTaskResponse> => {
-  const startTime = Date.now();
-  const pollInterval = 3000; // 3 seconds for completion polling
-  let pollCount = 0;
-
-  logger.debug({ msg: 'Starting to poll for task completion', taskId, maxWaitTime, pollInterval });
-
-  while (Date.now() - startTime < maxWaitTime) {
-    pollCount++;
-    const elapsedTime = Date.now() - startTime;
-
-    logger.trace({
-      msg: 'Polling for task completion',
-      taskId,
-      pollCount,
-      elapsedTime,
-      remainingTime: maxWaitTime - elapsedTime,
-    });
-
-    const taskDetails = await getTaskDetails(taskId);
-
-    logger.trace({
-      msg: 'Task completion status received',
-      taskId,
-      status: taskDetails.status,
-      pollCount,
-      hasOutput: !!taskDetails.output,
-      stepCount: taskDetails.steps?.length || 0,
-    });
-
-    if (taskDetails.status === 'finished') {
-      logger.debug({
-        msg: 'Task completed successfully',
-        taskId,
-        pollCount,
-        totalTime: elapsedTime,
-        stepCount: taskDetails.steps?.length || 0,
-        hasOutput: !!taskDetails.output,
-      });
-      return taskDetails;
-    }
-
-    if (taskDetails.status === 'failed' || taskDetails.status === 'stopped') {
-      logger.error({
-        msg: 'Task failed during completion polling',
-        taskId,
-        status: taskDetails.status,
-        pollCount,
-        elapsedTime,
-        stepCount: taskDetails.steps?.length || 0,
-      });
-      throw new Error(`Task failed with status: ${taskDetails.status}`);
-    }
-
-    logger.trace({
-      msg: 'Task still in progress, continuing to poll',
-      taskId,
-      status: taskDetails.status,
-      pollCount,
-      stepCount: taskDetails.steps?.length || 0,
-      nextPollIn: pollInterval,
-    });
-
-    await new Promise((resolve) => setTimeout(resolve, pollInterval));
-  }
-
-  logger.error({
-    msg: 'Timeout waiting for task completion',
-    taskId,
-    pollCount,
-    totalTime: Date.now() - startTime,
-    maxWaitTime,
-  });
-  throw new Error('Timeout waiting for task completion');
-};
-
 // Socket-based browser task functions
-export const createBrowserSession = async (proxy?: string): Promise<CreateBrowserSessionResponse> => {
+export const createBrowserSession = async (proxy?: string, onPrem?: boolean): Promise<CreateBrowserSessionResponse> => {
   logger.debug({ msg: 'Creating browser session via socket', proxy });
 
   try {
@@ -229,6 +18,7 @@ export const createBrowserSession = async (proxy?: string): Promise<CreateBrowse
       {
         type: mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION,
         proxy,
+        onPrem: onPrem,
       },
       60000, // 60 seconds timeout
     );
@@ -257,6 +47,8 @@ export const invokeBrowserTask = async (
   task: string,
   keepAlive?: boolean,
   hooks?: { name: string }[],
+  onPrem?: boolean,
+  toolSchemas?: any[],
   outputSchema?: {
     name: string;
     type: 'string' | 'number';
@@ -270,18 +62,22 @@ export const invokeBrowserTask = async (
     taskLength: task.length,
     keepAlive,
     hooksCount: hooks?.length || 0,
+    onPrem,
     outputSchemaFields: outputSchema?.length || 0,
   });
 
   try {
-    const response = await mindedConnection.awaitEmit<any, InvokeBrowserTaskResponse>(
+    const response = await mindedConnection.awaitEmit<InvokeBrowserTaskRequest, InvokeBrowserTaskResponse>(
       mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
       {
-        sessionId,
+        type: mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
         cdpUrl,
         task,
+        sessionId,
         keepAlive,
         hooks,
+        onPrem,
+        toolSchemas,
         outputSchema,
       },
       900000, // 15 minutes timeout

package/src/debugging/index.ts

@@ -0,0 +1 @@
+export { LLMDebugCallbackHandler } from './llmCallbackHandler';

package/src/debugging/llmCallbackHandler.ts

@@ -0,0 +1,126 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable no-debugger */
+
+import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
+import { Serialized } from '@langchain/core/load/serializable';
+import { LLMResult } from '@langchain/core/outputs';
+import { BaseMessage } from '@langchain/core/messages';
+
+/**
+ * A callback handler for debugging LLM interactions in MindedJS agents.
+ * This handler allows you to inspect messages before they are sent to the LLM
+ * and see the responses received.
+ *
+ * This class extends BaseCallbackHandler from LangChain and provides several
+ * callback methods. You can override additional methods for more comprehensive debugging:
+ *
+ * Available LangChain Callback Methods:
+ * - handleChatModelStart - For chat model requests (receives BaseMessage arrays)
+ * - handleLLMStart - For standard LLM requests (receives string prompts)
+ * - handleLLMNewToken - For streaming tokens as they're generated
+ * - handleLLMEnd - When LLM finishes generating
+ * - handleLLMError - When LLM encounters an error
+ * - handleChainStart - When a LangChain chain starts execution
+ * - handleChainEnd - When a chain completes execution
+ * - handleChainError - When a chain encounters an error
+ * - handleToolStart - When a tool/function starts execution
+ * - handleToolEnd - When a tool completes execution
+ * - handleToolError - When a tool encounters an error
+ * - handleAgentAction - When an agent takes an action
+ * - handleAgentFinish - When an agent completes its task
+ *
+ * For more details on all available callbacks and their signatures, see the LangChain documentation:
+ * https://js.langchain.com/docs/modules/callbacks/
+ *
+ * @example
+ * ```typescript
+ * import { Agent, LLMDebugCallbackHandler } from '@minded-ai/mindedjs';
+ *
+ * const debugHandler = new LLMDebugCallbackHandler();
+ *
+ * const agent = new Agent({
+ *   config: {
+ *     ...config,
+ *     llm: {
+ *       ...config.llm,
+ *       properties: {
+ *         ...config.llm.properties,
+ *         callbacks: [debugHandler],
+ *       },
+ *     },
+ *   },
+ *   // ... other agent configuration
+ * });
+ * ```
+ */
+export class LLMDebugCallbackHandler extends BaseCallbackHandler {
+  name = 'LLMDebugCallbackHandler';
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * Called when a chat model (like GPT-4) starts processing.
+   * This is triggered before the messages are sent to the LLM.
+   * The messages parameter contains the full conversation history.
+   */
+  async handleChatModelStart(
+    llm: Serialized,
+    messages: BaseMessage[][],
+    runId: string,
+    parentRunId?: string,
+    extraParams?: Record<string, unknown>,
+    tags?: string[],
+    metadata?: Record<string, unknown>,
+    name?: string,
+  ): Promise<void> {
+    console.log('handleChatModelStart called with messages:', messages);
+    // Set a breakpoint here to inspect messages
+    debugger;
+  }
+
+  /**
+   * Called when any LLM starts processing (including non-chat models).
+   * For chat models, the messages are available in extraParams.messages.
+   * This is a more general callback that works with all LLM types.
+   */
+  async handleLLMStart(
+    llm: Serialized,
+    prompts: string[],
+    runId: string,
+    parentRunId?: string,
+    extraParams?: Record<string, unknown>,
+    tags?: string[],
+    metadata?: Record<string, unknown>,
+    name?: string,
+  ): Promise<void> {
+    console.log('handleLLMStart called with extraParams.messages:', extraParams?.messages);
+    // Set a breakpoint here to inspect messages
+    debugger;
+  }
+
+  /**
+   * Called when a chat model finishes processing and returns a response.
+   * The output contains the generated message and metadata.
+   */
+  async handleChatModelEnd(output: LLMResult, runId: string, parentRunId?: string): Promise<void> {
+    console.log('handleChatModelEnd called with output:', output);
+  }
+
+  /**
+   * Called when any LLM finishes processing and returns a response.
+   * This is called for all LLM types, including chat models.
+   */
+  async handleLLMEnd(output: LLMResult, runId: string, parentRunId?: string): Promise<void> {
+    console.log('handleLLMEnd called with output:', output);
+  }
+
+  /**
+   * Called when an error occurs during LLM processing.
+   * This captures any errors that happen during the LLM call.
+   */
+  async handleLLMError(err: Error, runId: string, parentRunId?: string): Promise<void> {
+    console.log('handleLLMError called with error:', err);
+  }
+}

package/src/nodes/addBrowserTaskNode.ts

@@ -54,7 +54,7 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
     const zodSchema = z.object(schemaFields);
 
     // Create langchain tool
-    const tool = langchainTool(() => { }, {
+    const tool = langchainTool(() => {}, {
       name: 'browser-task',
       description: node.prompt,
       schema: zodSchema,
@@ -98,7 +98,7 @@ ${compiledPrompt}
 ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(inputParams, null, 2)}\n\n` : ''}`;
 
       // Create browser session using socket
-      const session = await createBrowserSession(node.proxy);
+      const session = await createBrowserSession(node.proxy, node.onPrem);
 
       if (!session.sessionId || !session.cdpUrl) {
         throw new Error('Failed to create browser session: missing session details');
@@ -137,6 +137,7 @@ ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(i
             liveUrl: session.liveViewUrl,
             keepAlive,
             hooks: node.hooks || [],
+            onPrem: node.onPrem,
             outputSchema: node.outputSchema,
           },
         },

package/src/nodes/addBrowserTaskRunNode.ts

@@ -8,6 +8,7 @@ import { createHistoryStep } from '../utils/history';
 import { HistoryStep } from '../types/Agent.types';
 import { v4 as uuidv4 } from 'uuid';
 import { invokeBrowserTask } from '../browserTask/executeBrowserTask';
+import { zodSchemaToParams } from '../platform/utils/tools';
 
 type AddBrowserTaskRunNodeParams = {
   graph: PreCompiledGraph;
@@ -18,7 +19,7 @@ type AddBrowserTaskRunNodeParams = {
 
 export const buildBrowserTaskRunNodeName = (nodeName: string) => `${nodeName}${internalNodesSuffix.BROWSER_TASK_RUN}`;
 
-export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeName }: AddBrowserTaskRunNodeParams) => {
+export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeName, agent }: AddBrowserTaskRunNodeParams) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
     logger.info(`Executing browser task run node for ${browserTaskNode.displayName}`);
 
@@ -30,13 +31,31 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
     const toolCall = toolCallObj.tool_calls[0];
     const { prompt, ...inputParams } = toolCall.args;
     const { sessionId, cdpUrl, keepAlive, hooks, outputSchema } = toolCallMindedMetadata;
+
     try {
       if (!sessionId || !cdpUrl || !prompt) {
         throw new Error('Missing required parameters: sessionId, cdpUrl, or prompt');
       }
 
+      const toolSchemas = agent.tools
+        .filter((tool) => tool.allowExecutionRequests === true)
+        .map((tool) => ({
+          name: tool.name,
+          description: tool.description,
+          inputParams: zodSchemaToParams(tool.input),
+        }));
+
       // Invoke browser task via socket
-      const result = await invokeBrowserTask(sessionId, cdpUrl, prompt, keepAlive, hooks, outputSchema);
+      const result = await invokeBrowserTask(
+        sessionId,
+        cdpUrl,
+        prompt,
+        keepAlive,
+        hooks,
+        browserTaskNode.onPrem,
+        toolSchemas,
+        outputSchema,
+      );
 
       logger.debug({
         msg: 'Browser task completed',

package/src/platform/mindedConnectionTypes.ts

@@ -51,6 +51,8 @@ export enum mindedConnectionSocketMessageType {
   INTERRUPT_SESSION_DEQUEUE = 'interrupt-session-dequeue',
   INTERRUPT_SESSION_HAS_MESSAGES = 'interrupt-session-has-messages',
   INTERRUPT_SESSION_GET_MESSAGES = 'interrupt-session-get-messages',
+  // Tool execution
+  EXECUTE_TOOL = 'execute-tool',
 }
 
 export type mindedConnectionSocketMessageTypeMap = {
@@ -91,6 +93,7 @@ export type mindedConnectionSocketMessageTypeMap = {
   [mindedConnectionSocketMessageType.INTERRUPT_SESSION_DEQUEUE]: InterruptSessionDequeueRequest;
   [mindedConnectionSocketMessageType.INTERRUPT_SESSION_HAS_MESSAGES]: InterruptSessionHasMessagesRequest;
   [mindedConnectionSocketMessageType.INTERRUPT_SESSION_GET_MESSAGES]: InterruptSessionGetMessagesRequest;
+  [mindedConnectionSocketMessageType.EXECUTE_TOOL]: ExecuteToolRequest;
 };
 
 export interface BasemindedConnectionSocketMessage {
@@ -441,6 +444,7 @@ export interface UpdateStateRequest extends BasemindedConnectionSocketMessage {
 export interface CreateBrowserSessionRequest extends BasemindedConnectionSocketMessage {
   type: mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION;
   proxy?: string; // 2-digit country code like 'IL'
+  onPrem?: boolean;
 }
 
 export interface CreateBrowserSessionResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
@@ -455,7 +459,19 @@ export interface InvokeBrowserTaskRequest extends BasemindedConnectionSocketMess
   task: string;
   sessionId: string;
   keepAlive?: boolean;
-  hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
+  hooks?: {
+    name: string;
+    lifecycle_phases?: string[];
+    config?: Record<string, any>;
+  }[]; // Array of hooks to be passed to the browser-use lambda
+  onPrem?: boolean;
+  toolSchemas?: any[];
+  outputSchema?: {
+    name: string;
+    type: 'string' | 'number';
+    description?: string;
+    required?: boolean;
+  }[];
 }
 
 export interface InvokeBrowserTaskResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
@@ -463,3 +479,19 @@ export interface InvokeBrowserTaskResponse extends BaseSdkConnectionSocketMessag
   steps?: any[];
   recordings?: any[];
 }
+
+export interface ExecuteToolRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.EXECUTE_TOOL;
+  toolName: string;
+  toolParams: any;
+  sessionId: string;
+  requestId: string;
+  error?: string;
+  cdpUrl?: string; // Optional CDP URL for browser context
+}
+
+export interface ExecuteToolResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  requestId: string;
+  result?: any;
+  state?: any;
+}

package/src/platform/toolExecutor.ts

@@ -0,0 +1,118 @@
+import { Tool } from '../types/Tools.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import extractToolStateResponse from '../utils/extractStateMemoryResponse';
+import { ToolMessage } from '@langchain/core/messages';
+
+export interface ToolExecutionRequest {
+  toolName: string;
+  toolParams: any;
+  sessionId: string;
+  requestId: string;
+  cdpUrl?: string;
+}
+
+export interface ToolExecutionResponse {
+  result?: any;
+  state?: any;
+  error?: string;
+}
+
+export class ToolExecutor {
+  private tools: Map<string, Tool<any, any>> = new Map();
+
+  constructor(private agent: Agent) {}
+
+  /**
+   * Register tools for standalone execution
+   */
+  registerTools(tools: Tool<any, any>[]) {
+    tools.forEach((tool) => {
+      if (tool.allowExecutionRequests === true) {
+        this.tools.set(tool.name, tool);
+      }
+    });
+  }
+
+  /**
+   * Execute a tool by name with given parameters
+   * This is called when browser-use requests tool execution via socket
+   */
+  async executeTool(request: ToolExecutionRequest): Promise<ToolExecutionResponse> {
+    const { toolName, toolParams, sessionId, cdpUrl } = request;
+
+    try {
+      // Find the tool
+      const tool = this.tools.get(toolName);
+      if (!tool) {
+        throw new Error(`Tool not found: ${toolName}`);
+      }
+
+      // We only allow to execute tools that are marked as allowed to be executed
+
+      logger.debug({
+        msg: '[ToolExecutor] Executing tool',
+        toolName,
+        sessionId,
+        hasCdpUrl: !!cdpUrl,
+      });
+
+      // Get the current state for the session
+      const state = await this.agent.getState(sessionId);
+      if (!state) {
+        throw new Error(`Session state not found: ${sessionId}`);
+      }
+
+      // Execute the tool
+      const startTime = Date.now();
+
+      if (cdpUrl) {
+        state.cdpUrl = cdpUrl;
+      }
+
+      const response = await tool.execute({
+        input: toolParams,
+        state,
+        agent: this.agent,
+      });
+      const endTime = Date.now();
+
+      logger.debug({
+        msg: '[ToolExecutor] Tool execution completed',
+        toolName,
+        executionTimeMs: endTime - startTime,
+      });
+
+      // Extract state updates from the response
+      let stateUpdate = {};
+      let result = response;
+
+      // If the response is a ToolMessage, extract state updates
+      if (response && typeof response === 'object' && 'content' in response) {
+        const toolMessage = response as ToolMessage;
+        stateUpdate = extractToolStateResponse(toolMessage);
+        // Extract the content as the result
+        result = { result: toolMessage.content };
+      }
+
+      // Update the session state if there are state changes
+      if (Object.keys(stateUpdate).length > 0) {
+        await this.agent.updateState({ sessionId, state: stateUpdate });
+      }
+
+      return {
+        result,
+      };
+    } catch (error) {
+      logger.error({
+        msg: '[ToolExecutor] Error executing tool',
+        toolName,
+        error: error instanceof Error ? error.message : String(error),
+      });
+
+      return {
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}