@minded-ai/mindedjs 2.0.7 → 2.0.8-beta-2

package/src/nodes/addBrowserTaskNode.ts

@@ -54,7 +54,7 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
     const zodSchema = z.object(schemaFields);
 
     // Create langchain tool
-    const tool = langchainTool(() => {}, {
+    const tool = langchainTool(() => { }, {
       name: 'browser-task',
       description: node.prompt,
       schema: zodSchema,
@@ -100,7 +100,12 @@ ${compiledPrompt}
 ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(inputParams, null, 2)}\n\n` : ''}`;
 
       // Create browser session using socket
-      const session = await createBrowserSession(node.proxy, node.onPrem);
+      const session = await createBrowserSession({
+        sessionId: state.sessionId,
+        proxy: node.proxy,
+        onPrem: node.onPrem,
+        localRun: node.localRun,
+      });
 
       if (!session.sessionId || !session.cdpUrl) {
         throw new Error('Failed to create browser session: missing session details');

package/src/nodes/addBrowserTaskRunNode.ts

@@ -57,6 +57,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
         keepAlive,
         hooks,
         browserTaskNode.onPrem,
+        browserTaskNode.localRun,
         toolSchemas,
         outputSchema,
       );

package/src/nodes/addRpaNode.ts

@@ -0,0 +1,289 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, RpaNode, RpaActionType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { chromium, Browser, Page } from 'playwright';
+import { LLMProviders } from '../types/LLM.types';
+
+type AddRpaNodeParams = {
+  graph: PreCompiledGraph;
+  node: RpaNode;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addRpaNode = async ({ graph, node, tools, emit, agent, llm }: AddRpaNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing RPA node`, node: node.displayName, sessionId: state.sessionId });
+
+    let browser: Browser | null = null;
+    let page: Page | null = null;
+
+    try {
+      // Get CDP URL from state
+      const cdpUrl = state.cdpUrl;
+      if (!cdpUrl) {
+        throw new Error('CDP URL not found in state. Make sure a browser session is available.');
+      }
+
+      logger.debug({
+        msg: '[RPA] Connecting to browser via CDP',
+        cdpUrl,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Connect to existing browser via CDP
+      browser = await chromium.connectOverCDP(cdpUrl);
+      const contexts = browser.contexts();
+      if (contexts.length === 0) {
+        throw new Error('No browser contexts found');
+      }
+
+      // Get the first page or create a new one
+      const pages = contexts[0].pages();
+      page = pages.length > 0 ? pages[0] : await contexts[0].newPage();
+
+      // Set viewport if specified
+      if (node.viewport) {
+        await page.setViewportSize(node.viewport);
+      }
+
+      // Execute each step
+      const results = [];
+      for (const [index, step] of node.steps.entries()) {
+        logger.debug({
+          msg: '[RPA] Executing step',
+          stepIndex: index + 1,
+          stepType: step.type,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+
+        try {
+          const result = await executeRpaStep(page, step, llm);
+          results.push({
+            stepIndex: index + 1,
+            type: step.type,
+            success: true,
+            result,
+          });
+        } catch (stepError) {
+          logger.error({
+            msg: '[RPA] Step execution failed',
+            stepIndex: index + 1,
+            stepType: step.type,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          results.push({
+            stepIndex: index + 1,
+            type: step.type,
+            success: false,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+          });
+
+          // Stop execution on error unless configured otherwise
+          throw stepError;
+        }
+      }
+
+      // Update history with RPA execution results
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            steps: node.steps,
+            results,
+            viewport: node.viewport,
+          },
+          messageIds: [],
+        }),
+      );
+
+      // Clear goto to allow natural flow progression
+      state.goto = null;
+
+      // Check for interrupts after RPA execution
+      await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId, state);
+
+      return state;
+    } catch (error) {
+      logger.error({
+        msg: '[RPA] Error executing RPA node',
+        error: error instanceof Error ? error.message : 'Unknown error',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      throw error;
+    } finally {
+      // Note: We don't close the browser as it's connected via CDP
+      // The browser session should remain active for other operations
+    }
+  };
+
+  graph.addNode(node.name, callback);
+};
+
+// Helper function to execute individual RPA steps
+async function executeRpaStep(page: Page, step: any, llm?: (typeof LLMProviders)[keyof typeof LLMProviders]): Promise<any> {
+  switch (step.type) {
+    case RpaActionType.CLICK:
+      if (step.xpath) {
+        await page.locator(`xpath=${step.xpath}`).click({ timeout: 30000 });
+      } else if (step.selector) {
+        await page.click(step.selector, { timeout: 30000 });
+      }
+      return { action: 'clicked' };
+
+    case RpaActionType.TYPE:
+      if (step.shouldReplaceExistingText) {
+        if (step.xpath) {
+          await page.locator(`xpath=${step.xpath}`).fill(step.text || '');
+        } else if (step.selector) {
+          await page.fill(step.selector, step.text || '');
+        }
+      } else {
+        if (step.xpath) {
+          await page.locator(`xpath=${step.xpath}`).type(step.text || '');
+        } else if (step.selector) {
+          await page.type(step.selector, step.text || '');
+        }
+      }
+      return { action: 'typed', text: step.text };
+
+    case RpaActionType.WAIT:
+      await page.waitForTimeout(step.waitTime || 1000);
+      return { action: 'waited', duration: step.waitTime };
+
+    case RpaActionType.GOTO:
+      await page.goto(step.url || '', { waitUntil: 'networkidle' });
+      return { action: 'navigated', url: step.url };
+
+    case RpaActionType.PRESS:
+      await page.keyboard.press(step.key || 'Enter');
+      return { action: 'pressed', key: step.key };
+
+    case RpaActionType.SELECT:
+      if (step.xpath) {
+        await page.locator(`xpath=${step.xpath}`).selectOption(step.value || '');
+      } else if (step.selector) {
+        await page.selectOption(step.selector, step.value || '');
+      }
+      return { action: 'selected', value: step.value };
+
+    case RpaActionType.SCREENSHOT:
+      const screenshot = await page.screenshot({ type: 'png' });
+      return {
+        action: 'screenshot',
+        description: step.description,
+        data: screenshot.toString('base64'),
+      };
+
+    case RpaActionType.EXTRACT_DATA: {
+      if (!llm) {
+        throw new Error('LLM is required for EXTRACT_DATA action');
+      }
+
+      // Get the page content for extraction
+      const pageContent = await page.content();
+      const url = page.url();
+
+      // Prepare the output schema description for the LLM
+      const schemaDescription = step.outputSchema
+        ?.map((field: any) => {
+          let desc = `- ${field.name} (${field.type})`;
+          if (field.description) desc += `: ${field.description}`;
+          if (field.required) desc += ' [REQUIRED]';
+          return desc;
+        })
+        .join('\n');
+
+      // Create a JSON schema for structured output
+      const jsonSchema: any = {
+        type: 'object',
+        properties: {},
+        required: [],
+      };
+
+      step.outputSchema?.forEach((field: any) => {
+        const fieldSchema: any = { type: field.type };
+        if (field.description) fieldSchema.description = field.description;
+
+        if (field.type === 'array' && field.items) {
+          fieldSchema.items = field.items;
+        } else if (field.type === 'object' && field.properties) {
+          fieldSchema.properties = field.properties;
+        }
+
+        jsonSchema.properties[field.name] = fieldSchema;
+        if (field.required) {
+          jsonSchema.required.push(field.name);
+        }
+      });
+
+      const prompt = `Extract data from the following web page according to the specified schema.
+
+URL: ${url}
+
+OUTPUT SCHEMA:
+${schemaDescription}
+
+WEB PAGE CONTENT:
+${pageContent}
+
+Extract the requested data and return it as a JSON object matching the schema. If a field cannot be found or extracted, use null for optional fields or provide a reasonable default for required fields.`;
+
+      try {
+        // Use the LLM to extract data with structured output
+        const messages = [
+          {
+            role: 'user' as const,
+            content: prompt,
+          },
+        ];
+
+        const response = await llm.invoke(messages, {
+          response_format: {
+            type: 'json_object' as const,
+            schema: jsonSchema,
+          },
+          temperature: 0.1,
+          model: 'gpt-4o-mini',
+        });
+
+        // Parse the response content
+        const extractedData = typeof response.content === 'string' ? JSON.parse(response.content) : response.content;
+
+        return {
+          action: 'extract_data',
+          url,
+          data: extractedData,
+        };
+      } catch (error) {
+        logger.error({
+          msg: '[RPA] Failed to extract data',
+          error: error instanceof Error ? error.message : 'Unknown error',
+          url,
+        });
+        throw error;
+      }
+    }
+
+    default:
+      throw new Error(`Unknown RPA action type: ${step.type}`);
+  }
+}

package/src/nodes/nodeFactory.ts

@@ -12,6 +12,7 @@ import { Agent } from '../agent';
 import { addJumpToNode } from './addJumpToNode';
 import { addJunctionNode } from './addJunctionNode';
 import { addBrowserTaskNode } from './addBrowserTaskNode';
+import { addRpaNode } from './addRpaNode';
 
 export const nodeFactory = ({
   graph,
@@ -51,6 +52,9 @@ export const nodeFactory = ({
     case NodeType.BROWSER_TASK:
       addBrowserTaskNode({ graph, node, agent, llm });
       break;
+    case NodeType.RPA:
+      addRpaNode({ graph, node, tools, emit, agent, llm });
+      break;
     default:
       throw new Error(`Unsupported node type: ${nodeType}`);
   }

package/src/types/Flows.types.ts

@@ -12,6 +12,7 @@ export enum NodeType {
   PROMPT_NODE = 'promptNode',
   JUMP_TO_NODE = 'jumpToNode',
   BROWSER_TASK = 'browserTask',
+  RPA = 'rpa',
 }
 
 export enum EdgeType {
@@ -121,6 +122,53 @@ export interface BrowserTaskNode extends BaseNode {
   proxy?: string; // 2-digit country code like 'IL'
   hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
   onPrem?: boolean;
+  localRun?: boolean;
+}
+
+export enum RpaActionType {
+  CLICK = 'click',
+  TYPE = 'type',
+  WAIT = 'wait',
+  SCREENSHOT = 'screenshot',
+  SELECT = 'select',
+  PRESS = 'press',
+  GOTO = 'goto',
+  EXTRACT_DATA = 'extract_data',
+}
+
+export interface RpaStep {
+  id: string;
+  type: RpaActionType;
+  xpath?: string;
+  selector?: string;
+  text?: string;
+  value?: string;
+  key?: string;
+  url?: string;
+  shouldReplaceExistingText?: boolean;
+  waitTime?: number;
+  description?: string;
+  outputSchema?: {
+    name: string;
+    type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+    description?: string;
+    required?: boolean;
+    items?: {
+      type: 'string' | 'number' | 'boolean' | 'object';
+      properties?: any;
+    };
+    properties?: any;
+  }[];
+}
+
+export interface RpaNode extends BaseNode {
+  type: NodeType.RPA;
+  steps: RpaStep[];
+  timeout?: number;
+  viewport?: {
+    width: number;
+    height: number;
+  };
 }
 
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;
@@ -145,7 +193,7 @@ export interface AppToolNode extends BaseNode, BaseAppNode {
   actionKey: string;
 }
 
-export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode;
+export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode | RpaNode;
 
 export interface BaseEdge {
   source: string;