@minded-ai/mindedjs 2.0.7 → 2.0.8-beta-1

package/src/nodes/addRpaNode.ts

@@ -0,0 +1,205 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, RpaNode, RpaActionType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { chromium, Browser, Page } from 'playwright';
+
+type AddRpaNodeParams = {
+  graph: PreCompiledGraph;
+  node: RpaNode;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+};
+
+export const addRpaNode = async ({ graph, node, tools, emit, agent }: AddRpaNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing RPA node`, node: node.displayName, sessionId: state.sessionId });
+
+    let browser: Browser | null = null;
+    let page: Page | null = null;
+
+    try {
+      // Get CDP URL from state
+      const cdpUrl = state.cdpUrl;
+      if (!cdpUrl) {
+        throw new Error('CDP URL not found in state. Make sure a browser session is available.');
+      }
+
+      logger.debug({
+        msg: '[RPA] Connecting to browser via CDP',
+        cdpUrl,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Connect to existing browser via CDP
+      browser = await chromium.connectOverCDP(cdpUrl);
+      const contexts = browser.contexts();
+      if (contexts.length === 0) {
+        throw new Error('No browser contexts found');
+      }
+
+      // Get the first page or create a new one
+      const pages = contexts[0].pages();
+      page = pages.length > 0 ? pages[0] : await contexts[0].newPage();
+
+      // Set viewport if specified
+      if (node.viewport) {
+        await page.setViewportSize(node.viewport);
+      }
+
+      // Execute each step
+      const results = [];
+      for (const [index, step] of node.steps.entries()) {
+        logger.debug({
+          msg: '[RPA] Executing step',
+          stepIndex: index + 1,
+          stepType: step.type,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+
+        try {
+          const result = await executeRpaStep(page, step);
+          results.push({
+            stepIndex: index + 1,
+            type: step.type,
+            success: true,
+            result,
+          });
+        } catch (stepError) {
+          logger.error({
+            msg: '[RPA] Step execution failed',
+            stepIndex: index + 1,
+            stepType: step.type,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          results.push({
+            stepIndex: index + 1,
+            type: step.type,
+            success: false,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+          });
+
+          // Stop execution on error unless configured otherwise
+          throw stepError;
+        }
+      }
+
+      // Update history with RPA execution results
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            steps: node.steps,
+            results,
+            viewport: node.viewport,
+          },
+          messageIds: [],
+        }),
+      );
+
+      // Clear goto to allow natural flow progression
+      state.goto = null;
+
+      // Check for interrupts after RPA execution
+      await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId, state);
+
+      return state;
+    } catch (error) {
+      logger.error({
+        msg: '[RPA] Error executing RPA node',
+        error: error instanceof Error ? error.message : 'Unknown error',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      throw error;
+    } finally {
+      // Note: We don't close the browser as it's connected via CDP
+      // The browser session should remain active for other operations
+    }
+  };
+
+  graph.addNode(node.name, callback);
+};
+
+// Helper function to execute individual RPA steps
+async function executeRpaStep(page: Page, step: any): Promise<any> {
+  switch (step.type) {
+    case RpaActionType.CLICK:
+      if (step.xpath) {
+        await page.locator(`xpath=${step.xpath}`).click({ timeout: 30000 });
+      } else if (step.selector) {
+        await page.click(step.selector, { timeout: 30000 });
+      }
+      return { action: 'clicked' };
+
+    case RpaActionType.TYPE:
+      if (step.shouldReplaceExistingText) {
+        if (step.xpath) {
+          await page.locator(`xpath=${step.xpath}`).fill(step.text || '');
+        } else if (step.selector) {
+          await page.fill(step.selector, step.text || '');
+        }
+      } else {
+        if (step.xpath) {
+          await page.locator(`xpath=${step.xpath}`).type(step.text || '');
+        } else if (step.selector) {
+          await page.type(step.selector, step.text || '');
+        }
+      }
+      return { action: 'typed', text: step.text };
+
+    case RpaActionType.WAIT:
+      await page.waitForTimeout(step.waitTime || 1000);
+      return { action: 'waited', duration: step.waitTime };
+
+    case RpaActionType.GOTO:
+      await page.goto(step.url || '', { waitUntil: 'networkidle' });
+      return { action: 'navigated', url: step.url };
+
+    case RpaActionType.PRESS:
+      await page.keyboard.press(step.key || 'Enter');
+      return { action: 'pressed', key: step.key };
+
+    case RpaActionType.SELECT:
+      if (step.xpath) {
+        await page.locator(`xpath=${step.xpath}`).selectOption(step.value || '');
+      } else if (step.selector) {
+        await page.selectOption(step.selector, step.value || '');
+      }
+      return { action: 'selected', value: step.value };
+
+    case RpaActionType.HOVER:
+      if (step.xpath) {
+        await page.locator(`xpath=${step.xpath}`).hover();
+      } else if (step.selector) {
+        await page.hover(step.selector);
+      }
+      return { action: 'hovered' };
+
+    case RpaActionType.SCREENSHOT:
+      const screenshot = await page.screenshot({ type: 'png' });
+      return {
+        action: 'screenshot',
+        description: step.description,
+        data: screenshot.toString('base64'),
+      };
+
+    default:
+      throw new Error(`Unknown RPA action type: ${step.type}`);
+  }
+}

package/src/nodes/nodeFactory.ts

@@ -12,6 +12,7 @@ import { Agent } from '../agent';
 import { addJumpToNode } from './addJumpToNode';
 import { addJunctionNode } from './addJunctionNode';
 import { addBrowserTaskNode } from './addBrowserTaskNode';
+import { addRpaNode } from './addRpaNode';
 
 export const nodeFactory = ({
   graph,
@@ -51,6 +52,9 @@ export const nodeFactory = ({
     case NodeType.BROWSER_TASK:
       addBrowserTaskNode({ graph, node, agent, llm });
       break;
+    case NodeType.RPA:
+      addRpaNode({ graph, node, tools, emit, agent });
+      break;
     default:
       throw new Error(`Unsupported node type: ${nodeType}`);
   }

package/src/types/Flows.types.ts

@@ -12,6 +12,7 @@ export enum NodeType {
   PROMPT_NODE = 'promptNode',
   JUMP_TO_NODE = 'jumpToNode',
   BROWSER_TASK = 'browserTask',
+  RPA = 'rpa',
 }
 
 export enum EdgeType {
@@ -121,6 +122,42 @@ export interface BrowserTaskNode extends BaseNode {
   proxy?: string; // 2-digit country code like 'IL'
   hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
   onPrem?: boolean;
+  localRun?: boolean;
+}
+
+export enum RpaActionType {
+  CLICK = 'click',
+  TYPE = 'type',
+  WAIT = 'wait',
+  SCREENSHOT = 'screenshot',
+  SELECT = 'select',
+  HOVER = 'hover',
+  PRESS = 'press',
+  GOTO = 'goto',
+}
+
+export interface RpaStep {
+  id: string;
+  type: RpaActionType;
+  xpath?: string;
+  selector?: string;
+  text?: string;
+  value?: string;
+  key?: string;
+  url?: string;
+  shouldReplaceExistingText?: boolean;
+  waitTime?: number;
+  description?: string;
+}
+
+export interface RpaNode extends BaseNode {
+  type: NodeType.RPA;
+  steps: RpaStep[];
+  timeout?: number;
+  viewport?: {
+    width: number;
+    height: number;
+  };
 }
 
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;
@@ -145,7 +182,7 @@ export interface AppToolNode extends BaseNode, BaseAppNode {
   actionKey: string;
 }
 
-export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode;
+export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode | RpaNode;
 
 export interface BaseEdge {
   source: string;