@minded-ai/mindedjs 2.0.9 → 2.0.10

package/src/nodes/addRpaNode.ts

@@ -0,0 +1,199 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, RpaNode, RpaActionType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { chromium, Browser, Page } from 'playwright';
+import { LLMProviders } from '../types/LLM.types';
+import { AIMessage, ToolMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { executeRpaStep } from './rpaStepsExecutor';
+
+type AddRpaNodeParams = {
+  graph: PreCompiledGraph;
+  node: RpaNode;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addRpaNode = async ({ graph, node, agent, llm }: AddRpaNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing RPA node`, node: node.displayName, sessionId: state.sessionId });
+
+    let browser: Browser | null = null;
+    let page: Page | null = null;
+
+    // Create tool call for RPA execution
+    const toolCallId = uuidv4();
+    const aiMessageId = uuidv4();
+
+    // Get CDP URL from state
+    const cdpUrl = state.cdpUrl;
+
+    const toolCall = {
+      id: toolCallId,
+      name: 'rpa-task',
+      args: {
+        steps: node.steps,
+      },
+    };
+
+    // Create AI message with tool call
+    const aiMessage = new AIMessage({
+      id: aiMessageId,
+      content: '',
+      tool_calls: [toolCall],
+      additional_kwargs: {
+        mindedMetadata: {
+          nodeType: NodeType.RPA,
+          nodeDisplayName: node.displayName,
+          sessionId: state.sessionId,
+          cdpUrl: cdpUrl || undefined,
+        },
+      },
+    });
+
+    state.messages.push(aiMessage);
+
+    try {
+      if (!cdpUrl) {
+        throw new Error('CDP URL not found in state. Make sure a browser session is available.');
+      }
+
+      logger.debug({
+        msg: '[RPA] Connecting to browser via CDP',
+        cdpUrl,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Connect to existing browser via CDP
+      browser = await chromium.connectOverCDP(cdpUrl);
+      const contexts = browser.contexts();
+      if (contexts.length === 0) {
+        throw new Error('No browser contexts found');
+      }
+
+      // Get the first page or create a new one
+      const pages = contexts[0].pages();
+      page = pages.length > 0 ? pages[0] : await contexts[0].newPage();
+
+      // Execute each step
+      const results = []; // Collect all extracted data
+
+      for (const [index, step] of node.steps.entries()) {
+        logger.debug({
+          msg: '[RPA] Executing step',
+          stepIndex: index + 1,
+          stepType: step.type,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+
+        try {
+          const result = await executeRpaStep(page, step, state, llm);
+
+          // Collect extracted data
+          if (step.type === RpaActionType.EXTRACT_DATA && result.data) {
+            results.push({
+              stepIndex: index + 1,
+              url: result.url,
+              data: result.data,
+            });
+          }
+        } catch (stepError) {
+          logger.error({
+            msg: '[RPA] Step execution failed',
+            stepIndex: index + 1,
+            stepType: step.type,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          // Stop execution on error unless configured otherwise
+          throw stepError;
+        }
+      }
+
+      // Create tool message with results
+      const toolMessage = new ToolMessage({
+        id: toolCallId,
+        content: JSON.stringify({
+          result: results,
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'success',
+      });
+
+      // Update messages - remove old AI message and add updated one with tool message
+      state.messages.push(toolMessage);
+
+      // Update history with RPA execution results
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            steps: node.steps,
+            results,
+          },
+          messageIds: [aiMessageId, toolMessage.id!],
+        }),
+      );
+
+      return state;
+    } catch (error) {
+      logger.error({
+        msg: '[RPA] Error executing RPA node',
+        error: error instanceof Error ? error.message : 'Unknown error',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Create error tool message
+      const errorToolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'error',
+      });
+
+      // Update messages - remove old AI message and add updated one with error tool message
+      state.messages.push(errorToolMessage);
+
+      // Update history with error
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            error: error instanceof Error ? error.message : 'Unknown error',
+            steps: node.steps,
+          },
+          messageIds: [aiMessageId, errorToolMessage.id!],
+        }),
+      );
+
+      return state;
+    } finally {
+      // Note: We don't close the browser as it's connected via CDP
+      // The browser session should remain active for other operations
+    }
+  };
+
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addToolNode.ts

@@ -34,7 +34,7 @@ export const addToolNode = async ({
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     logger.debug({ msg: `[Node] Executing tool node`, node: toolNode.displayName });
 
-    const tool = langchainTool(() => { }, {
+    const tool = langchainTool(() => {}, {
       name: matchedTool.name,
       description: matchedTool.description,
       schema: matchedTool.input,
@@ -52,7 +52,7 @@ export const addToolNode = async ({
       }
     }
     if (finalMessage) {
-      const compiledPrompt = compilePrompt(finalMessage, { memory: state.memory, env: process.env });
+      const compiledPrompt = compilePrompt(finalMessage, { state: state, memory: state.memory, env: process.env });
       const systemMessage = new SystemMessage(compiledPrompt);
       if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
         state.messages[0] = systemMessage;

package/src/nodes/nodeFactory.ts

@@ -12,6 +12,7 @@ import { Agent } from '../agent';
 import { addJumpToNode } from './addJumpToNode';
 import { addJunctionNode } from './addJunctionNode';
 import { addBrowserTaskNode } from './addBrowserTaskNode';
+import { addRpaNode } from './addRpaNode';
 
 export const nodeFactory = ({
   graph,
@@ -51,6 +52,9 @@ export const nodeFactory = ({
     case NodeType.BROWSER_TASK:
       addBrowserTaskNode({ graph, node, agent, llm });
       break;
+    case NodeType.RPA:
+      addRpaNode({ graph, node, tools, emit, agent, llm });
+      break;
     default:
       throw new Error(`Unsupported node type: ${nodeType}`);
   }

package/src/nodes/rpaStepsExecutor.ts

@@ -0,0 +1,175 @@
+import { Page } from 'playwright';
+import { RpaActionType } from '../types/Flows.types';
+import { LLMProviders } from '../types/LLM.types';
+import { logger } from '../utils/logger';
+import { z } from 'zod';
+import { SystemMessage } from '@langchain/core/messages';
+import { compilePrompt } from './compilePrompt';
+import { stateAnnotation } from '../types/LangGraph.types';
+import { createZodSchemaFromFields } from '../utils/schemaUtils';
+
+// Helper function to execute individual RPA steps
+export async function executeRpaStep(
+  page: Page,
+  step: any,
+  state: typeof stateAnnotation.State,
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders],
+): Promise<any> {
+  // Create params object with memory and secrets
+  const params = {
+    env: process.env,
+    state: state,
+  };
+
+  switch (step.type) {
+    case RpaActionType.CLICK: {
+      // Compile selectors in case they contain placeholders
+      const compiledXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (compiledXpath) {
+        await page.locator(`xpath=${compiledXpath}`).click({ timeout: 5000 });
+      } else if (compiledSelector) {
+        await page.click(compiledSelector, { timeout: 5000 });
+      }
+      return { action: 'clicked' };
+    }
+
+    case RpaActionType.TYPE: {
+      // Compile text and selectors
+      const compiledText = step.text ? compilePrompt(step.text, params) : '';
+      const compiledTypeXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledTypeSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (step.shouldReplaceExistingText) {
+        if (compiledTypeXpath) {
+          await page.locator(`xpath=${compiledTypeXpath}`).fill(compiledText, { timeout: 5000 });
+        } else if (compiledTypeSelector) {
+          await page.fill(compiledTypeSelector, compiledText, { timeout: 5000 });
+        }
+      } else {
+        if (compiledTypeXpath) {
+          await page.locator(`xpath=${compiledTypeXpath}`).type(compiledText, { timeout: 5000 });
+        } else if (compiledTypeSelector) {
+          await page.type(compiledTypeSelector, compiledText, { timeout: 5000 });
+        }
+      }
+      return { action: 'typed', text: compiledText };
+    }
+
+    case RpaActionType.WAIT:
+      await page.waitForTimeout(step.waitTime || 3000);
+      return { action: 'waited', duration: step.waitTime };
+
+    case RpaActionType.GOTO: {
+      // Compile URL
+      const compiledUrl = step.url ? compilePrompt(step.url, params) : '';
+      await page.goto(compiledUrl, { waitUntil: 'load', timeout: 5000 });
+      return { action: 'navigated', url: compiledUrl };
+    }
+
+    case RpaActionType.PRESS: {
+      // Compile key
+      const compiledKey = step.key ? compilePrompt(step.key, params) : 'Enter';
+      await page.keyboard.press(compiledKey);
+      return { action: 'pressed', key: compiledKey };
+    }
+
+    case RpaActionType.SELECT: {
+      // Compile value and selectors
+      const compiledValue = step.value ? compilePrompt(step.value, params) : '';
+      const compiledSelectXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledSelectSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (compiledSelectXpath) {
+        await page.locator(`xpath=${compiledSelectXpath}`).selectOption(compiledValue, { timeout: 5000 });
+      } else if (compiledSelectSelector) {
+        await page.selectOption(compiledSelectSelector, compiledValue, { timeout: 5000 });
+      }
+      return { action: 'selected', value: compiledValue };
+    }
+
+    case RpaActionType.SCREENSHOT: {
+      const screenshot = await page.screenshot({ type: 'png' });
+      // Compile description if provided
+      const compiledDescription = step.description ? compilePrompt(step.description, params) : undefined;
+      return {
+        action: 'screenshot',
+        description: compiledDescription,
+        data: screenshot.toString('base64'),
+      };
+    }
+
+    case RpaActionType.EXTRACT_DATA: {
+      if (!llm) {
+        throw new Error('LLM is required for EXTRACT_DATA action');
+      }
+
+      // Get the page content for extraction
+      const pageContent = await page.content();
+      const url = page.url();
+
+      // Convert outputSchema to Zod schema
+      const zodSchema = createZodSchemaFromFields(step.outputSchema, { result: z.string().describe('The extracted data') });
+
+      const prompt = `Extract data from the following web page according to the specified structure.
+                      Extract the requested data and return it as a structured object.
+                      WEB PAGE CONTENT:
+                      ${pageContent}`;
+
+      try {
+        // Check if LLM supports structured output
+        const supportsStructuredOutput = 'withStructuredOutput' in llm && typeof (llm as any).withStructuredOutput === 'function';
+
+        let extractedData: any;
+
+        if (supportsStructuredOutput) {
+          // Use withStructuredOutput for guaranteed schema compliance
+          const structuredLLM = (llm as any).withStructuredOutput(zodSchema);
+          const messages = [new SystemMessage(prompt)];
+          extractedData = await structuredLLM.invoke(messages);
+        } else {
+          // Fallback to regular LLM call if withStructuredOutput is not available
+          logger.warn({
+            msg: '[RPA] LLM does not support withStructuredOutput, using regular invoke',
+            url,
+          });
+
+          const response = await llm.invoke([new SystemMessage(prompt)]);
+
+          // Try to parse the response as JSON
+          const responseContent = typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
+          extractedData = JSON.parse(responseContent);
+
+          // Validate against schema
+          try {
+            extractedData = zodSchema.parse(extractedData);
+          } catch (validationError) {
+            logger.error({
+              msg: '[RPA] Extracted data validation failed',
+              error: validationError instanceof Error ? validationError.message : 'Unknown error',
+              url,
+            });
+            throw validationError;
+          }
+        }
+
+        return {
+          action: 'extract_data',
+          url,
+          data: extractedData,
+        };
+      } catch (error) {
+        logger.error({
+          msg: '[RPA] Failed to extract data',
+          error: error instanceof Error ? error.message : 'Unknown error',
+          url,
+        });
+        throw error;
+      }
+    }
+
+    default:
+      throw new Error(`Unknown RPA action type: ${step.type}`);
+  }
+}

package/src/types/Flows.types.ts

@@ -12,6 +12,7 @@ export enum NodeType {
   PROMPT_NODE = 'promptNode',
   JUMP_TO_NODE = 'jumpToNode',
   BROWSER_TASK = 'browserTask',
+  RPA = 'rpa',
 }
 
 export enum EdgeType {
@@ -124,6 +125,47 @@ export interface BrowserTaskNode extends BaseNode {
   localRun?: boolean;
 }
 
+export enum RpaActionType {
+  CLICK = 'click',
+  TYPE = 'type',
+  WAIT = 'wait',
+  SCREENSHOT = 'screenshot',
+  SELECT = 'select',
+  PRESS = 'press',
+  GOTO = 'goto',
+  EXTRACT_DATA = 'extract_data',
+}
+
+export interface RpaStep {
+  id: string;
+  type: RpaActionType;
+  xpath?: string;
+  selector?: string;
+  text?: string;
+  value?: string;
+  key?: string;
+  url?: string;
+  shouldReplaceExistingText?: boolean;
+  waitTime?: number;
+  description?: string;
+  outputSchema?: {
+    name: string;
+    type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+    description?: string;
+    required?: boolean;
+    items?: {
+      type: 'string' | 'number' | 'boolean' | 'object';
+      properties?: any;
+    };
+    properties?: any;
+  }[];
+}
+
+export interface RpaNode extends BaseNode {
+  type: NodeType.RPA;
+  steps: RpaStep[];
+}
+
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;
 
 export interface JunctionNode extends BaseNode {
@@ -146,7 +188,7 @@ export interface AppToolNode extends BaseNode, BaseAppNode {
   actionKey: string;
 }
 
-export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode;
+export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode | RpaNode;
 
 export interface BaseEdge {
   source: string;

package/src/utils/schemaUtils.ts

@@ -0,0 +1,68 @@
+import { z } from 'zod';
+
+export interface SchemaField {
+  name: string;
+  type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+  description?: string;
+  required?: boolean;
+}
+
+/**
+ * Converts an array of schema field definitions to a Zod object schema
+ * @param fields Array of field definitions
+ * @param defaultSchema Optional default schema to use if no fields are provided
+ * @returns Zod object schema
+ */
+export function createZodSchemaFromFields(
+  fields?: SchemaField[],
+  defaultSchema?: Record<string, z.ZodTypeAny>
+): z.ZodObject<Record<string, z.ZodTypeAny>> {
+  const schemaFields: Record<string, z.ZodTypeAny> = {};
+
+  if (fields && fields.length > 0) {
+    for (const field of fields) {
+      let fieldSchema: z.ZodTypeAny;
+
+      // Create appropriate Zod type based on field type
+      switch (field.type) {
+        case 'string':
+          fieldSchema = z.string();
+          break;
+        case 'number':
+          fieldSchema = z.number();
+          break;
+        case 'boolean':
+          fieldSchema = z.boolean();
+          break;
+        case 'array':
+          // For arrays, we'll default to array of any unless more specific
+          fieldSchema = z.array(z.any());
+          break;
+        case 'object':
+          // For objects, we'll default to record of any unless more specific
+          fieldSchema = z.record(z.any());
+          break;
+        default:
+          // Default to string for unknown types
+          fieldSchema = z.string();
+      }
+
+      // Add description if available
+      if (field.description) {
+        fieldSchema = fieldSchema.describe(field.description);
+      }
+
+      // Handle optional fields
+      if (field.required === false) {
+        fieldSchema = fieldSchema.optional();
+      }
+
+      schemaFields[field.name] = fieldSchema;
+    }
+  } else if (defaultSchema) {
+    // Use default schema if no fields provided
+    return z.object(defaultSchema);
+  }
+
+  return z.object(schemaFields);
+}