@minded-ai/mindedjs 2.0.9-beta.3 → 2.0.10

package/dist/utils/schemaUtils.d.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+export interface SchemaField {
+    name: string;
+    type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+    description?: string;
+    required?: boolean;
+}
+/**
+ * Converts an array of schema field definitions to a Zod object schema
+ * @param fields Array of field definitions
+ * @param defaultSchema Optional default schema to use if no fields are provided
+ * @returns Zod object schema
+ */
+export declare function createZodSchemaFromFields(fields?: SchemaField[], defaultSchema?: Record<string, z.ZodTypeAny>): z.ZodObject<Record<string, z.ZodTypeAny>>;
+//# sourceMappingURL=schemaUtils.d.ts.map

package/dist/utils/schemaUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemaUtils.d.ts","sourceRoot":"","sources":["../../src/utils/schemaUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,CAAC,EAAE,WAAW,EAAE,EACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,GAC3C,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAiD3C"}

package/dist/utils/schemaUtils.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createZodSchemaFromFields = createZodSchemaFromFields;
+const zod_1 = require("zod");
+/**
+ * Converts an array of schema field definitions to a Zod object schema
+ * @param fields Array of field definitions
+ * @param defaultSchema Optional default schema to use if no fields are provided
+ * @returns Zod object schema
+ */
+function createZodSchemaFromFields(fields, defaultSchema) {
+    const schemaFields = {};
+    if (fields && fields.length > 0) {
+        for (const field of fields) {
+            let fieldSchema;
+            // Create appropriate Zod type based on field type
+            switch (field.type) {
+                case 'string':
+                    fieldSchema = zod_1.z.string();
+                    break;
+                case 'number':
+                    fieldSchema = zod_1.z.number();
+                    break;
+                case 'boolean':
+                    fieldSchema = zod_1.z.boolean();
+                    break;
+                case 'array':
+                    // For arrays, we'll default to array of any unless more specific
+                    fieldSchema = zod_1.z.array(zod_1.z.any());
+                    break;
+                case 'object':
+                    // For objects, we'll default to record of any unless more specific
+                    fieldSchema = zod_1.z.record(zod_1.z.any());
+                    break;
+                default:
+                    // Default to string for unknown types
+                    fieldSchema = zod_1.z.string();
+            }
+            // Add description if available
+            if (field.description) {
+                fieldSchema = fieldSchema.describe(field.description);
+            }
+            // Handle optional fields
+            if (field.required === false) {
+                fieldSchema = fieldSchema.optional();
+            }
+            schemaFields[field.name] = fieldSchema;
+        }
+    }
+    else if (defaultSchema) {
+        // Use default schema if no fields provided
+        return zod_1.z.object(defaultSchema);
+    }
+    return zod_1.z.object(schemaFields);
+}
+//# sourceMappingURL=schemaUtils.js.map

package/dist/utils/schemaUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemaUtils.js","sourceRoot":"","sources":["../../src/utils/schemaUtils.ts"],"names":[],"mappings":";;AAeA,8DAoDC;AAnED,6BAAwB;AASxB;;;;;GAKG;AACH,SAAgB,yBAAyB,CACvC,MAAsB,EACtB,aAA4C;IAE5C,MAAM,YAAY,GAAiC,EAAE,CAAC;IAEtD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,WAAyB,CAAC;YAE9B,kDAAkD;YAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,QAAQ;oBACX,WAAW,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;oBACzB,MAAM;gBACR,KAAK,QAAQ;oBACX,WAAW,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;oBACzB,MAAM;gBACR,KAAK,SAAS;oBACZ,WAAW,GAAG,OAAC,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM;gBACR,KAAK,OAAO;oBACV,iEAAiE;oBACjE,WAAW,GAAG,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;oBAC/B,MAAM;gBACR,KAAK,QAAQ;oBACX,mEAAmE;oBACnE,WAAW,GAAG,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChC,MAAM;gBACR;oBACE,sCAAsC;oBACtC,WAAW,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,CAAC;YAED,+BAA+B;YAC/B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gBACtB,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YAED,yBAAyB;YACzB,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC7B,WAAW,GAAG,WAAW,CAAC,QAAQ,EAAE,CAAC;YACvC,CAAC;YAED,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC;QACzC,CAAC;IACH,CAAC;SAAM,IAAI,aAAa,EAAE,CAAC;QACzB,2CAA2C;QAC3C,OAAO,OAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;AAChC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "2.0.9-beta.3",
+  "version": "2.0.10",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -70,4 +70,4 @@
   "peerDependencies": {
     "playwright": "^1.55.0"
   }
-}
+}

package/src/agent.ts

@@ -465,7 +465,7 @@ export class Agent {
       // Add playbooks to messages
       const combinedPlaybooks = combinePlaybooks(this.playbooks);
       if (combinedPlaybooks) {
-        const compiledPrompt = compilePrompt(combinedPlaybooks, { memory: state.memory });
+        const compiledPrompt = compilePrompt(combinedPlaybooks, { state: state, memory: state.memory, env: process.env });
         const systemMessage = new SystemMessage(compiledPrompt);
         if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
           state.messages[0] = systemMessage;

package/src/browserTask/executeBrowserTask.py

@@ -1,5 +1,9 @@
 import asyncio
-from browser_use import Agent, BrowserSession, BrowserProfile
+import json
+from typing import List, Optional, Dict, Any, TypedDict
+from pydantic import BaseModel, create_model, Field
+from browser_use import Agent, Controller
+from browser_use.browser import BrowserProfile, BrowserSession
 from browser_use.llm import ChatOpenAI
 import os
 import sys
@@ -7,7 +11,39 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-async def main(session_id: str, cdp_url: str, task: str):
+class OutputSchemaItemRequired(TypedDict):
+  name: str
+  type: str  # 'string' | 'number'
+
+class OutputSchemaItem(OutputSchemaItemRequired, total=False):
+  description: str
+  required: bool
+
+
+def create_pydantic_model_from_schema(output_schema: Optional[List[Dict[str, Any]]]) -> Optional[type[BaseModel]]:
+  if not output_schema:
+    return None
+
+  field_definitions: Dict[str, Any] = {}
+  for item in output_schema:
+    type_mapping = {
+      'string': str,
+      'number': float,
+    }
+    field_type = type_mapping.get(item.get('type'), str)
+    description = item.get('description', '')
+    is_required = item.get('required', True)
+    if is_required:
+      field_definitions[item['name']] = (field_type, Field(description=description))
+    else:
+      field_definitions[item['name']] = (Optional[field_type], Field(default=None, description=description))
+
+  if field_definitions:
+    return create_model('DynamicOutputModel', **field_definitions)
+  return None
+
+
+async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Optional[str] = None):
   llm = ChatOpenAI(
     model="gpt-4.1",
     api_key=os.getenv("OPENAI_API_KEY"),
@@ -24,19 +60,52 @@ async def main(session_id: str, cdp_url: str, task: str):
     browser_profile=BrowserProfile(downloads_path=folder_path),
     cdp_url=cdp_url
   )
+
+  output_schema = None
+  if output_schema_json:
+    try:
+      output_schema = json.loads(output_schema_json)
+    except Exception:
+      output_schema = None
+
+  output_model = create_pydantic_model_from_schema(output_schema)
+
   agent = Agent(
     task=task,
     llm=llm,
+    controller=Controller(output_model=output_model) if output_model is not None else Controller(),
     available_file_paths=available_files,
-    browser=browser_session,
+    browser_session=browser_session,
   )
   history = await agent.run()
-  print(history.final_result())
+  final_result = history.final_result()
+  
+      # Parse result with output model if provided
+  parsed_result = None
+  if output_model and final_result:
+      try:
+          # Validate and parse the result using the Pydantic model
+          parsed_result = output_model.model_validate_json(final_result)
+          # Convert to dict for JSON serialization
+          parsed_result = parsed_result.model_dump()
+      except Exception as e:
+          print(f"Failed to parse result with output model: {e}")
+          # Fall back to raw result
+          parsed_result = final_result
+  else:
+      parsed_result = final_result
+  
+  print("___RESULT___")
+  if parsed_result:
+    print(json.dumps(parsed_result, indent=2))
+  else:
+    print(final_result)
     
 if __name__ == '__main__':
-  if len(sys.argv) < 3:
-    raise SystemExit("Usage: uv run uploadG2n.py <session_id> <cdp_url> <task>")
+  if len(sys.argv) < 4:
+    raise SystemExit("Usage: uv run executeBrowserTask.py <session_id> <cdp_url> <task> [output_schema_json]")
   session_id = sys.argv[1]
   cdp_url = sys.argv[2]
   task = sys.argv[3]
-  asyncio.run(main(session_id, cdp_url, task))
+  output_schema_json = sys.argv[4] if len(sys.argv) > 4 else None
+  asyncio.run(main(session_id, cdp_url, task, output_schema_json))

package/src/browserTask/executeBrowserTask.ts

@@ -1,4 +1,4 @@
-import { exec as execCb } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import { logger } from '../utils/logger';
 import { mindedConnection } from '../platform/mindedConnection';
 import path from 'path';
@@ -11,9 +11,7 @@ import {
   DestroyBrowserSessionRequest,
   DestroyBrowserSessionResponse,
 } from '../platform/mindedConnectionTypes';
-import { promisify } from 'node:util';
 import { kill, getOrStartLocalCDP } from './localBrowserTask';
-const exec = promisify(execCb);
 
 // Socket-based browser task functions
 export const createBrowserSession = async ({ sessionId, proxy, onPrem, localRun }: { sessionId: string, proxy?: string, onPrem?: boolean, localRun?: boolean }): Promise<CreateBrowserSessionResponse> => {
@@ -96,30 +94,72 @@ export const invokeBrowserTask = async (
     required?: boolean;
   }[],
 ): Promise<InvokeBrowserTaskResponse> => {
-  logger.debug({
-    msg: 'Invoking browser task via socket',
-    sessionId,
-    taskLength: task.length,
-    keepAlive,
-    hooksCount: hooks?.length || 0,
-    onPrem,
-    outputSchemaFields: outputSchema?.length || 0,
-  });
 
   try {
 
     if (localRun) {
       const pythonScriptPath = path.resolve(__dirname, 'executeBrowserTask.py');
-      const command = `uv run ${pythonScriptPath} "${sessionId}" "${cdpUrl}" "${task.replace(/"/g, '\\"')}"`;
-      const { stdout, stderr } = await exec(command);
-      logger.info({ message: 'Operator finished', stdout, stderr });
+      const outputSchemaArg = JSON.stringify(outputSchema || []);
+
+      const args = ['run', pythonScriptPath, sessionId, cdpUrl, task, outputSchemaArg];
+
+      logger.info({ message: 'Spawning Python process', args });
+
+      const child = spawn('uv', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+
+      let stdoutBuffer = '';
+      let stderrBuffer = '';
+
+      child.stdout.on('data', (data) => {
+        const text = data.toString();
+        stdoutBuffer += text;
+        process.stdout.write(text);
+      });
+
+      child.stderr.on('data', (data) => {
+        const text = data.toString();
+        stderrBuffer += text;
+        process.stderr.write(text);
+      });
+
+      const exitCode: number = await new Promise((resolve, reject) => {
+        child.on('error', (err) => reject(err));
+        child.on('close', (code) => resolve(code ?? 1));
+      });
+
+      if (exitCode !== 0) {
+        logger.error({ message: 'Operator failed', exitCode, stderr: stderrBuffer });
+        throw new Error(`Local browser task failed with exit code ${exitCode}`);
+      }
+
+      logger.info({ message: 'Operator finished' });
+
+      let result = stdoutBuffer.split('___RESULT___')[1]?.trim() || stdoutBuffer;
+      if (outputSchema?.length) {
+        try {
+          result = JSON.parse(result);
+        } catch (error) {
+          logger.debug({ message: 'Failed to parse result', error });
+        }
+      }
+
       return {
-        result: stdout,
+        result,
         steps: [],
         recordings: [],
       };
     }
 
+    logger.debug({
+      msg: 'Invoking browser task via socket',
+      sessionId,
+      taskLength: task.length,
+      keepAlive,
+      hooksCount: hooks?.length || 0,
+      onPrem,
+      outputSchemaFields: outputSchema?.length || 0,
+    });
+
     const response = await mindedConnection.awaitEmit<InvokeBrowserTaskRequest, InvokeBrowserTaskResponse>(
       mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
       {

package/src/edges/createPromptRouter.ts

@@ -111,17 +111,19 @@ export const createPromptRouter = ({
       steps: stepsStr,
       messages: messagesStr,
       memory: state.memory,
+      state: state,
+      env: process.env,
     });
 
     // Define response schema
     const responseSchema = includeReasoning
       ? z.object({
-        nextNodeId: z.string(),
-        reasoning: z.string(),
-      })
+          nextNodeId: z.string(),
+          reasoning: z.string(),
+        })
       : z.object({
-        nextNodeId: z.string(),
-      });
+          nextNodeId: z.string(),
+        });
 
     let attempts = 0;
     let lastError: Error | null = null;

package/src/index.ts

@@ -48,6 +48,8 @@ export type {
   VoiceTriggerNode,
   WebhookTriggerNode,
   InterfaceTriggerNode,
+  RpaNode,
+  RpaStep,
 } from './types/Flows.types';
 export {
   NodeType,
@@ -57,6 +59,7 @@ export {
   AppNodeMetadataType,
   NodeMetadata,
   KnownTriggerNames,
+  RpaActionType,
 } from './types/Flows.types';
 export type { Tool, ToolExecuteInput } from './types/Tools.types';
 

package/src/nodes/addAppToolNode.ts

@@ -70,7 +70,7 @@ export const addAppToolNode = async ({
     User instructions for choosing tool parameters are:
         ${node.prompt ? `${node.prompt}` : 'no instructions set by the user'}`;
 
-    const compiledPrompt = compilePrompt(message, { memory: state.memory });
+    const compiledPrompt = compilePrompt(message, { state: state, memory: state.memory, env: process.env });
     const systemMessage = new SystemMessage(compiledPrompt);
     if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
       state.messages[0] = systemMessage;

package/src/nodes/addBrowserTaskNode.ts

@@ -9,10 +9,10 @@ import { v4 as uuidv4 } from 'uuid';
 import { Agent } from '../agent';
 import { createBrowserSession } from '../browserTask/executeBrowserTask';
 import { tool as langchainTool } from '@langchain/core/tools';
-import { z } from 'zod';
 import { LLMProviders } from '../types/LLM.types';
 import { compilePrompt } from './compilePrompt';
 import { combinePlaybooks } from '../playbooks/playbooks';
+import { createZodSchemaFromFields } from '../utils/schemaUtils';
 
 type AddBrowserTaskNodeParams = {
   graph: PreCompiledGraph;
@@ -26,35 +26,10 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
     logger.info({ msg: `Executing browser task node ${node.displayName}`, prompt: node.prompt });
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     // Create Zod schema from inputSchema
-    const schemaFields: Record<string, z.ZodTypeAny> = {};
-    if (node.inputSchema) {
-      for (const field of node.inputSchema) {
-        let fieldSchema: z.ZodTypeAny;
-
-        if (field.type === 'string') {
-          fieldSchema = z.string();
-        } else if (field.type === 'number') {
-          fieldSchema = z.number();
-        } else {
-          fieldSchema = z.string(); // Default to string
-        }
-
-        if (field.description) {
-          fieldSchema = fieldSchema.describe(field.description);
-        }
-
-        if (field.required === false) {
-          fieldSchema = fieldSchema.optional();
-        }
-
-        schemaFields[field.name] = fieldSchema;
-      }
-    }
-
-    const zodSchema = z.object(schemaFields);
+    const zodSchema = createZodSchemaFromFields(node.inputSchema);
 
     // Create langchain tool
-    const tool = langchainTool(() => { }, {
+    const tool = langchainTool(() => {}, {
       name: 'browser-task',
       description: node.prompt,
       schema: zodSchema,
@@ -62,7 +37,7 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
 
     const combinedPlaybooks = combinePlaybooks(agent.playbooks);
     if (combinedPlaybooks) {
-      const compiledPrompt = compilePrompt(combinedPlaybooks, { memory: state.memory });
+      const compiledPrompt = compilePrompt(combinedPlaybooks, { state: state, memory: state.memory, env: process.env });
       const systemMessage = new SystemMessage(compiledPrompt);
       if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
         state.messages[0] = systemMessage;
@@ -86,11 +61,12 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
       const promptParams = {
         input: inputParams,
         memory: state.memory,
+        state: state,
         system: { currentTime: new Date().toISOString() },
       };
 
       // Compile the prompt with parameters
-      const compiledPrompt = compilePrompt(node.prompt, promptParams);
+      const compiledPrompt = compilePrompt(node.prompt, { ...promptParams, env: process.env });
 
       // Build the full prompt with compiled content
       const fullPrompt = `
@@ -145,6 +121,7 @@ ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(i
             keepAlive,
             hooks: node.hooks || [],
             onPrem: node.onPrem,
+            localRun: node.localRun,
             outputSchema: node.outputSchema,
           },
         },

package/src/nodes/addPromptNode.ts

@@ -51,7 +51,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
     if (combinedPlaybooks) {
       finalMessage = combinedPlaybooks + '\n\n' + currentPromptNode;
     }
-    const compiledPrompt = compilePrompt(finalMessage, { memory: state.memory });
+    const compiledPrompt = compilePrompt(finalMessage, { state: state, memory: state.memory, env: process.env });
     const systemMessage = new SystemMessage(compiledPrompt);
     if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
       state.messages[0] = systemMessage;

package/src/nodes/addRpaNode.ts

@@ -0,0 +1,199 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, RpaNode, RpaActionType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { chromium, Browser, Page } from 'playwright';
+import { LLMProviders } from '../types/LLM.types';
+import { AIMessage, ToolMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { executeRpaStep } from './rpaStepsExecutor';
+
+type AddRpaNodeParams = {
+  graph: PreCompiledGraph;
+  node: RpaNode;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addRpaNode = async ({ graph, node, agent, llm }: AddRpaNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing RPA node`, node: node.displayName, sessionId: state.sessionId });
+
+    let browser: Browser | null = null;
+    let page: Page | null = null;
+
+    // Create tool call for RPA execution
+    const toolCallId = uuidv4();
+    const aiMessageId = uuidv4();
+
+    // Get CDP URL from state
+    const cdpUrl = state.cdpUrl;
+
+    const toolCall = {
+      id: toolCallId,
+      name: 'rpa-task',
+      args: {
+        steps: node.steps,
+      },
+    };
+
+    // Create AI message with tool call
+    const aiMessage = new AIMessage({
+      id: aiMessageId,
+      content: '',
+      tool_calls: [toolCall],
+      additional_kwargs: {
+        mindedMetadata: {
+          nodeType: NodeType.RPA,
+          nodeDisplayName: node.displayName,
+          sessionId: state.sessionId,
+          cdpUrl: cdpUrl || undefined,
+        },
+      },
+    });
+
+    state.messages.push(aiMessage);
+
+    try {
+      if (!cdpUrl) {
+        throw new Error('CDP URL not found in state. Make sure a browser session is available.');
+      }
+
+      logger.debug({
+        msg: '[RPA] Connecting to browser via CDP',
+        cdpUrl,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Connect to existing browser via CDP
+      browser = await chromium.connectOverCDP(cdpUrl);
+      const contexts = browser.contexts();
+      if (contexts.length === 0) {
+        throw new Error('No browser contexts found');
+      }
+
+      // Get the first page or create a new one
+      const pages = contexts[0].pages();
+      page = pages.length > 0 ? pages[0] : await contexts[0].newPage();
+
+      // Execute each step
+      const results = []; // Collect all extracted data
+
+      for (const [index, step] of node.steps.entries()) {
+        logger.debug({
+          msg: '[RPA] Executing step',
+          stepIndex: index + 1,
+          stepType: step.type,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+
+        try {
+          const result = await executeRpaStep(page, step, state, llm);
+
+          // Collect extracted data
+          if (step.type === RpaActionType.EXTRACT_DATA && result.data) {
+            results.push({
+              stepIndex: index + 1,
+              url: result.url,
+              data: result.data,
+            });
+          }
+        } catch (stepError) {
+          logger.error({
+            msg: '[RPA] Step execution failed',
+            stepIndex: index + 1,
+            stepType: step.type,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          // Stop execution on error unless configured otherwise
+          throw stepError;
+        }
+      }
+
+      // Create tool message with results
+      const toolMessage = new ToolMessage({
+        id: toolCallId,
+        content: JSON.stringify({
+          result: results,
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'success',
+      });
+
+      // Update messages - remove old AI message and add updated one with tool message
+      state.messages.push(toolMessage);
+
+      // Update history with RPA execution results
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            steps: node.steps,
+            results,
+          },
+          messageIds: [aiMessageId, toolMessage.id!],
+        }),
+      );
+
+      return state;
+    } catch (error) {
+      logger.error({
+        msg: '[RPA] Error executing RPA node',
+        error: error instanceof Error ? error.message : 'Unknown error',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Create error tool message
+      const errorToolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'error',
+      });
+
+      // Update messages - remove old AI message and add updated one with error tool message
+      state.messages.push(errorToolMessage);
+
+      // Update history with error
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            error: error instanceof Error ? error.message : 'Unknown error',
+            steps: node.steps,
+          },
+          messageIds: [aiMessageId, errorToolMessage.id!],
+        }),
+      );
+
+      return state;
+    } finally {
+      // Note: We don't close the browser as it's connected via CDP
+      // The browser session should remain active for other operations
+    }
+  };
+
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addToolNode.ts

@@ -52,7 +52,7 @@ export const addToolNode = async ({
       }
     }
     if (finalMessage) {
-      const compiledPrompt = compilePrompt(finalMessage, { memory: state.memory });
+      const compiledPrompt = compilePrompt(finalMessage, { state: state, memory: state.memory, env: process.env });
       const systemMessage = new SystemMessage(compiledPrompt);
       if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
         state.messages[0] = systemMessage;