@minded-ai/mindedjs 2.0.7 → 2.0.8-beta-1

package/dist/types/Flows.types.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.internalNodesSuffix = exports.KnownTriggerNames = exports.ActionInputParamType = exports.AppNodeMetadataType = exports.TriggerType = exports.EdgeType = exports.NodeType = void 0;
+exports.internalNodesSuffix = exports.KnownTriggerNames = exports.ActionInputParamType = exports.AppNodeMetadataType = exports.RpaActionType = exports.TriggerType = exports.EdgeType = exports.NodeType = void 0;
 var NodeType;
 (function (NodeType) {
     NodeType["TRIGGER"] = "trigger";
@@ -10,6 +10,7 @@ var NodeType;
     NodeType["PROMPT_NODE"] = "promptNode";
     NodeType["JUMP_TO_NODE"] = "jumpToNode";
     NodeType["BROWSER_TASK"] = "browserTask";
+    NodeType["RPA"] = "rpa";
 })(NodeType || (exports.NodeType = NodeType = {}));
 var EdgeType;
 (function (EdgeType) {
@@ -25,6 +26,17 @@ var TriggerType;
     TriggerType["VOICE"] = "voice";
     TriggerType["INTERFACE"] = "interface";
 })(TriggerType || (exports.TriggerType = TriggerType = {}));
+var RpaActionType;
+(function (RpaActionType) {
+    RpaActionType["CLICK"] = "click";
+    RpaActionType["TYPE"] = "type";
+    RpaActionType["WAIT"] = "wait";
+    RpaActionType["SCREENSHOT"] = "screenshot";
+    RpaActionType["SELECT"] = "select";
+    RpaActionType["HOVER"] = "hover";
+    RpaActionType["PRESS"] = "press";
+    RpaActionType["GOTO"] = "goto";
+})(RpaActionType || (exports.RpaActionType = RpaActionType = {}));
 var AppNodeMetadataType;
 (function (AppNodeMetadataType) {
     AppNodeMetadataType["APP_TOOL"] = "appTool";

package/dist/types/Flows.types.js.map

@@ -1 +1 @@
-{"version":3,"file":"Flows.types.js","sourceRoot":"","sources":["../../src/types/Flows.types.ts"],"names":[],"mappings":";;;AAMA,IAAY,QAQX;AARD,WAAY,QAAQ;IAClB,+BAAmB,CAAA;IACnB,iCAAqB,CAAA;IACrB,yBAAa,CAAA;IACb,gCAAoB,CAAA;IACpB,sCAA0B,CAAA;IAC1B,uCAA2B,CAAA;IAC3B,wCAA4B,CAAA;AAC9B,CAAC,EARW,QAAQ,wBAAR,QAAQ,QAQnB;AAED,IAAY,QAIX;AAJD,WAAY,QAAQ;IAClB,kDAAsC,CAAA;IACtC,gDAAoC,CAAA;IACpC,wCAA4B,CAAA;AAC9B,CAAC,EAJW,QAAQ,wBAAR,QAAQ,QAInB;AAWD,IAAY,WAMX;AAND,WAAY,WAAW;IACrB,0BAAW,CAAA;IACX,kCAAmB,CAAA;IACnB,gCAAiB,CAAA;IACjB,8BAAe,CAAA;IACf,sCAAuB,CAAA;AACzB,CAAC,EANW,WAAW,2BAAX,WAAW,QAMtB;AAsID,IAAY,mBAEX;AAFD,WAAY,mBAAmB;IAC7B,2CAAoB,CAAA;AACtB,CAAC,EAFW,mBAAmB,mCAAnB,mBAAmB,QAE9B;AAsBD,IAAY,oBAOX;AAPD,WAAY,oBAAoB;IAC9B,yCAAiB,CAAA;IACjB,yCAAiB,CAAA;IACjB,2CAAmB,CAAA;IACnB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,mCAAW,CAAA;AACb,CAAC,EAPW,oBAAoB,oCAApB,oBAAoB,QAO/B;AAQD,IAAY,iBAIX;AAJD,WAAY,iBAAiB;IAC3B,wCAAmB,CAAA;IACnB,4DAAuC,CAAA;IACvC,oDAA+B,CAAA;AACjC,CAAC,EAJW,iBAAiB,iCAAjB,iBAAiB,QAI5B;AAED,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,4DAAqC,CAAA;IACrC,4CAAqB,CAAA;IACrB,2DAAoC,CAAA;AACtC,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B"}
+{"version":3,"file":"Flows.types.js","sourceRoot":"","sources":["../../src/types/Flows.types.ts"],"names":[],"mappings":";;;AAMA,IAAY,QASX;AATD,WAAY,QAAQ;IAClB,+BAAmB,CAAA;IACnB,iCAAqB,CAAA;IACrB,yBAAa,CAAA;IACb,gCAAoB,CAAA;IACpB,sCAA0B,CAAA;IAC1B,uCAA2B,CAAA;IAC3B,wCAA4B,CAAA;IAC5B,uBAAW,CAAA;AACb,CAAC,EATW,QAAQ,wBAAR,QAAQ,QASnB;AAED,IAAY,QAIX;AAJD,WAAY,QAAQ;IAClB,kDAAsC,CAAA;IACtC,gDAAoC,CAAA;IACpC,wCAA4B,CAAA;AAC9B,CAAC,EAJW,QAAQ,wBAAR,QAAQ,QAInB;AAWD,IAAY,WAMX;AAND,WAAY,WAAW;IACrB,0BAAW,CAAA;IACX,kCAAmB,CAAA;IACnB,gCAAiB,CAAA;IACjB,8BAAe,CAAA;IACf,sCAAuB,CAAA;AACzB,CAAC,EANW,WAAW,2BAAX,WAAW,QAMtB;AAyFD,IAAY,aASX;AATD,WAAY,aAAa;IACvB,gCAAe,CAAA;IACf,8BAAa,CAAA;IACb,8BAAa,CAAA;IACb,0CAAyB,CAAA;IACzB,kCAAiB,CAAA;IACjB,gCAAe,CAAA;IACf,gCAAe,CAAA;IACf,8BAAa,CAAA;AACf,CAAC,EATW,aAAa,6BAAb,aAAa,QASxB;AAwED,IAAY,mBAEX;AAFD,WAAY,mBAAmB;IAC7B,2CAAoB,CAAA;AACtB,CAAC,EAFW,mBAAmB,mCAAnB,mBAAmB,QAE9B;AAsBD,IAAY,oBAOX;AAPD,WAAY,oBAAoB;IAC9B,yCAAiB,CAAA;IACjB,yCAAiB,CAAA;IACjB,2CAAmB,CAAA;IACnB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,mCAAW,CAAA;AACb,CAAC,EAPW,oBAAoB,oCAApB,oBAAoB,QAO/B;AAQD,IAAY,iBAIX;AAJD,WAAY,iBAAiB;IAC3B,wCAAmB,CAAA;IACnB,4DAAuC,CAAA;IACvC,oDAA+B,CAAA;AACjC,CAAC,EAJW,iBAAiB,iCAAjB,iBAAiB,QAI5B;AAED,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,4DAAqC,CAAA;IACrC,4CAAqB,CAAA;IACrB,2DAAoC,CAAA;AACtC,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B"}

package/dist/utils/extractStateMemoryResponse.d.ts

@@ -0,0 +1,5 @@
+import { ToolMessage } from '@langchain/core/messages';
+import { State } from '../types/LangGraph.types';
+declare const extractToolStateResponse: (toolMessage: ToolMessage) => Partial<State>;
+export default extractToolStateResponse;
+//# sourceMappingURL=extractStateMemoryResponse.d.ts.map

package/dist/utils/extractStateMemoryResponse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractStateMemoryResponse.d.ts","sourceRoot":"","sources":["../../src/utils/extractStateMemoryResponse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,WAAW,EAA0C,MAAM,0BAA0B,CAAC;AAE5G,OAAO,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AAEjD,QAAA,MAAM,wBAAwB,gBAAiB,WAAW,KAAG,OAAO,CAAC,KAAK,CAazE,CAAC;AA4EF,eAAe,wBAAwB,CAAC"}

package/dist/utils/extractStateMemoryResponse.js

@@ -0,0 +1,91 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const messages_1 = require("@langchain/core/messages");
+const logger_1 = require("./logger");
+const extractToolStateResponse = (toolMessage) => {
+    try {
+        const parsed = JSON.parse(toolMessage.content);
+        if (typeof parsed === 'object' && parsed !== null && 'state' in parsed) {
+            if (parsed.state.messages) {
+                parsed.state.messages = constructMessagesFromToolMessage(parsed.state.messages);
+            }
+            return parsed.state;
+        }
+    }
+    catch (err) {
+        logger_1.logger.error({ message: 'Error parsing tool state response', err });
+    }
+    return {};
+};
+const constructMessagesFromToolMessage = (messages) => {
+    if (!Array.isArray(messages)) {
+        logger_1.logger.error({ msg: 'Expected messages to be an array', messages });
+        return [];
+    }
+    return messages
+        .map((messageObj) => {
+        try {
+            // Handle case where message is already a BaseMessage instance
+            if (messageObj instanceof messages_1.BaseMessage) {
+                return messageObj;
+            }
+            // Handle case where message is a plain object that needs reconstruction
+            const messageType = messageObj.id[2];
+            const content = messageObj.kwargs.content || '';
+            const id = messageObj.kwargs.id;
+            const additionalKwargs = messageObj.additional_kwargs || {};
+            switch (messageType) {
+                case 'AIMessage':
+                    return new messages_1.AIMessage({
+                        content,
+                        id,
+                        additional_kwargs: additionalKwargs,
+                        tool_calls: messageObj.tool_calls || [],
+                        invalid_tool_calls: messageObj.invalid_tool_calls || [],
+                    });
+                case 'HumanMessage':
+                    return new messages_1.HumanMessage({
+                        content,
+                        id,
+                        additional_kwargs: additionalKwargs,
+                    });
+                case 'SystemMessage':
+                    return new messages_1.SystemMessage({
+                        content,
+                        id,
+                        additional_kwargs: additionalKwargs,
+                    });
+                case 'ToolMessage':
+                    return new messages_1.ToolMessage({
+                        content,
+                        id,
+                        additional_kwargs: additionalKwargs,
+                        tool_call_id: messageObj.tool_call_id || '',
+                    });
+                default:
+                    // Default to HumanMessage for unknown types
+                    logger_1.logger.warn({
+                        msg: 'Unknown message type, defaulting to HumanMessage',
+                        messageType,
+                        messageObj,
+                    });
+                    return null;
+            }
+        }
+        catch (err) {
+            logger_1.logger.error({
+                msg: 'Error reconstructing message',
+                err,
+                messageObj,
+            });
+            // Return a fallback HumanMessage in case of error
+            return new messages_1.HumanMessage({
+                content: messageObj.content || 'Error reconstructing message',
+                id: messageObj.id,
+            });
+        }
+    })
+        .filter(Boolean);
+};
+exports.default = extractToolStateResponse;
+//# sourceMappingURL=extractStateMemoryResponse.js.map

package/dist/utils/extractStateMemoryResponse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractStateMemoryResponse.js","sourceRoot":"","sources":["../../src/utils/extractStateMemoryResponse.ts"],"names":[],"mappings":";;AAAA,uDAA4G;AAC5G,qCAAkC;AAGlC,MAAM,wBAAwB,GAAG,CAAC,WAAwB,EAAkB,EAAE;IAC5E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAiB,CAAC,CAAC;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC1B,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,gCAAgC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClF,CAAC;YACD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,mCAAmC,EAAE,GAAG,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,gCAAgC,GAAG,CAAC,QAAe,EAAiB,EAAE;IAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,kCAAkC,EAAE,QAAQ,EAAE,CAAC,CAAC;QACpE,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QAClB,IAAI,CAAC;YACH,8DAA8D;YAC9D,IAAI,UAAU,YAAY,sBAAW,EAAE,CAAC;gBACtC,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,wEAAwE;YACxE,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YAChD,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,gBAAgB,GAAG,UAAU,CAAC,iBAAiB,IAAI,EAAE,CAAC;YAE5D,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,WAAW;oBACd,OAAO,IAAI,oBAAS,CAAC;wBACnB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;wBACnC,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;wBACvC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,IAAI,EAAE;qBACxD,CAAC,CAAC;gBACL,KAAK,cAAc;oBACjB,OAAO,IAAI,uBAAY,CAAC;wBACtB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;qBACpC,CAAC,CAAC;gBACL,KAAK,eAAe;oBAClB,OAAO,IAAI,wBAAa,CAAC;wBACvB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;qBACpC,CAAC,CAAC;gBACL,KAAK,aAAa;oBAChB,OAAO,IAAI,sBAAW,CAAC;wBACrB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;wBACnC,YAAY,EAAE,UAAU,CAAC,YAAY,IAAI,EAAE;qBAC5C,CAAC,CAAC;gBAEL;oBACE,4CAA4C;oBAC5C,eAAM,CAAC,IAAI,CAAC;wBACV,GAAG,EAAE,kDAAkD;wBACvD,WAAW;wBACX,UAAU;qBACX,CAAC,CAAC;oBACH,OAAO,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAM,CAAC,KAAK,CAAC;gBACX,GAAG,EAAE,8BAA8B;gBACnC,GAAG;gBACH,UAAU;aACX,CAAC,CAAC;YACH,kDAAkD;YAClD,OAAO,IAAI,uBAAY,CAAC;gBACtB,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,8BAA8B;gBAC7D,EAAE,EAAE,UAAU,CAAC,EAAE;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAkB,CAAC;AACtC,CAAC,CAAC;AAEF,kBAAe,wBAAwB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "2.0.7",
+  "version": "2.0.8-beta-1",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -13,7 +13,7 @@
     "docs"
   ],
   "scripts": {
-    "build": "tsc && npm run copy-templates",
+    "build": "tsc && npm run copy-templates && rsync -a --include='*/' --include='*.py' --exclude='*' src/ dist/",
     "copy-templates": "cp -r src/cli/lambdaHandlerTemplate.ts dist/cli/",
     "watch": "tsc -w",
     "test": "NODE_ENV=test mocha --parallel --jobs auto --retries 1 -r ts-node/register -r ./test/setup.ts \"test/**/*.test.ts\"",
@@ -66,5 +66,8 @@
     "ws": "^8.15.1",
     "zod": "^3.25.74",
     "zod-to-json-schema": "^3.24.6"
+  },
+  "peerDependencies": {
+    "playwright": "^1.55.0"
   }
 }

package/src/browserTask/executeBrowserTask.py

@@ -0,0 +1,42 @@
+import asyncio
+from browser_use import Agent, BrowserSession, BrowserProfile
+from browser_use.llm import ChatOpenAI
+import os
+import sys
+from dotenv import load_dotenv
+
+load_dotenv()
+
+async def main(session_id: str, cdp_url: str, task: str):
+  llm = ChatOpenAI(
+    model="gpt-4.1",
+    api_key=os.getenv("OPENAI_API_KEY"),
+  )
+  folder_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'downloads')
+  
+  # Create folder if it doesn't exist
+  os.makedirs(folder_path, exist_ok=True)
+  
+  available_files = [os.path.join(folder_path, file) for file in os.listdir(folder_path)]
+  
+  # Set downloads_path to the local tools/files directory
+  browser_session = BrowserSession(
+    browser_profile=BrowserProfile(downloads_path=folder_path),
+    cdp_url=cdp_url
+  )
+  agent = Agent(
+    task=task,
+    llm=llm,
+    available_file_paths=available_files,
+    browser=browser_session,
+  )
+  history = await agent.run()
+  print(history.final_result())
+    
+if __name__ == '__main__':
+  if len(sys.argv) < 3:
+    raise SystemExit("Usage: uv run uploadG2n.py <session_id> <cdp_url> <task>")
+  session_id = sys.argv[1]
+  cdp_url = sys.argv[2]
+  task = sys.argv[3]
+  asyncio.run(main(session_id, cdp_url, task))

package/src/browserTask/executeBrowserTask.ts

@@ -1,5 +1,7 @@
+import { exec as execCb } from 'node:child_process';
 import { logger } from '../utils/logger';
 import { mindedConnection } from '../platform/mindedConnection';
+import path from 'path';
 import {
   mindedConnectionSocketMessageType,
   CreateBrowserSessionResponse,
@@ -9,11 +11,22 @@ import {
   DestroyBrowserSessionRequest,
   DestroyBrowserSessionResponse,
 } from '../platform/mindedConnectionTypes';
+import { promisify } from 'node:util';
+import { kill, getOrStartLocalCDP } from './localBrowserTask';
+const exec = promisify(execCb);
 
 // Socket-based browser task functions
-export const createBrowserSession = async (proxy?: string, onPrem?: boolean): Promise<CreateBrowserSessionResponse> => {
+export const createBrowserSession = async ({ sessionId, proxy, onPrem, localRun }: { sessionId: string, proxy?: string, onPrem?: boolean, localRun?: boolean }): Promise<CreateBrowserSessionResponse> => {
   logger.debug({ msg: 'Creating browser session via socket', proxy });
 
+  if (localRun) {
+    const { cdpUrl } = await getOrStartLocalCDP({ headless: false });
+    return {
+      sessionId,
+      cdpUrl
+    }
+  }
+
   const response = await mindedConnection.awaitEmit<CreateBrowserSessionRequest, CreateBrowserSessionResponse>(
     mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION,
     {
@@ -38,9 +51,16 @@ export const createBrowserSession = async (proxy?: string, onPrem?: boolean): Pr
   return response;
 };
 
-export const destroyBrowserSession = async (sessionId: string, onPrem?: boolean): Promise<DestroyBrowserSessionResponse> => {
+export const destroyBrowserSession = async ({ sessionId, onPrem, localRun }: { sessionId: string, onPrem?: boolean, localRun?: boolean }): Promise<DestroyBrowserSessionResponse> => {
   logger.debug({ msg: 'Destroying browser session via socket', sessionId, onPrem });
 
+  if (localRun) {
+    await kill();
+    return {
+      success: true,
+    };
+  }
+
   const response = await mindedConnection.awaitEmit<DestroyBrowserSessionRequest, DestroyBrowserSessionResponse>(
     mindedConnectionSocketMessageType.DESTROY_BROWSER_SESSION,
     {
@@ -67,6 +87,7 @@ export const invokeBrowserTask = async (
   keepAlive?: boolean,
   hooks?: { name: string }[],
   onPrem?: boolean,
+  localRun?: boolean,
   toolSchemas?: any[],
   outputSchema?: {
     name: string;
@@ -86,6 +107,19 @@ export const invokeBrowserTask = async (
   });
 
   try {
+
+    if (localRun) {
+      const pythonScriptPath = path.resolve(__dirname, 'executeBrowserTask.py');
+      const command = `uv run ${pythonScriptPath} "${sessionId}" "${cdpUrl}" "${task.replace(/"/g, '\\"')}"`;
+      const { stdout, stderr } = await exec(command);
+      logger.info({ message: 'Operator finished', stdout, stderr });
+      return {
+        result: stdout,
+        steps: [],
+        recordings: [],
+      };
+    }
+
     const response = await mindedConnection.awaitEmit<InvokeBrowserTaskRequest, InvokeBrowserTaskResponse>(
       mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
       {

package/src/browserTask/localBrowserTask.ts

@@ -0,0 +1,250 @@
+// getOrStartLocalCDP.ts
+import { spawn, ChildProcess } from "child_process";
+import * as fs from "fs/promises";
+import * as fscb from "fs";
+import * as path from "path";
+import * as http from "http";
+import { chromium } from "playwright";
+import { logger } from "../utils/logger";
+
+let localBrowserTask: {
+  cdpUrl: string;
+  proc: ChildProcess;
+} | null = null;
+
+export type StartChromiumOptions = {
+  /** Headless by default. Set to false to see a window. */
+  headless?: boolean;
+  /** Extra CLI flags to pass to Chromium. */
+  extraArgs?: string[];
+  /** Provide your own user-data-dir. If omitted, a temp dir is created and later removable via `kill()`. */
+  userDataDir?: string;
+  /** Extra env vars. */
+  env?: NodeJS.ProcessEnv;
+  /** How long to wait for the DevTools port to come up (ms). Default: 10000 */
+  timeoutMs?: number;
+};
+
+/**
+ * Launch Chromium with --remote-debugging-port=0 and return the CDP ws URL.
+ * Works with Chrome/Chromium/Chrome for Testing.
+ */
+export async function getOrStartLocalCDP(
+  opts: StartChromiumOptions
+): Promise<{ cdpUrl: string }> {
+
+  if (localBrowserTask) {
+    return {
+      cdpUrl: localBrowserTask.cdpUrl,
+    };
+  }
+
+  const {
+    env,
+    timeoutMs = 10_000,
+  } = opts;
+
+  const executablePath = chromium.executablePath();
+
+  const profileDir = path.join(__dirname, "profile");
+  await fs.mkdir(profileDir, { recursive: true });
+
+  const userDataDir = opts.userDataDir ?? profileDir;
+
+  const args = [
+    "--new-window",
+    `--user-data-dir=${userDataDir}`,
+    // "--headless",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-background-networking",
+    "--disable-back-forward-cache",
+    "--disable-popup-blocking",
+    "--simulate-outdated-no-au=\"Tue, 31 Dec 2099 23:59:59 GMT\"",
+    "--disable-renderer-backgrounding",
+    "--no-service-autorun",
+    "--disable-background-timer-throttling",
+    "--allow-legacy-extension-manifests",
+    "--allow-pre-commit-input",
+    "--log-level=2",
+    "--unsafely-disable-devtools-self-xss-warnings",
+    "--metrics-recording-only",
+    "--disable-window-activation",
+    "--disable-dev-shm-usage",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-search-engine-choice-screen",
+    "--disable-print-preview",
+    "--disable-external-intent-requests",
+    "--disable-desktop-notifications",
+    "--disable-component-update",
+    "--generate-pdf-document-outline",
+    "--disable-focus-on-load",
+    "--disable-speech-api",
+    "--silent-debugger-extension-api",
+    "--suppress-message-center-popups",
+    "--disable-default-apps",
+    "--enable-network-information-downlink-max",
+    "--use-mock-keychain",
+    "--disable-ipc-flooding-protection",
+    "--safebrowsing-disable-auto-update",
+    "--install-autogenerated-theme=0,0,0",
+    "--disable-blink-features=AutomationControlled",
+    "--no-first-run",
+    "--extensions-on-chrome-urls",
+    "--ash-no-nudges",
+    "--no-pings",
+    "--test-type=gpu",
+    "--disable-client-side-phishing-detection",
+    "--password-store=basic",
+    "--enable-features=NetworkService,NetworkServiceInProcess",
+    "--disable-extensions-http-throttling",
+    "--disable-speech-synthesis-api",
+    "--no-default-browser-check",
+    "--disable-hang-monitor",
+    "--hide-crash-restore-bubble",
+    "--disable-domain-reliability",
+    "--export-tagged-pdf",
+    "--disable-sync",
+    "--disable-features=AcceptCHFrame,AutoExpandDetailsElement,AvoidUnnecessaryBeforeUnloadCheckSync,CertificateTransparencyComponentUpdater,DestroyProfileOnBrowserClose,DialMediaRouteProvider,ExtensionManifestV2Disabled,GlobalMediaControls,HttpsUpgrades,ImprovedCookieControls,LazyFrameLoading,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,Translate,AutomationControlled,BackForwardCache,OptimizationHints,ProcessPerSiteUpToMainFrameThreshold,InterestFeedContentSuggestions,CalculateNativeWinOcclusion,HeavyAdPrivacyMitigations,PrivacySandboxSettings4,AutofillServerCommunication,CrashReporting,OverscrollHistoryNavigation,InfiniteSessionRestore,ExtensionDisableUnsupportedDeveloper",
+    "--disable-prompt-on-repost",
+    "--disable-infobars",
+    "--disable-datasaver-prompt",
+    "--noerrdialogs",
+    "--disable-breakpad",
+    "--disable-field-trial-config",
+    "--window-size=1920,1080",
+    "--window-position=0,0",
+    "--enable-extensions",
+    "--disable-extensions-file-access-check",
+    "--enable-extension-activity-logging",
+    `--remote-debugging-port=0`,
+    "--remote-debugging-address=0.0.0.0",
+    "about:blank",
+  ];
+
+  const proc = spawn(executablePath, args, {
+    stdio: ["ignore", "ignore", "pipe"], // stderr is useful for debugging
+    env: { ...process.env, ...env },
+  });
+
+  // If Chromium dies early, surface the error.
+  const earlyExit = new Promise<never>((_, reject) => {
+    proc.once("exit", (code, signal) => {
+      reject(
+        new Error(`Chromium exited prematurely (code=${code}, signal=${signal}).`)
+      );
+    });
+  });
+
+  const devtoolsPortFile = path.join(userDataDir, "DevToolsActivePort");
+
+  // Wait for DevToolsActivePort file to appear, then read it for port + browserId.
+  const whenReady = (async () => {
+    const startedAt = Date.now();
+    while (Date.now() - startedAt < timeoutMs) {
+      try {
+        // Access succeeds once the file is written.
+        await fs.access(devtoolsPortFile, fscb.constants.F_OK);
+        break;
+      } catch {
+        await delay(50);
+      }
+    }
+    // If still not there, bail.
+    await fs.access(devtoolsPortFile, fscb.constants.F_OK).catch(() => {
+      throw new Error(
+        `Timed out after ${timeoutMs}ms waiting for DevToolsActivePort at ${devtoolsPortFile}`
+      );
+    });
+
+    const raw = await fs.readFile(devtoolsPortFile, "utf8");
+    const [portLine] = raw.trim().split(/\r?\n/);
+    const port = Number(portLine);
+    if (!Number.isFinite(port)) {
+      throw new Error(`Invalid DevTools port read from file: "${portLine}"`);
+    }
+
+    // Optional: ping /json/version to ensure the endpoint is responsive.
+    await waitForHttpOk(`http://127.0.0.1:${port}/json/version`, timeoutMs);
+
+    const cdpUrl = await fetchWsFromVersionEndpoint(port, timeoutMs);
+
+    localBrowserTask = {
+      cdpUrl,
+      proc,
+    };
+
+    return { cdpUrl };
+  })();
+
+  const { cdpUrl } = await Promise.race([whenReady, earlyExit]);
+
+  logger.info({ message: 'Local browser task started', cdpUrl });
+
+  return { cdpUrl };
+}
+
+// ---- helpers ----
+function delay(ms: number) {
+  return new Promise<void>((r) => setTimeout(r, ms));
+}
+
+async function waitForHttpOk(url: string, timeoutMs: number) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const ok = await httpGetOk(url).catch(() => false);
+    if (ok) return;
+    await delay(50);
+  }
+  throw new Error(`Timed out after ${timeoutMs}ms waiting for ${url}`);
+}
+
+function httpGetOk(urlStr: string): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const req = http.get(urlStr, (res) => {
+      // Drain data to allow socket reuse.
+      res.resume();
+      resolve(res.statusCode === 200);
+    });
+    req.on("error", reject);
+    req.setTimeout(3000, () => {
+      req.destroy(new Error("HTTP timeout"));
+    });
+  });
+}
+
+async function fetchWsFromVersionEndpoint(
+  port: number,
+  timeoutMs: number
+): Promise<string> {
+  const urlStr = `http://127.0.0.1:${port}/json/version`;
+  const body = await httpGetBody(urlStr, timeoutMs);
+  const parsed = JSON.parse(body);
+  if (!parsed.webSocketDebuggerUrl) {
+    throw new Error(`No webSocketDebuggerUrl in ${urlStr} response`);
+  }
+  return parsed.webSocketDebuggerUrl as string;
+}
+
+function httpGetBody(urlStr: string, timeoutMs: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const req = http.get(urlStr, (res) => {
+      let data = "";
+      res.setEncoding("utf8");
+      res.on("data", (chunk) => (data += chunk));
+      res.on("end", () => {
+        if (res.statusCode === 200) resolve(data);
+        else reject(new Error(`HTTP ${res.statusCode}: ${data}`));
+      });
+    });
+    req.on("error", reject);
+    req.setTimeout(timeoutMs, () => req.destroy(new Error("HTTP timeout")));
+  });
+}
+
+export async function kill() {
+  if (localBrowserTask) {
+    localBrowserTask.proc.kill("SIGTERM");
+    await delay(300);
+    localBrowserTask.proc.kill("SIGKILL");
+  }
+}

package/src/index.ts

@@ -48,6 +48,8 @@ export type {
   VoiceTriggerNode,
   WebhookTriggerNode,
   InterfaceTriggerNode,
+  RpaNode,
+  RpaStep,
 } from './types/Flows.types';
 export {
   NodeType,
@@ -57,6 +59,7 @@ export {
   AppNodeMetadataType,
   NodeMetadata,
   KnownTriggerNames,
+  RpaActionType,
 } from './types/Flows.types';
 export type { Tool, ToolExecuteInput } from './types/Tools.types';
 

package/src/nodes/addBrowserTaskNode.ts

@@ -54,7 +54,7 @@ export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowser
     const zodSchema = z.object(schemaFields);
 
     // Create langchain tool
-    const tool = langchainTool(() => {}, {
+    const tool = langchainTool(() => { }, {
       name: 'browser-task',
       description: node.prompt,
       schema: zodSchema,
@@ -100,7 +100,12 @@ ${compiledPrompt}
 ${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(inputParams, null, 2)}\n\n` : ''}`;
 
       // Create browser session using socket
-      const session = await createBrowserSession(node.proxy, node.onPrem);
+      const session = await createBrowserSession({
+        sessionId: state.sessionId,
+        proxy: node.proxy,
+        onPrem: node.onPrem,
+        localRun: node.localRun,
+      });
 
       if (!session.sessionId || !session.cdpUrl) {
         throw new Error('Failed to create browser session: missing session details');

package/src/nodes/addBrowserTaskRunNode.ts

@@ -57,6 +57,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
         keepAlive,
         hooks,
         browserTaskNode.onPrem,
+        browserTaskNode.localRun,
         toolSchemas,
         outputSchema,
       );