@minded-ai/mindedjs 2.0.34 → 2.0.35-beta.1

package/dist/utils/extractToolMemoryResponse.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const extractToolMemoryResponse = (toolMessage) => {
+    try {
+        const parsed = JSON.parse(toolMessage.content);
+        if (typeof parsed === 'object' && parsed !== null && 'memory' in parsed) {
+            return parsed.memory;
+        }
+    }
+    catch (error) {
+        console.error('Error parsing tool memory response', error);
+    }
+    return {};
+};
+exports.default = extractToolMemoryResponse;
+//# sourceMappingURL=extractToolMemoryResponse.js.map

package/dist/utils/extractToolMemoryResponse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractToolMemoryResponse.js","sourceRoot":"","sources":["../../src/utils/extractToolMemoryResponse.ts"],"names":[],"mappings":";;AAEA,MAAM,yBAAyB,GAAG,CAAS,WAAwB,EAAmB,EAAE;IACtF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAiB,CAAC,CAAC;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;YACxE,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,kBAAe,yBAAyB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "2.0.34",
+  "version": "2.0.35-beta.1",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -51,6 +51,7 @@
     "typescript-eslint": "^8.32.1"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.717.0",
     "@langchain/core": "^0.3.55",
     "@langchain/langgraph": "^0.2.72",
     "@types/chrome-remote-interface": "^0.31.14",

package/src/nodes/addPromptNode.ts

@@ -17,6 +17,9 @@ import { compilePrompt } from './compilePrompt';
 import { v4 as uuidv4 } from 'uuid';
 import { AnalyticsEventName } from '../types/Analytics.types';
 import { trackAnalyticsEvent } from '../internalTools/analytics';
+import { withBrowserSession } from '../toolsLibrary/withBrowserSession';
+import { getConfig } from '../platform/config';
+import { createBrowserSession } from '../browserTask/executeBrowserTask';
 
 type AddPromptNodeParams = {
   graph: PreCompiledGraph;
@@ -47,13 +50,77 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
 
     const scopedTools = tools
       .filter((tool) => tool.isGlobal || node.prompt.includes(`{tools.${tool.name}}`))
-      .map((tool) =>
-        langchainTool((input: z.infer<typeof tool.input>) => tool.execute({ input, state, agent, invokedFromPromptNode: true }), {
+      .map((tool) => {
+        const executeWrapper = async (input: z.infer<typeof tool.input>) => {
+          // Check if this is an RPA tool
+          if (tool.type === 'rpa') {
+            // Generate toolCallId only for RPA tools (needed for S3 screenshot/log storage)
+            const { browserTaskMode } = getConfig();
+
+            // Ensure browser session exists
+            if (!state.cdp?.url) {
+              const session = await createBrowserSession({
+                sessionId: state.sessionId,
+                browserTaskMode,
+              });
+
+              if (!session.sessionId || !session.cdpUrl) {
+                throw new Error('Failed to create browser session: missing session details');
+              }
+              state.cdp = {
+                url: session.cdpUrl,
+                sessionId: session.sessionId,
+                liveUrl: session.liveViewUrl,
+              };
+            }
+
+            // Wrap with browser session and screenshot capture
+            const { result, logs } = await withBrowserSession(
+              {
+                sessionId: state.sessionId,
+                browserTaskMode,
+                captureScreenshots: true,
+              },
+              async (page) => {
+                // Pass page directly to RPA tool execution
+                const toolResult = await tool.execute({
+                  input,
+                  state,
+                  agent,
+                  page,
+                  invokedFromPromptNode: true,
+                });
+
+                return toolResult;
+              }
+            );
+
+            // Add logs to tool message metadata if available
+            if (result && logs.length > 0) {
+              const toolMessage = result as any;
+              if (toolMessage) {
+                if (!toolMessage.additional_kwargs) {
+                  toolMessage.additional_kwargs = {};
+                }
+                toolMessage.additional_kwargs.mindedMetadata = {
+                  logs,
+                };
+              }
+            }
+
+            return result;
+          } else {
+            // Non-RPA tool - execute normally
+            return tool.execute({ input, state, agent, invokedFromPromptNode: true });
+          }
+        };
+
+        return langchainTool(executeWrapper, {
           name: tool.name,
           description: tool.description,
           schema: tool.input,
-        }),
-      );
+        });
+      });
 
     // Get edges for the current node and format them
     const edges = agent.flows?.flatMap((flow: any) => flow.edges) || [];

package/src/nodes/addToolNode.ts

@@ -49,7 +49,7 @@ export const addToolNode = async ({
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     logger.debug({ msg: `[Node] Executing tool node`, node: toolNode.displayName });
 
-    const tool = langchainTool(() => {}, {
+    const tool = langchainTool(() => { }, {
       name: matchedTool.name,
       description: matchedTool.description,
       schema: matchedTool.input,
@@ -82,6 +82,17 @@ export const addToolNode = async ({
     logger.debug({ msg: '[Tool] Model execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     state.goto = null;
+
+    // Add functionCallType metadata if tool is RPA
+    if (matchedTool.type === 'rpa') {
+      if (!AIToolCallMessage.additional_kwargs) {
+        AIToolCallMessage.additional_kwargs = {};
+      }
+      AIToolCallMessage.additional_kwargs.mindedMetadata = {
+        functionCallType: 'rpa',
+      };
+    }
+
     state.messages.push(AIToolCallMessage);
 
     const historyMsg = createHistoryStep<HistoryStep>(state.history, {

package/src/nodes/addToolRunNode.ts

@@ -10,6 +10,8 @@ import { logger } from '../utils/logger';
 import { HistoryStep } from '../types/Agent.types';
 import { createHistoryStep } from '../utils/history';
 import { ToolMessage } from '@langchain/core/messages';
+import { withBrowserSession } from '../toolsLibrary/withBrowserSession';
+import { getConfig } from '../platform/config';
 
 type AddToolRunNodeParams = {
   graph: PreCompiledGraph;
@@ -33,10 +35,55 @@ export const addToolRunNode = async ({ graph, tools, toolNode, attachedToNodeNam
     const executeWrapper = async (input: z.infer<typeof matchedTool.input>) => {
       try {
         const startTime = Date.now();
-        const response = await matchedTool.execute({ input, state, agent });
-        const endTime = Date.now();
-        logger.debug({ msg: '[Tool] Tool execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
-        return response as ToolMessage;
+        const toolCallId = toolCallObj.tool_calls[0].id;
+
+        // Check if this is an RPA tool
+        if (matchedTool.type === 'rpa') {
+          const { browserTaskMode } = getConfig();
+
+          // Wrap with browser session and screenshot capture
+          const { result, logs } = await withBrowserSession(
+            {
+              sessionId: state.sessionId,
+              browserTaskMode,
+              captureScreenshots: true,
+              toolCallId,
+            },
+            async (page) => {
+              // Pass page directly to RPA tool execution
+              const toolResult = await matchedTool.execute({
+                input,
+                state,
+                agent,
+                page,
+              });
+
+              return toolResult;
+            }
+          );
+
+          const endTime = Date.now();
+          logger.debug({ msg: '[Tool] RPA tool execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
+
+          // Create tool message with logs in metadata
+          const toolMessage = result as ToolMessage;
+          if (toolMessage && logs.length > 0) {
+            if (!toolMessage.additional_kwargs) {
+              toolMessage.additional_kwargs = {};
+            }
+            toolMessage.additional_kwargs.mindedMetadata = {
+              logs,
+            };
+          }
+
+          return toolMessage;
+        } else {
+          // Non-RPA tool - execute normally
+          const response = await matchedTool.execute({ input, state, agent });
+          const endTime = Date.now();
+          logger.debug({ msg: '[Tool] Tool execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
+          return response as ToolMessage;
+        }
       } catch (err) {
         logger.error({ message: '[Tool] Error executing tool', err, node: toolNode.displayName });
         throw err;

package/src/toolsLibrary/withBrowserSession.ts

@@ -5,10 +5,186 @@ import { logger } from '../utils/logger';
 import * as fs from 'fs';
 import * as path from 'path';
 import { CookieStore } from '../browserTask/CookieStore';
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 
 // Initialize cookie store
 const cookieStore = new CookieStore(path.join(process.cwd(), 'local_storage', 'cookies'));
 
+/**
+ * Screenshot capture class that takes screenshots before and after each action
+ * and uploads them to S3
+ */
+class ScreenshotCapture {
+  private s3Client: S3Client | null = null;
+  private s3Bucket: string = '';
+  private s3Prefix: string;
+  private sessionId: string;
+  private stepCounter = 0;
+  private enabled: boolean;
+
+  constructor(sessionId: string) {
+    this.sessionId = sessionId;
+    this.enabled = false;
+
+    // Initialize S3 configuration from environment
+    const bucket = process.env.SCREENSHOT_S3_BUCKET || process.env.S3_BUCKET_AGENTSFORCE;
+    const region = process.env.AWS_REGION || 'us-east-1';
+    this.s3Prefix = 'browser-task/screenshots/';
+
+    if (bucket) {
+      this.s3Bucket = bucket;
+      this.s3Client = new S3Client({ region });
+      this.enabled = true;
+      logger.info({
+        message: 'Screenshot capture ENABLED',
+        sessionId: this.sessionId,
+        bucket: this.s3Bucket,
+        region,
+      });
+    } else {
+      logger.info({
+        message: 'Screenshot capture DISABLED - no S3 bucket configured',
+        sessionId: this.sessionId,
+      });
+    }
+  }
+
+  async captureScreenshot(page: Page, phase: 'before' | 'after'): Promise<void> {
+    if (!this.enabled || !this.s3Client) return;
+
+    try {
+      // Increment step counter only on 'after' phase
+      const stepNumber = phase === 'before' ? this.stepCounter : this.stepCounter++;
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const filename = `${this.sessionId}/step_${stepNumber.toString().padStart(4, '0')}_${phase}_${timestamp}.png`;
+      const s3Key = `${this.s3Prefix}${filename}`;
+
+      // Take screenshot
+      const screenshotBuffer = await page.screenshot({ type: 'png' });
+
+      // Upload to S3 with KMS encryption
+      const putParams: any = {
+        Bucket: this.s3Bucket,
+        Key: s3Key,
+        Body: screenshotBuffer,
+        ContentType: 'image/png',
+        ServerSideEncryption: 'aws:kms',
+      };
+
+      // Only add tagging if enabled (to avoid permission issues on-prem)
+      if (process.env.ENABLE_SCREENSHOTS_TAGGING === 'true') {
+        putParams.Tagging = 'retention=30d';
+      }
+
+      await this.s3Client.send(new PutObjectCommand(putParams));
+
+      logger.debug({
+        message: 'Screenshot uploaded',
+        s3Key,
+        phase,
+        stepNumber,
+        sessionId: this.sessionId,
+        size: screenshotBuffer.length,
+      });
+    } catch (error) {
+      logger.warn({
+        message: 'Failed to capture screenshot',
+        sessionId: this.sessionId,
+        phase,
+        error,
+      });
+      // Don't throw - continue execution
+    }
+  }
+}
+
+/**
+ * Logs capture class that accumulates logs from Page actions
+ */
+class LogsCapture {
+  private logEntries: string[] = [];
+
+  addLog(message: string): void {
+    this.logEntries.push(message);
+  }
+
+  getLogs(): string[] {
+    return this.logEntries;
+  }
+}
+
+/**
+ * Create a proxy page that intercepts actions and captures screenshots and logs
+ */
+const createProxyPage = (
+  page: Page,
+  screenshotCapture: ScreenshotCapture,
+  logsCapture?: LogsCapture,
+): Page => {
+  // List of methods to intercept for screenshot capture and logging
+  const methodsToIntercept = ['click', 'fill', 'type', 'goto', 'selectOption', 'check', 'uncheck', 'setInputFiles'];
+
+  const proxyPage = new Proxy(page, {
+    get(target, prop, receiver) {
+      const originalValue = Reflect.get(target, prop, receiver);
+
+      // If it's one of the methods we want to intercept
+      if (typeof prop === 'string' && methodsToIntercept.includes(prop)) {
+        return async function (this: any, ...args: any[]) {
+          // Log the action
+          if (logsCapture) {
+            const actionDescription = formatActionLog(prop, args);
+            logsCapture.addLog(actionDescription);
+          }
+
+          // Capture "before" screenshot
+          await screenshotCapture.captureScreenshot(target, 'before');
+
+          // Call the original method
+          const result = await originalValue.apply(target, args);
+
+          // Capture "after" screenshot
+          await screenshotCapture.captureScreenshot(target, 'after');
+
+          return result;
+        };
+      }
+
+      return originalValue;
+    },
+  });
+
+  return proxyPage;
+};
+
+/**
+ * Format action logs for display
+ */
+const formatActionLog = (action: string, args: any[]): string => {
+  const timestamp = new Date().toISOString();
+
+  switch (action) {
+    case 'goto':
+      return `[${timestamp}] Navigate to: ${args[0]}`;
+    case 'click':
+      return `[${timestamp}] Click: ${args[0]}`;
+    case 'fill':
+      return `[${timestamp}] Fill "${args[0]}" with: ${args[1]}`;
+    case 'type':
+      return `[${timestamp}] Type in "${args[0]}": ${args[1]}`;
+    case 'selectOption':
+      return `[${timestamp}] Select option in "${args[0]}": ${JSON.stringify(args[1])}`;
+    case 'check':
+      return `[${timestamp}] Check: ${args[0]}`;
+    case 'uncheck':
+      return `[${timestamp}] Uncheck: ${args[0]}`;
+    case 'setInputFiles':
+      return `[${timestamp}] Set files for "${args[0]}"`;
+    default:
+      return `[${timestamp}] ${action}: ${JSON.stringify(args[0])}`;
+  }
+};
+
 // Helper function to load cookies for a session
 const loadCookiesForSession = async (context: BrowserContext, sessionId: string): Promise<void> => {
   const stored = await cookieStore.load();
@@ -41,9 +217,19 @@ const saveCookiesForSession = async (context: BrowserContext, sessionId: string)
 };
 
 export const withBrowserSession = async <T>(
-  { sessionId, browserTaskMode }: { sessionId: string; browserTaskMode: BrowserTaskMode },
+  {
+    sessionId,
+    browserTaskMode,
+    captureScreenshots = false,
+    toolCallId,
+  }: {
+    sessionId: string;
+    browserTaskMode: BrowserTaskMode;
+    captureScreenshots?: boolean;
+    toolCallId?: string;
+  },
   callback: (page: Page) => Promise<T>,
-): Promise<T> => {
+): Promise<{ result: T; logs: string[] }> => {
   const { cdpUrl } = await createBrowserSession({ sessionId, browserTaskMode });
   if (!cdpUrl) {
     throw new Error('CDP URL was not provided by the browser session.');
@@ -67,9 +253,20 @@ export const withBrowserSession = async <T>(
   }
   logger.info({ msg: 'Connected to CDP', cdpUrl, sessionId });
   const context: BrowserContext = browser.contexts()[0] ?? (await browser.newContext());
-  const page: Page = context.pages()[0] ?? (await context.newPage());
+  let page: Page = context.pages()[0] ?? (await context.newPage());
   logger.info({ msg: 'Created page', sessionId });
 
+  // Initialize screenshot and log capture if enabled
+  let screenshotCapture: ScreenshotCapture | undefined;
+  let logsCapture: LogsCapture | undefined;
+
+  if (captureScreenshots && toolCallId) {
+    screenshotCapture = new ScreenshotCapture(sessionId);
+    logsCapture = new LogsCapture();
+    page = createProxyPage(page, screenshotCapture, logsCapture);
+    logger.info({ msg: 'Screenshot and logs capture enabled', sessionId, toolCallId });
+  }
+
   // Load cookies for this session (only for local mode, best-effort)
   const noCookies = process.env.MINDED_NO_COOKIES === 'true';
   if (browserTaskMode === BrowserTaskMode.LOCAL && !noCookies) {
@@ -116,7 +313,9 @@ export const withBrowserSession = async <T>(
   }
 
   try {
-    return await callback(page);
+    const result = await callback(page);
+    const logs = logsCapture?.getLogs() || [];
+    return { result, logs };
   } catch (e) {
     logger.error({ msg: 'Error in browser session callback', sessionId, error: e });
 
@@ -157,7 +356,7 @@ export const withBrowserSession = async <T>(
       sessionId,
       localRun: browserTaskMode === BrowserTaskMode.LOCAL,
       onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
-    }).catch(() => {});
+    }).catch(() => { });
   }
 };
 

package/src/types/Tools.types.ts

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { Agent } from '../';
 import { State } from './LangGraph.types';
+import { Page } from 'playwright';
 
 // Valid return type for new API
 export type ToolReturnType = { result?: any } | void;
@@ -11,6 +12,7 @@ export interface Tool<Input extends z.ZodSchema, Memory = any> {
   input: Input;
   isGlobal?: boolean;
   allowExecutionRequests?: boolean;
+  type?: 'rpa' | 'default';
   execute: (input: ToolExecuteInput<Input, Memory>) => Promise<ToolReturnType>;
 }
 
@@ -18,6 +20,7 @@ export interface ToolExecuteInput<Input extends z.ZodSchema, Memory = any> {
   input: z.infer<Input>;
   state: State<Memory>;
   agent: Agent;
+  page?: Page;
   invokedFromPromptNode?: boolean;
 }