@minded-ai/mindedjs 3.1.24 → 3.1.26

package/src/toolsLibrary/withBrowserSession.ts

@@ -4,13 +4,55 @@ import { createBrowserSession, destroyBrowserSession } from '../browserTask/exec
 import { logger } from '../utils/logger';
 import * as fs from 'fs';
 import * as path from 'path';
-import { randomUUID } from 'crypto';
 import { awaitEmit, emit, isConnected } from '../platform/mindedConnection';
-import { mindedConnectionSocketMessageType, BrowserRunEventRequest, BrowserRunEventType } from '../platform/mindedConnectionTypes';
+import {
+  BrowserRunEventRequest,
+  BrowserRunEventType,
+  mindedConnectionSocketMessageType,
+  RpaBreakpointHitMessage,
+  GetActiveRpaBreakpointResponse,
+} from '../platform/mindedConnectionTypes';
 import { ProxyConfig } from '../types/Tools.types';
+import { randomUUID } from 'crypto';
 import { getConfig } from '../platform/config';
 import { fetchSessionDataViaSocket, saveSessionDataViaSocket } from './browserSessionPersistence';
 
+// Type for RPA breakpoint info (returned from backend)
+export interface RpaBreakpointInfo {
+  toolName: string;
+  stepNumber: number;
+}
+
+// Type for breakpoint hit result
+export interface BreakpointHitResult {
+  toolCallId: string;
+  toolName: string;
+  stepNumber: number;
+  cdpUrl: string;
+  logs: string[];
+}
+
+// Type for withBrowserSession result
+export interface BrowserSessionResult<T> {
+  result: T;
+  breakpointHit?: boolean;
+  breakpointInfo?: BreakpointHitResult;
+}
+
+/**
+ * Custom error thrown when an RPA breakpoint is hit
+ * This is used to cleanly exit the RPA task and keep the browser session alive
+ */
+export class RpaBreakpointHitError extends Error {
+  constructor(
+    message: string,
+    public readonly breakpointInfo: BreakpointHitResult,
+  ) {
+    super(message);
+    this.name = 'RpaBreakpointHitError';
+  }
+}
+
 /**
  * Sends a browser run analytics event to the backend (for local browsers only).
  * Events are stored individually and joined by browser_run_id in worker aggregations.
@@ -100,7 +142,6 @@ function stripHtmlContent(html: string): string {
 class ScreenshotCapture {
   private sessionId: string;
   private toolCallId: string;
-  private stepCounter = 0;
 
   constructor(sessionId: string, toolCallId: string) {
     this.sessionId = sessionId;
@@ -112,7 +153,7 @@ class ScreenshotCapture {
     });
   }
 
-  async captureScreenshot(page: Page, phase: 'before' | 'after'): Promise<void> {
+  async captureScreenshot(page: Page, phase: 'before' | 'after', stepNumber: number): Promise<void> {
     try {
       const timestamp = new Date().toISOString();
 
@@ -125,7 +166,7 @@ class ScreenshotCapture {
         type: mindedConnectionSocketMessageType.RPA_SCREENSHOT_UPLOAD,
         sessionId: this.sessionId,
         toolCallId: this.toolCallId,
-        stepNumber: this.stepCounter++,
+        stepNumber,
         phase,
         screenshotData: screenshotBase64,
         timestamp,
@@ -135,6 +176,7 @@ class ScreenshotCapture {
           sessionId: this.sessionId,
           toolCallId: this.toolCallId,
           phase,
+          stepNumber,
           error,
         });
       });
@@ -144,6 +186,7 @@ class ScreenshotCapture {
         sessionId: this.sessionId,
         toolCallId: this.toolCallId,
         phase,
+        stepNumber,
         error,
       });
       // Don't throw - continue execution
@@ -166,6 +209,9 @@ class LogsCapture {
     this.toolCallId = toolCallId;
   }
 
+  /**
+   * Add a log entry
+   */
   async addLog(message: string): Promise<void> {
     this.logEntries.push(message);
 
@@ -223,6 +269,117 @@ class LogsCapture {
   }
 }
 
+/**
+ * Context for breakpoint checking
+ */
+interface BreakpointContext {
+  toolCallId: string;
+  toolName: string;
+  sessionId: string;
+  cdpUrl: string;
+  checkBreakpoints: boolean; // Only check breakpoints in dev environment with socket connection
+}
+
+/**
+ * Fetch active breakpoint from backend via socket
+ * Returns null if no breakpoint is set or if not connected
+ */
+const fetchActiveBreakpoint = async (toolName: string): Promise<RpaBreakpointInfo | null> => {
+  if (!isConnected()) {
+    return null;
+  }
+
+  try {
+    const response = await awaitEmit<
+      { type: string; toolName: string },
+      GetActiveRpaBreakpointResponse
+    >(
+      mindedConnectionSocketMessageType.GET_ACTIVE_RPA_BREAKPOINT,
+      {
+        type: mindedConnectionSocketMessageType.GET_ACTIVE_RPA_BREAKPOINT,
+        toolName,
+      },
+    );
+
+    if (response.error) {
+      logger.warn({ msg: 'Error fetching active breakpoint from backend', error: response.error });
+      return null;
+    }
+
+    return response.breakpoint;
+  } catch (error) {
+    logger.warn({ msg: 'Failed to fetch active breakpoint from backend', error });
+    return null;
+  }
+};
+
+/**
+ * Check if we've hit a breakpoint and handle it
+ * Queries backend for active breakpoint (only in dev environment)
+ * Throws RpaBreakpointHitError if breakpoint is hit
+ */
+const checkBreakpoint = async (
+  stepNumber: number,
+  logsCapture: LogsCapture,
+  page: Page,
+  context: BreakpointContext,
+): Promise<void> => {
+  const { toolCallId, toolName, sessionId, cdpUrl, checkBreakpoints } = context;
+
+  // Skip breakpoint checking if not enabled (non-dev environment or no socket connection)
+  if (!checkBreakpoints) {
+    return;
+  }
+
+  // Fetch active breakpoint from backend
+  const activeBreakpoint = await fetchActiveBreakpoint(toolName);
+
+  if (!activeBreakpoint) {
+    return;
+  }
+
+  // Check if we've hit the breakpoint (match by toolName and stepNumber)
+  if (activeBreakpoint.stepNumber === stepNumber) {
+    logger.info({
+      msg: 'RPA breakpoint hit!',
+      toolName,
+      stepNumber,
+      sessionId,
+      cdpUrl,
+    });
+
+    // Emit breakpoint hit message to backend (agentId is added by backend from socket connection)
+    const breakpointMessage: RpaBreakpointHitMessage = {
+      type: mindedConnectionSocketMessageType.RPA_BREAKPOINT_HIT,
+      sessionId,
+      toolCallId,
+      toolName,
+      stepNumber,
+      cdpUrl,
+      logs: logsCapture.getLogs(),
+    };
+
+    try {
+      await awaitEmit(mindedConnectionSocketMessageType.RPA_BREAKPOINT_HIT, breakpointMessage);
+      logger.info({ msg: 'RPA breakpoint hit message sent to backend', toolCallId, stepNumber });
+    } catch (error) {
+      logger.error({ msg: 'Failed to send breakpoint hit message', error });
+    }
+
+    // Throw error to gracefully stop execution (will be caught by withBrowserSession)
+    throw new RpaBreakpointHitError(
+      `RPA breakpoint hit at step ${stepNumber} for tool ${toolName}`,
+      {
+        toolCallId,
+        toolName,
+        stepNumber,
+        cdpUrl,
+        logs: logsCapture.getLogs(),
+      },
+    );
+  }
+};
+
 /**
  * Create a proxy locator that intercepts actions and captures screenshots and logs
  */
@@ -231,30 +388,76 @@ const createProxyLocator = (
   page: Page,
   screenshotCapture: ScreenshotCapture,
   logsCapture: LogsCapture,
+  breakpointContext: BreakpointContext,
+  getNextStepNumber: () => number,
   selector?: string,
 ): Locator => {
   // List of methods to intercept on locators
-  const locatorMethodsToIntercept = ['click', 'fill', 'type', 'selectOption', 'check', 'uncheck', 'setInputFiles'];
+  const locatorMethodsToIntercept = [
+    'click',
+    'fill',
+    'type',
+    'selectOption',
+    'check',
+    'uncheck',
+    'setInputFiles',
+    'waitFor',
+    'evaluate',
+    'evaluateAll',
+  ];
+  // Methods that return locators and need special handling
+  const locatorMethodsThatReturnLocators = [
+    'getByRole',
+    'getByText',
+    'getByLabel',
+    'getByPlaceholder',
+    'getByAltText',
+    'getByTitle',
+    'getByTestId',
+    'filter',
+  ];
 
   const proxyLocator = new Proxy(locator, {
     get(target, prop, receiver) {
       const originalValue = Reflect.get(target, prop, receiver);
 
+      // Handle methods that return locators - return a proxied locator
+      if (typeof prop === 'string' && locatorMethodsThatReturnLocators.includes(prop)) {
+        return function (this: any, ...args: any[]) {
+          const newLocator = originalValue.apply(target, args);
+          // For getBy* methods, try to extract a description from args
+          const description = args.length > 0 ? JSON.stringify(args[0]).substring(0, 50) : selector;
+          return createProxyLocator(
+            newLocator,
+            page,
+            screenshotCapture,
+            logsCapture,
+            breakpointContext,
+            getNextStepNumber,
+            description,
+          );
+        };
+      }
+
       // If it's one of the methods we want to intercept
       if (typeof prop === 'string' && locatorMethodsToIntercept.includes(prop)) {
         return async function (this: any, ...args: any[]) {
+          // Get the step number for this action
+          const stepNumber = getNextStepNumber();
+
           // Log the action (this will trigger immediate upload)
           const actionDescription = formatLocatorActionLog(prop, selector, args);
           await logsCapture.addLog(actionDescription);
 
-          // Capture "before" screenshot
-          await screenshotCapture.captureScreenshot(page, 'before');
 
           // Call the original method
           const result = await originalValue.apply(target, args);
 
           // Capture "after" screenshot
-          await screenshotCapture.captureScreenshot(page, 'after');
+          await screenshotCapture.captureScreenshot(page, 'after', stepNumber);
+
+          // Check for breakpoint after executing the action
+          await checkBreakpoint(stepNumber, logsCapture, page, breakpointContext);
 
           return result;
         };
@@ -270,7 +473,13 @@ const createProxyLocator = (
 /**
  * Create a proxy page that intercepts actions and captures screenshots and logs
  */
-const createProxyPage = (page: Page, screenshotCapture: ScreenshotCapture, logsCapture: LogsCapture): Page => {
+const createProxyPage = (
+  page: Page,
+  screenshotCapture: ScreenshotCapture,
+  logsCapture: LogsCapture,
+  breakpointContext: BreakpointContext,
+  getNextStepNumber: () => number,
+): Page => {
   // List of methods to intercept for screenshot capture and logging
   const methodsToIntercept = [
     'click',
@@ -283,9 +492,19 @@ const createProxyPage = (page: Page, screenshotCapture: ScreenshotCapture, logsC
     'setInputFiles',
     'waitForTimeout',
     'evaluate',
+    'waitForSelector',
   ];
   // Methods that return locators and need special handling
-  const locatorMethods = ['locator'];
+  const locatorMethods = [
+    'locator',
+    'getByRole',
+    'getByText',
+    'getByLabel',
+    'getByPlaceholder',
+    'getByAltText',
+    'getByTitle',
+    'getByTestId',
+  ];
 
   const proxyPage = new Proxy(page, {
     get(target, prop, receiver) {
@@ -295,27 +514,38 @@ const createProxyPage = (page: Page, screenshotCapture: ScreenshotCapture, logsC
       if (typeof prop === 'string' && locatorMethods.includes(prop)) {
         return function (this: any, ...args: any[]) {
           const locator = originalValue.apply(target, args);
-          // Extract selector from args if available (first argument is usually the selector)
-          const selector = args[0] && typeof args[0] === 'string' ? args[0] : undefined;
-          return createProxyLocator(locator, target, screenshotCapture, logsCapture, selector);
+          // Extract selector/description from args
+          let selector: string | undefined;
+          if (prop === 'locator' && args[0] && typeof args[0] === 'string') {
+            selector = args[0];
+          } else if (prop.startsWith('getBy')) {
+            // For getBy* methods, create a descriptive string
+            const methodName = prop.replace('getBy', '');
+            const argDescription = args.length > 0 ? JSON.stringify(args[0]).substring(0, 50) : '';
+            selector = `${methodName}(${argDescription})`;
+          }
+          return createProxyLocator(locator, target, screenshotCapture, logsCapture, breakpointContext, getNextStepNumber, selector);
         };
       }
 
       // If it's one of the methods we want to intercept with screenshots
       if (typeof prop === 'string' && methodsToIntercept.includes(prop)) {
         return async function (this: any, ...args: any[]) {
+          // Get the step number for this action
+          const stepNumber = getNextStepNumber();
+
           // Log the action (this will trigger immediate upload)
           const actionDescription = formatActionLog(prop, args);
           await logsCapture.addLog(actionDescription);
 
-          // Capture "before" screenshot
-          await screenshotCapture.captureScreenshot(target, 'before');
-
           // Call the original method
           const result = await originalValue.apply(target, args);
 
           // Capture "after" screenshot
-          await screenshotCapture.captureScreenshot(target, 'after');
+          await screenshotCapture.captureScreenshot(target, 'after', stepNumber);
+
+          // Check for breakpoint before executing the action
+          await checkBreakpoint(stepNumber, logsCapture, target, breakpointContext);
 
           return result;
         };
@@ -351,6 +581,8 @@ const formatActionLog = (action: string, args: any[]): string => {
       return `Set files for "${args[0]}"`;
     case 'waitForTimeout':
       return `Wait for timeout: ${args[0]}ms`;
+    case 'waitForSelector':
+      return `Wait for selector: ${args[0]}${args[1] ? ` (timeout: ${args[1].timeout || 'default'}ms)` : ''}`;
     case 'evaluate':
       return `Evaluate: ${typeof args[0] === 'function' ? 'function' : String(args[0]).substring(0, 100)}`;
     default:
@@ -379,6 +611,14 @@ const formatLocatorActionLog = (action: string, selector: string | undefined, ar
       return `Uncheck locator: ${locatorDescription}`;
     case 'setInputFiles':
       return `Set files for locator "${locatorDescription}"`;
+    case 'waitFor': {
+      const waitForOptions = args[0] || {};
+      return `Wait for locator "${locatorDescription}"${waitForOptions.state ? ` (state: ${waitForOptions.state})` : ''}${waitForOptions.timeout ? ` (timeout: ${waitForOptions.timeout}ms)` : ''}`;
+    }
+    case 'evaluate':
+      return `Evaluate on locator "${locatorDescription}": ${typeof args[0] === 'function' ? 'function' : String(args[0]).substring(0, 100)}`;
+    case 'evaluateAll':
+      return `Evaluate all on locator "${locatorDescription}": ${typeof args[0] === 'function' ? 'function' : String(args[0]).substring(0, 100)}`;
     default:
       return `Locator ${action}: ${JSON.stringify(args[0])}`;
   }
@@ -392,6 +632,7 @@ export const withBrowserSession = async <T>(
     proxyConfig,
     toolName,
     persistSession = true,
+    checkBreakpoints = false,
   }: {
     sessionId: string;
     browserTaskMode: BrowserTaskMode;
@@ -399,9 +640,11 @@ export const withBrowserSession = async <T>(
     proxyConfig?: ProxyConfig;
     toolName: string;
     persistSession?: boolean;
+    /** Enable breakpoint checking - only works in dev environment with socket connection */
+    checkBreakpoints?: boolean;
   },
   callback: (page: Page) => Promise<T>,
-): Promise<{ result: T }> => {
+): Promise<BrowserSessionResult<T>> => {
   const isLocalBrowser = browserTaskMode === BrowserTaskMode.LOCAL;
 
   // For local browsers: generate browserRunId locally
@@ -468,7 +711,24 @@ export const withBrowserSession = async <T>(
   // Initialize screenshot and log capture if enabled
   const screenshotCapture = new ScreenshotCapture(sessionId, toolCallId);
   const logsCapture = new LogsCapture(sessionId, toolCallId);
-  page = createProxyPage(page, screenshotCapture, logsCapture);
+
+  // Create shared step counter for consistent step numbering across logs and screenshots
+  let stepCounter = 0;
+  const getNextStepNumber = (): number => {
+    return stepCounter++;
+  };
+
+  // Create breakpoint context for checking during execution
+  // Breakpoints are checked by querying the backend via socket (only in dev environment)
+  const breakpointContext: BreakpointContext = {
+    toolCallId,
+    toolName,
+    sessionId,
+    cdpUrl,
+    checkBreakpoints: checkBreakpoints && isConnected(), // Only check if enabled and connected
+  };
+
+  page = createProxyPage(page, screenshotCapture, logsCapture, breakpointContext, getNextStepNumber);
 
   // Load session data (cookies + localStorage) once at startup (if persistence is enabled)
   if (persistSession) {
@@ -552,6 +812,8 @@ export const withBrowserSession = async <T>(
   }
 
   const startTime = Date.now();
+  let breakpointHitResult: BreakpointHitResult | null = null;
+
   try {
     const result = await callback(page);
     // Finalize logs upload (wait for any pending uploads)
@@ -560,6 +822,24 @@ export const withBrowserSession = async <T>(
     }
     return { result };
   } catch (e) {
+    // Check if this is a breakpoint hit - handle gracefully instead of throwing
+    if (e instanceof RpaBreakpointHitError) {
+      logger.info({
+        msg: 'RPA breakpoint hit - returning gracefully to stop execution',
+        toolCallId: e.breakpointInfo.toolCallId,
+        stepNumber: e.breakpointInfo.stepNumber,
+      });
+      breakpointHitResult = e.breakpointInfo;
+      // Finalize logs upload before returning
+      await logsCapture.finalize().catch(() => { });
+      // Return with breakpoint hit info - result is null since we stopped early
+      return {
+        result: null as unknown as T,
+        breakpointHit: true,
+        breakpointInfo: breakpointHitResult,
+      };
+    }
+
     const errorMessage = e instanceof Error ? e.message : JSON.stringify(e);
     await logsCapture.addLog(`Error: ${errorMessage}`);
     // Finalize logs upload (wait for any pending uploads)
@@ -612,54 +892,56 @@ export const withBrowserSession = async <T>(
     const endTime = Date.now();
     logger.info({ msg: 'Browser session execution time', toolName, toolCallId, sessionId, executionTimeMs: endTime - startTime });
 
-    // Capture final screenshot and upload final logs before session ends
-    try {
-      await screenshotCapture.captureScreenshot(page, 'after');
-      logger.info({ msg: 'Final screenshot captured', sessionId });
-    } catch (error) {
-      logger.warn({ msg: 'Failed to capture final screenshot', sessionId, error });
-    }
-
-    if (logsCapture) {
-      try {
-        await logsCapture.finalize();
-        logger.info({ msg: 'Final logs uploaded', sessionId });
-      } catch (error) {
-        logger.warn({ msg: 'Failed to upload final logs', sessionId, error });
+    // If breakpoint was hit, don't destroy the session - keep it alive for Copilot
+    if (breakpointHitResult) {
+      logger.info({
+        msg: 'Breakpoint hit - keeping browser session alive for Copilot debugging',
+        sessionId,
+        cdpUrl: breakpointHitResult.cdpUrl,
+      });
+      // Skip cleanup - don't run any of the code below
+    } else {
+      if (logsCapture) {
+        try {
+          await logsCapture.finalize();
+          logger.info({ msg: 'Final logs uploaded', sessionId });
+        } catch (error) {
+          logger.warn({ msg: 'Failed to upload final logs', sessionId, error });
+        }
       }
-    }
 
-    // Save cookies and localStorage before destroying session (if persistence is enabled)
-    if (persistSession) {
-      try {
-        await saveSessionDataViaSocket(context, page, sessionId);
-      } catch (error) {
-        logger.warn({ message: 'Failed to save session data', sessionId, error });
+      // Save cookies and localStorage before destroying session (if persistence is enabled)
+      if (persistSession) {
+        try {
+          await saveSessionDataViaSocket(context, page, sessionId);
+        } catch (error) {
+          logger.warn({ message: 'Failed to save session data', sessionId, error });
+        }
       }
-    }
 
-    // Destroy browser session (pass analytics fields for backend to track managed browsers)
-    destroyBrowserSession({
-      sessionId,
-      localRun: browserTaskMode === BrowserTaskMode.LOCAL,
-      onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
-      browserRunId,
-      success: true,
-    }).catch(() => {});
-
-    // For local browsers: send completion event
-    // For managed browsers: backend handles tracking via destroyBrowserSession
-    if (isLocalBrowser) {
-      await sendBrowserRunEvent({
-        eventType: 'completed',
+      // Destroy browser session (pass analytics fields for backend to track managed browsers)
+      destroyBrowserSession({
+        sessionId,
+        localRun: browserTaskMode === BrowserTaskMode.LOCAL,
+        onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
         browserRunId,
-        browserSessionId: sessionId,
-        toolCallId,
-        toolName,
-        proxyConfig,
-      });
+        success: true,
+      }).catch(() => { });
+
+      // For local browsers: send completion event
+      // For managed browsers: backend handles tracking via destroyBrowserSession
+      if (isLocalBrowser) {
+        await sendBrowserRunEvent({
+          eventType: 'completed',
+          browserRunId,
+          browserSessionId: sessionId,
+          toolCallId,
+          toolName,
+          proxyConfig,
+        });
+      }
     }
-  }
+  };
 };
 
-const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

package/src/types/Flows.types.ts

@@ -310,6 +310,7 @@ export enum ActionInputParamType {
 }
 
 export interface Flow {
+  id: string;
   name: string;
   nodes: Node[];
   edges: Edge[];