crawlforge-mcp-server 6.12.0 → 6.14.0

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -15,6 +15,7 @@ import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
+import { actionQueue as sharedActionQueue } from '../../core/browser/actionQueue.js';
 
 // Recording / replay helpers
 import {
@@ -198,7 +199,11 @@ const ScrapeWithActionsSchema = z.object({
     // — the standard pool is Chromium and camoufox exists only on the stealth
     // path — so "camoufox" without stealth is refused rather than quietly
     // downgraded (see executeSession).
-    engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto')
+    engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto'),
+    // Answer a cookie/consent wall (autoconsent) after the initial load and
+    // after each navigate action: "reject" opts out, "accept" opts in. Off by
+    // default so a chain that clicks the banner itself keeps working.
+    consent: z.enum(['off', 'reject', 'accept']).default('off')
   }).optional(),
 
   // Content extraction options
@@ -215,7 +220,7 @@ const ScrapeWithActionsSchema = z.object({
 
   // Error handling
   continueOnActionError: z.boolean().default(false),
-  maxRetries: z.number().min(0).max(3).default(1),
+  maxRetries: z.number().min(0).max(3).default(0),
   screenshotOnError: z.boolean().default(true),
 
   // ── Recording / replay ──────────────────────────────────────────────────
@@ -255,7 +260,9 @@ export class ScrapeWithActionsTool extends EventEmitter {
       extractContentTool = null,
       enableLogging = true,
       enableCaching = false,
-      maxConcurrentSessions = 3,
+      // The process-wide browser-slot limiter shared with browser_session open.
+      // Injectable for tests only; CRAWLFORGE_MAX_ACTION_SESSIONS sizes it.
+      actionQueue = sharedActionQueue,
       defaultBrowserOptions = {
         viewportWidth: 1280,
         viewportHeight: 720,
@@ -274,7 +281,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
     this.extractContentTool = extractContentTool || new ExtractContentTool();
     this.enableLogging = enableLogging;
     this.enableCaching = enableCaching;
-    this.maxConcurrentSessions = maxConcurrentSessions;
+    this.actionQueue = actionQueue;
     this.defaultBrowserOptions = defaultBrowserOptions;
 
     // Active sessions tracking
@@ -322,69 +329,80 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
       this.stats.totalSessions++;
       const sessionId = this.generateSessionId();
-      const startTime = Date.now();
+      const enqueuedAt = Date.now();
 
       if (this.enableLogging) {
         console.error(`Starting scrape session ${sessionId} with ${validated.actions.length} actions on ${validated.url}`);
       }
 
-      // Check concurrent sessions limit
-      if (this.activeSessions.size >= this.maxConcurrentSessions) {
-        throw new Error(`Maximum concurrent sessions (${this.maxConcurrentSessions}) reached`);
-      }
-
-      // Create session context
-      const sessionContext = {
-        id: sessionId,
-        url: validated.url,
-        startTime,
-        params: validated,
-        states: [],
-        screenshots: [],
-        actionResults: [],
-        errors: [],
-        status: 'initializing'
-      };
+      // The whole chain runs inside a slot of the shared browser budget. Over
+      // the limit a call waits for one rather than being refused, and fails
+      // only after CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS with the queue depth.
+      return await this.actionQueue.run(
+        () => this.runSession(sessionId, validated, enqueuedAt),
+        { label: 'scrape_with_actions' }
+      );
 
-      this.activeSessions.set(sessionId, sessionContext);
-      this.emit('sessionStarted', sessionContext);
+    } catch (error) {
+      this.log('error', `Scrape with actions failed: ${error.message}`);
+      throw new Error(`Scrape with actions failed: ${error.message}`);
+    }
+  }
 
-      try {
-        const result = await this.executeSession(sessionContext);
-        
-        this.stats.successfulSessions++;
-        this.stats.totalActions += validated.actions.length;
-        this.stats.successfulActions += result.actionResults.filter(r => r.success).length;
-        this.stats.failedActions += result.actionResults.filter(r => !r.success).length;
-        
-        const executionTime = Date.now() - startTime;
-        this.updateAverageSessionTime(executionTime);
-        this.updateAverageActionsPerSession(validated.actions.length);
-        this.updateStats();
-
-        if (this.enableCaching) {
-          this.sessionResults.set(sessionId, {
-            result,
-            timestamp: Date.now(),
-            ttl: 3600000
-          });
-        }
+  async runSession(sessionId, validated, enqueuedAt) {
+    // The clock starts once the slot is held, so executionTime and the
+    // recording's timings measure the run, not the wait before it.
+    const startTime = Date.now();
+    const queuedMs = startTime - enqueuedAt;
+
+    // Create session context
+    const sessionContext = {
+      id: sessionId,
+      url: validated.url,
+      startTime,
+      params: validated,
+      states: [],
+      screenshots: [],
+      actionResults: [],
+      errors: [],
+      status: 'initializing'
+    };
 
-        this.activeSessions.delete(sessionId);
-        this.emit('sessionCompleted', result);
+    this.activeSessions.set(sessionId, sessionContext);
+    this.emit('sessionStarted', sessionContext);
 
-        return result;
+    try {
+      const result = await this.executeSession(sessionContext);
+      result.queued_ms = queuedMs;
 
-      } catch (error) {
-        this.stats.failedSessions++;
-        this.activeSessions.delete(sessionId);
-        this.emit('sessionFailed', { sessionId, url: validated.url, error });
-        throw error;
+      this.stats.successfulSessions++;
+      this.stats.totalActions += validated.actions.length;
+      this.stats.successfulActions += result.actionResults.filter(r => r.success).length;
+      this.stats.failedActions += result.actionResults.filter(r => !r.success).length;
+      
+      const executionTime = Date.now() - startTime;
+      this.updateAverageSessionTime(executionTime);
+      this.updateAverageActionsPerSession(validated.actions.length);
+      this.updateStats();
+
+      if (this.enableCaching) {
+        this.sessionResults.set(sessionId, {
+          result,
+          timestamp: Date.now(),
+          ttl: 3600000
+        });
       }
 
+      this.activeSessions.delete(sessionId);
+      this.emit('sessionCompleted', result);
+
+      return result;
+
     } catch (error) {
-      this.log('error', `Scrape with actions failed: ${error.message}`);
-      throw new Error(`Scrape with actions failed: ${error.message}`);
+      this.stats.failedSessions++;
+      this.activeSessions.delete(sessionId);
+      this.emit('sessionFailed', { sessionId, url: validated.url, error });
+      throw error;
     }
   }
 
@@ -446,6 +464,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         continueOnError: params.continueOnActionError,
         timeout: browserOptions.timeout || 30000,
         retryChain: params.maxRetries,
+        screenshotOnError: params.screenshotOnError,
         metadata: {
           sessionId: sessionContext.id,
           originalActionCount: params.actions.length,
@@ -483,6 +502,12 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
     // Process action results
     const actionResults = this.processActionResults(chainResult.results);
+    // Every attempt the executor made (retryChain > 0 replays the whole
+    // chain), in the same shape as actionResults; `attempt` is the one whose
+    // results are reported above.
+    const attempts = (chainResult.attempts || [
+      { attempt: 1, success: chainResult.success, error: chainResult.error, results: chainResult.results }
+    ]).map((a) => ({ ...a, results: this.processActionResults(a.results || []) }));
     const intermediateStates = params.captureIntermediateStates ?
       await this.extractIntermediateStates(chainResult.capturedStates || [], params) : [];
 
@@ -525,8 +550,13 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // was chosen (an "auto" that fell back to Chromium says so here).
       engine: browserOptions.stealthMode?.engine || 'chromium',
       ...(warnings.length ? { warnings } : {}),
+      // The landing page's consent result; each navigate action's own is on
+      // its result. Absent when consent handling is off.
+      ...(chainResult.consent ? { consent: chainResult.consent } : {}),
 
       actionResults,
+      attempt: chainResult.attempt ?? attempts.length,
+      attempts,
       totalActions: params.actions.length,
       successfulActions: actionResults.filter(r => r.success).length,
       failedActions: actionResults.filter(r => !r.success).length,

package/src/tools/extract/extractEmbeddedState.js

@@ -19,6 +19,9 @@ import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
 // the caller an exact way to ask for less.
 const LARGE_RESULT_BYTES = 256_000;
 
+const BLOCKED_HINT =
+  'stealth_mode operation:"scrape" renders the page in a browser and returns the HTML; the embedded state is in the returned html. Do not repeat this call with a different path.';
+
 /**
  * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
  */
@@ -81,8 +84,15 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
       }]
     };
   } catch (error) {
+    // A block or server error is not a path problem: the generic hint
+    // ("call again without `path`") would send the caller back into the same
+    // wall. Naming the next step here keeps withAuth from appending it.
+    const blocked = /^HTTP (403|429|444|5\d\d)\b/.test(error.message);
     return {
-      content: [{ type: 'text', text: `Failed to extract embedded state: ${error.message}` }],
+      content: [{
+        type: 'text',
+        text: `Failed to extract embedded state: ${error.message}` + (blocked ? `\nNext step: ${BLOCKED_HINT}` : '')
+      }],
       isError: true
     };
   }

package/src/utils/HumanBehaviorSimulator.js

@@ -102,7 +102,7 @@ export class HumanBehaviorSimulator {
    * Simulate human-like mouse movement to target coordinates
    */
   async simulateMouseMovement(page, fromX, fromY, toX, toY) {
-    if (!this.config.mouseMovements.enabled) {
+    if (!this.config.mouseMovements.enabled || this.browserHumanizesCursor(page)) {
       await page.mouse.move(toX, toY);
       return;
     }
@@ -130,6 +130,25 @@ export class HumanBehaviorSimulator {
     this.stats.mouseMovements++;
   }
 
+  /**
+   * Whether the browser already animates every page.mouse.move itself.
+   *
+   * camoufox launched with `humanize` (StealthBrowserManager sets it from
+   * simulateHumanBehavior, on by default) draws its own curved path for each
+   * move, at ~1-2 s a call whatever the distance (measured 2026-09-29: 10 px
+   * took 950 ms, across the window 2.2 s). Our path is 11-101 moves, so the
+   * two stacked turned one click into 10-20 s and hit the action backstop; one
+   * move to the target is still a humanized path. camoufox is the only
+   * Firefox this server launches.
+   */
+  browserHumanizesCursor(page) {
+    try {
+      return page.context().browser()?.browserType().name() === 'firefox';
+    } catch {
+      return false;
+    }
+  }
+
   /**
    * Generate natural mouse movement path using Bezier curves
    */
@@ -312,7 +331,10 @@ export class HumanBehaviorSimulator {
    * Simulate human-like clicking with hover and delay
    */
   async simulateClick(page, selector, options = {}) {
-    const element = await page.waitForSelector(selector);
+    // `timeout` bounds the wait for the element; left unset it is Playwright's
+    // 30 s default. It is not a mouse.click option, so it stays out of those.
+    const { timeout, ...clickOptions } = options;
+    const element = await page.waitForSelector(selector, { timeout });
     const boundingBox = await element.boundingBox();
     
     if (!boundingBox) {
@@ -339,7 +361,7 @@ export class HumanBehaviorSimulator {
     await this.delay(clickDelay);
 
     // Perform the click
-    await page.mouse.click(clickX, clickY, options);
+    await page.mouse.click(clickX, clickY, clickOptions);
 
     this.stats.totalInteractions++;
   }
@@ -361,13 +383,13 @@ export class HumanBehaviorSimulator {
     } = options;
 
     if (target) {
-      // Scroll to specific element
-      await page.evaluate((sel) => {
-        const element = document.querySelector(sel);
-        if (element) {
-          element.scrollIntoView({ behavior: 'smooth', block: 'center' });
-        }
-      }, target);
+      // Scroll to specific element. A locator, not document.querySelector: a
+      // snapshot ref resolves to `aria-ref=…`, which is not CSS, and its element
+      // may sit in a shadow root or an iframe that querySelector cannot reach.
+      const element = page.locator(target).first();
+      if (await element.count()) {
+        await element.evaluate((el) => el.scrollIntoView({ behavior: 'smooth', block: 'center' }));
+      }
       return;
     }
 
@@ -455,10 +477,9 @@ export class HumanBehaviorSimulator {
     let textLength = 0;
     
     if (selector) {
-      textLength = await page.evaluate((sel) => {
-        const element = document.querySelector(sel);
-        return element ? element.textContent.length : 0;
-      }, selector);
+      // A locator for the same reason as simulateScroll's target.
+      const element = page.locator(selector).first();
+      textLength = (await element.count()) ? await element.evaluate((el) => el.textContent.length) : 0;
     } else {
       textLength = await page.evaluate(() => document.body.textContent.length);
     }