crawlforge-mcp-server 6.12.0 → 6.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.12.0",
+  "version": "6.14.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,6 +113,7 @@
     "package.json"
   ],
   "dependencies": {
+    "@duckduckgo/autoconsent": "^16.43.0",
     "@modelcontextprotocol/core": "^2.0.0",
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",

package/server.js

@@ -41,6 +41,7 @@ import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 import AuthManager from "./src/core/AuthManager.js";
 import { makeWithAuth } from "./src/server/withAuth.js";
+import { stripScreenshotData } from "./src/server/screenshotResources.js";
 // Transport helpers
 import { connectStdio } from "./src/server/transports/stdio.js";
 import { connectStreamableHttp } from "./src/server/transports/streamableHttp.js";
@@ -110,7 +111,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.12.0",
+  version: "6.14.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -940,13 +941,13 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript, select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements with stable refs (@e1, @e2 ...), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
     actions: z.array(z.object({
-      type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']),
-      selector: z.string().optional().describe("A CSS selector, or a @e1 ref from an earlier snapshot action in this chain"),
+      type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']).describe("executeJavaScript is disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; on the hosted API it is refused."),
+      selector: z.string().optional().describe("A CSS selector, or a ref from an earlier snapshot action in this chain (@e1, including elements inside shadow roots and iframes)"),
       text: z.string().optional(),
       key: z.string().optional(),
       script: z.string().optional(),
@@ -1013,7 +1014,8 @@ registerToolIfEnabled("scrape_with_actions", {
       viewportHeight: z.number().min(600).max(1080).default(720),
       timeout: z.number().min(10000).max(120000).default(30000),
       stealth: z.boolean().default(false).describe("Run the action chain in the stealth browser (randomized fingerprint, WebRTC/canvas spoofing) instead of the standard browser pool. Renders JavaScript; it does not solve challenges."),
-      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium.")
+      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium."),
+      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged.")
     }).optional().describe("Browser configuration options"),
     extractionOptions: z.object({
       selectors: z.record(z.string()).optional(),
@@ -1022,7 +1024,7 @@ registerToolIfEnabled("scrape_with_actions", {
       includeImages: z.boolean().default(true)
     }).optional().describe("Content extraction options. selectors results are returned as content.json.extracted, so include \"json\" in formats when passing selectors — without it the extraction is not part of the response."),
     continueOnActionError: z.boolean().default(false).describe("Continue executing actions if one fails"),
-    maxRetries: z.number().min(0).max(3).default(1).describe("Maximum retry attempts on failure"),
+    maxRetries: z.number().min(0).max(3).default(0).describe("Whole-chain retries on failure (0-3). A retry re-navigates to the starting URL and replays every action; each attempt is reported under attempts[]."),
     screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
     respect_robots: COMPLIANCE_PARAMS.respect_robots,
     ...MAX_INLINE_CHARS_PARAM,
@@ -1036,17 +1038,11 @@ registerToolIfEnabled("scrape_with_actions", {
     // resources (the documented contract) and annotate each with its URI.
     // The base64 `data` is dropped once stored, as the stealth_mode block
     // below already does: a failed chain's error screenshot came back inline
-    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09).
-    if (Array.isArray(result.screenshots)) {
-      result.screenshots = result.screenshots.map((shot) => {
-        if (shot?.actionId && shot?.data) {
-          resourceRegistry.storeScreenshot(shot.actionId, shot.data);
-          const { data, ...rest } = shot;
-          return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
-        }
-        return shot;
-      });
-    }
+    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09). The
+    // same bytes also sit in actionResults[].result, content.screenshots and
+    // attempts[].results, so every copy is stripped, not just the top-level
+    // array (Phase 0, 0.4).
+    stripScreenshotData(result, (actionId, data) => resourceRegistry.storeScreenshot(actionId, data));
 
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {
@@ -1056,7 +1052,7 @@ registerToolIfEnabled("scrape_with_actions", {
 
 // Tool: browser_session
 registerToolIfEnabled("browser_session", {
-  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ...), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
+  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ..., including elements inside open shadow roots and iframes), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
   annotations: { title: "Browser Session", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     operation: z.enum(["open", "snapshot", "act", "read", "screenshot", "close", "list"]).describe("open a session, observe it, act on it, read it, or close it"),

package/src/cli/commands/actions.js

@@ -12,6 +12,7 @@ export function register(program) {
     .description('Run browser automation actions against a URL')
     .requiredOption('--script <file>', 'JSON file containing action script')
     .option('--screenshot', 'Capture screenshots during action execution')
+    .addHelpText('after', '\nexecuteJavaScript actions need ALLOW_JAVASCRIPT_EXECUTION=true in the server environment; otherwise they are refused.\n')
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };

package/src/core/ActionExecutor.js

@@ -10,6 +10,8 @@ import { createHash } from 'node:crypto';
 import { assertUrlAllowed } from '../utils/ssrfGuard.js';
 import { browserPreflight } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
+import { settlePage } from './browser/settle.js';
+import { handleConsent } from './browser/consent.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -22,11 +24,12 @@ const JS_EXECUTION_TIMEOUT_MS = parseInt(process.env.JS_EXECUTION_TIMEOUT_MS ||
 // for, the backstop can only say "timed out".
 const ACTION_TIMEOUT_GRACE_MS = 2000;
 
-// Ceiling for a single error-recovery strategy. By the time recovery runs the
-// action has already spent its whole timeout failing, so each strategy gets a
-// bounded slice — granting it another full deadline made a chain that was never
-// going to work cost several times its stated timeout.
-const RECOVERY_TIMEOUT_MS = 5000;
+// Total error-recovery budget per action, shared by every strategy it tries
+// (their delay() pauses included). By the time recovery runs the action has
+// already spent its whole timeout failing, so recovery gets one bounded slice —
+// giving each strategy another full deadline made a chain that was never going
+// to work cost several times its stated timeout.
+const RECOVERY_TIMEOUT_MS = 3000;
 
 // The only states locator.waitFor()/page.waitForSelector() accept. The rest of
 // the wait-action enum (enabled/disabled/stable) are ElementHandle states and
@@ -176,6 +179,8 @@ const ActionChainSchema = z.object({
   continueOnError: z.boolean().default(false),
   timeout: z.number().min(1000).max(300000).default(30000),
   retryChain: z.number().min(0).max(3).default(0),
+  // Per-chain override of the constructor's enableScreenshotOnError.
+  screenshotOnError: z.boolean().optional(),
   metadata: z.record(z.any()).prefault({})
 });
 
@@ -272,6 +277,10 @@ export class ActionExecutor extends EventEmitter {
         browserOptions,
         startTime,
         results: [],
+        // One entry per run of the chain (retryChain replays it); `attempt` is
+        // the 1-based run whose per-action results are in `results`.
+        attempt: 0,
+        attempts: [],
         errors: [],
         screenshots: [],
         metadata: {
@@ -296,6 +305,13 @@ export class ActionExecutor extends EventEmitter {
         page = await this.initializePage(url, browserOptions);
         executionContext.page = page;
 
+        // Cookie/consent wall on the landing page (browserOptions.consent,
+        // off unless asked for). A `navigate` action does the same for its
+        // page — see executeNavigateAction.
+        if (browserOptions.consent && browserOptions.consent !== 'off') {
+          executionContext.consent = await handleConsent(page, browserOptions.consent);
+        }
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
 
@@ -318,7 +334,7 @@ export class ActionExecutor extends EventEmitter {
         executionContext.error = error.message;
 
         // Capture error screenshot if enabled
-        if (this.enableScreenshotOnError && page) {
+        if ((validatedChain.screenshotOnError ?? this.enableScreenshotOnError) && page) {
           try {
             const errorScreenshot = await this.captureScreenshot(page, {
               fullPage: true,
@@ -378,6 +394,7 @@ export class ActionExecutor extends EventEmitter {
           screenshotCount: executionContext.screenshots.length,
           capturedStates: undefined,
           capturedStateCount: (executionContext.capturedStates || []).length,
+          attempts: undefined, // each carries a copy of a results array
           results: executionContext.results.map(r => (
             r?.result?.data !== undefined
               ? { ...r, result: { ...r.result, data: undefined, dataBytes: typeof r.result.data === 'string' ? r.result.data.length : undefined } }
@@ -400,8 +417,11 @@ export class ActionExecutor extends EventEmitter {
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
+        consent: executionContext.consent,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
+        attempt: executionContext.attempt,
+        attempts: executionContext.attempts,
         screenshots: executionContext.screenshots,
         capturedStates: executionContext.capturedStates || [],
         metadata: executionContext.metadata,
@@ -424,6 +444,8 @@ export class ActionExecutor extends EventEmitter {
         // results, the error screenshot, and any intermediate-state
         // captures) instead of discarding them.
         results: executionContext?.results || [],
+        attempt: executionContext?.attempt ?? 0,
+        attempts: executionContext?.attempts || [],
         screenshots: executionContext?.screenshots || [],
         capturedStates: executionContext?.capturedStates || []
       };
@@ -440,9 +462,11 @@ export class ActionExecutor extends EventEmitter {
     let lastError;
 
     for (let attempt = 0; attempt <= chain.retryChain; attempt++) {
+      executionContext.attempt = attempt + 1;
       try {
         if (attempt > 0) {
           this.log('info', 'Retrying chain execution, attempt ' + (attempt + 1));
+          // The failed run's results are already recorded in attempts[].
           executionContext.results = []; // Clear previous results on retry
           executionContext.capturedStates = []; // Clear previous captures on retry
           // Replaying the chain against whatever the failed attempt left behind
@@ -454,7 +478,14 @@ export class ActionExecutor extends EventEmitter {
 
         // Execute actions in sequence
         for (let i = 0; i < chain.actions.length; i++) {
-          const action = chain.actions[i];
+          // chain.timeout is the deadline for actions that name none and the
+          // ceiling for those that do — actionTimeout() reads it off the
+          // action, so without this the chain's timeout never reached a
+          // Playwright call (executeActionsOnPage applies the same default).
+          const action = {
+            ...chain.actions[i],
+            timeout: Math.min(chain.actions[i].timeout || chain.timeout, chain.timeout)
+          };
           const actionResult = await this.executeActionInternal(page, action, executionContext);
           
           executionContext.results.push(actionResult);
@@ -509,10 +540,16 @@ export class ActionExecutor extends EventEmitter {
           }
         }
 
+        executionContext.attempts.push({
+          attempt: attempt + 1, success: true, results: [...executionContext.results]
+        });
         return { success: true, attempt: attempt + 1 };
 
       } catch (error) {
         lastError = error;
+        executionContext.attempts.push({
+          attempt: attempt + 1, success: false, error: error.message, results: [...executionContext.results]
+        });
         this.log('warn', 'Chain execution attempt ' + (attempt + 1) + ' failed: ' + error.message);
         
         if (attempt < chain.retryChain) {
@@ -754,26 +791,33 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Deadline for one recovery strategy — bounded, see RECOVERY_TIMEOUT_MS.
+   * Deadline for one Playwright call inside a recovery strategy: what is left
+   * of the shared budget (see RECOVERY_TIMEOUT_MS), never more than the
+   * action's own timeout. Strategies call it after their delay() so the pause
+   * is charged to the same budget. Floored at 1 because Playwright reads a
+   * timeout of 0 as "wait forever".
    * @param {Object} action - Action configuration
+   * @param {number} [deadline] - Epoch ms at which the recovery budget ends
    * @returns {number} Timeout in ms
    */
-  recoveryTimeout(action) {
-    return Math.min(this.actionTimeout(action), RECOVERY_TIMEOUT_MS);
+  recoveryTimeout(action, deadline) {
+    const left = deadline === undefined ? RECOVERY_TIMEOUT_MS : deadline - Date.now();
+    return Math.max(1, Math.min(this.actionTimeout(action), left));
   }
 
   /**
    * Resolve a caller-supplied selector.
    *
    * A selector starting with `@` names a ref a prior snapshot action assigned,
-   * and resolves to the attribute selector that snapshot stamped on the
-   * element; anything else is already a CSS selector. Every selector a caller
+   * and resolves to the Playwright selector that snapshot recorded for it
+   * (`aria-ref=…` from the native snapshot, `[data-cf-ref=…]` from the walk
+   * fallback); anything else is already a CSS selector. Every selector a caller
    * writes goes through here, so refs work in every action type without any
-   * schema change — and, because a ref resolves to an ordinary CSS selector,
-   * in the stealth human-behaviour paths too.
+   * schema change — including the stealth human-behaviour paths, which take
+   * Playwright selectors.
    * @param {Page} page - Playwright page
    * @param {string} selector - CSS selector, or an `@e1` snapshot ref
-   * @returns {string} CSS selector
+   * @returns {string} Playwright selector
    */
   resolveSelector(page, selector) {
     return isRef(selector) ? resolveRef(page, selector) : selector;
@@ -829,23 +873,25 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Give a navigation started by the action that just ran a chance to commit.
+   * Let the page settle after the action that just ran — a navigation commit
+   * and load, or an SPA re-render that fires no load event at all.
    *
    * Playwright auto-waits on the element it acts on, but nothing waits on the
-   * *document* a click or keypress may have replaced, so the next action could
-   * run against the outgoing page. A page that never navigated is already past
-   * this state and returns immediately; a page that doesn't settle in time is
-   * not itself an action failure, hence the catch.
+   * *document* a click or keypress may have changed, so the next action could
+   * run against the outgoing page. settlePage waits for `load` and then a quiet
+   * window (no DOM mutations, no requests in flight), capped and never longer
+   * than what is left of the action's `timeout`; a page that doesn't settle in
+   * time is not itself an action failure, so it never throws.
    * @param {Page} page - Playwright page
-   * @param {number} timeout - Timeout in ms
-   * @returns {Promise<void>}
+   * @param {number} timeout - The action's timeout in ms
+   * @param {number} startedAt - When the action started (Date.now()); the settle
+   *   gets only the remainder, so a slow element wait plus a busy page cannot
+   *   outrun the backstop and turn a successful action into a timeout
+   * @returns {Promise<{waited_ms: number, settled_by: string}>}
    */
-  async settleAfterInteraction(page, timeout) {
-    try {
-      await page.waitForLoadState('domcontentloaded', { timeout });
-    } catch {
-      // Ignored on purpose — see above.
-    }
+  async settleAfterInteraction(page, timeout, startedAt) {
+    const remaining = Math.max(1, timeout - (Date.now() - startedAt));
+    return await settlePage(page, { timeout: remaining });
   }
 
   /**
@@ -927,6 +973,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Click result
    */
   async executeClickAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const locator = this.elementLocator(page, action.selector);
 
@@ -944,7 +991,8 @@ export class ActionExecutor extends EventEmitter {
         button: action.button,
         clickCount: action.clickCount,
         delay: action.delay,
-        force: action.force
+        force: action.force,
+        timeout
       });
     } else {
       // Standard click behavior
@@ -965,7 +1013,7 @@ export class ActionExecutor extends EventEmitter {
 
     // A click can follow a link or submit a form; let that navigation commit
     // before the next action runs against the outgoing document.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1017,6 +1065,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Press result
    */
   async executePressAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const keyOptions = { timeout };
     if (action.modifiers?.length > 0) {
@@ -1030,7 +1079,7 @@ export class ActionExecutor extends EventEmitter {
     }
 
     // Enter on a form field navigates as often as a click does.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       key: action.key,
@@ -1128,6 +1177,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Select result
    */
   async executeSelectAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const values = action.values?.length ? action.values : [action.value];
 
@@ -1136,7 +1186,7 @@ export class ActionExecutor extends EventEmitter {
 
     // Faceted-search dropdowns commonly submit the form on change, so let any
     // navigation commit before the next action runs.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1181,6 +1231,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Navigate result
    */
   async executeNavigateAction(page, action, executionContext) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
@@ -1192,10 +1243,18 @@ export class ActionExecutor extends EventEmitter {
     });
     page.__crawlforgeGateWarnings = gateWarnings;
 
+    // Consent handling gets at most what is left of the action's deadline, so
+    // it cannot push a navigation that succeeded past the backstop.
+    const consentMode = executionContext?.browserOptions?.consent;
+    const consent = consentMode && consentMode !== 'off'
+      ? await handleConsent(page, consentMode, { timeout: Math.min(2000, timeout - (Date.now() - startedAt)) })
+      : undefined;
+
     return {
       url: action.url,
       finalUrl: page.url(),
-      waitUntil: action.waitUntil || 'domcontentloaded'
+      waitUntil: action.waitUntil || 'domcontentloaded',
+      ...(consent ? { consent } : {})
     };
   }
 
@@ -1281,8 +1340,8 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Execute snapshot action - the page's interactive elements, each stamped
-   * with a ref later actions can target instead of a guessed CSS selector.
+   * Execute snapshot action - the page's interactive elements, each carrying
+   * a ref later actions can target instead of a guessed CSS selector.
    * @param {Page} page - Playwright page
    * @param {Object} action - Snapshot action
    * @returns {Promise<Object>} Snapshot tree with refs
@@ -1290,7 +1349,9 @@ export class ActionExecutor extends EventEmitter {
   async executeSnapshotAction(page, action) {
     return await captureSnapshot(page, {
       interactiveOnly: action.interactiveOnly,
-      maxNodes: action.maxNodes
+      maxNodes: action.maxNodes,
+      // The render wait before the walk stays inside the action's deadline.
+      timeout: this.actionTimeout(action)
     });
   }
 
@@ -1500,16 +1561,35 @@ export class ActionExecutor extends EventEmitter {
       return { success: false };
     }
 
+    // An element that is not in the document cannot be force-clicked, scrolled
+    // into view or focused, so every strategy below would only spend the
+    // budget re-waiting for it. count() answers at once without waiting.
+    if (action.selector) {
+      let matches = 1;
+      try {
+        matches = await this.elementLocator(page, action.selector).count();
+      } catch (_) {
+        // A ref that cannot be resolved, or a page stub in tests: let the
+        // strategies decide.
+      }
+      if (matches === 0) return { success: false };
+    }
+
     const strategies = this.errorRecoveryStrategies.get(action.type) || [];
     // `retries` caps how many strategies get a turn. Walking all of them
     // unconditionally would add a second full round of timeouts to every action
     // that was never going to succeed.
     const budget = Math.max(0, action.retries ?? 1);
+    // One deadline shared by every strategy this action tries, so recovery
+    // costs at most RECOVERY_TIMEOUT_MS on top of the action's own timeout
+    // however many strategies are registered.
+    const deadline = Date.now() + RECOVERY_TIMEOUT_MS;
 
     for (const strategy of strategies.slice(0, budget)) {
+      if (Date.now() >= deadline) break;
       try {
         this.log('info', 'Attempting error recovery with strategy: ' + strategy.name);
-        const result = await strategy.recover(page, action, error, executionContext);
+        const result = await strategy.recover(page, action, error, executionContext, deadline);
         
         if (result.success) {
           return {
@@ -1534,21 +1614,20 @@ export class ActionExecutor extends EventEmitter {
     this.errorRecoveryStrategies.set('click', [
       {
         name: 'waitAndRetry',
-        recover: async (page, action) => {
+        recover: async (page, action, _error, _context, deadline) => {
           await this.delay(1000);
           await this.elementLocator(page, action.selector)
-            .click({ force: true, timeout: this.recoveryTimeout(action) });
+            .click({ force: true, timeout: this.recoveryTimeout(action, deadline) });
           return { success: true, data: { recovered: true, strategy: 'waitAndRetry' } };
         }
       },
       {
         name: 'scrollIntoView',
-        recover: async (page, action) => {
-          const timeout = this.recoveryTimeout(action);
+        recover: async (page, action, _error, _context, deadline) => {
           const locator = this.elementLocator(page, action.selector);
-          await locator.scrollIntoViewIfNeeded({ timeout });
+          await locator.scrollIntoViewIfNeeded({ timeout: this.recoveryTimeout(action, deadline) });
           await this.delay(500);
-          await locator.click({ timeout });
+          await locator.click({ timeout: this.recoveryTimeout(action, deadline) });
           return { success: true, data: { recovered: true, strategy: 'scrollIntoView' } };
         }
       }
@@ -1558,33 +1637,20 @@ export class ActionExecutor extends EventEmitter {
     this.errorRecoveryStrategies.set('type', [
       {
         name: 'focusAndRetry',
-        recover: async (page, action) => {
-          const timeout = this.recoveryTimeout(action);
+        recover: async (page, action, _error, _context, deadline) => {
           const locator = this.elementLocator(page, action.selector);
-          await locator.focus({ timeout });
+          await locator.focus({ timeout: this.recoveryTimeout(action, deadline) });
           await this.delay(500);
-          await locator.pressSequentially(action.text, { delay: action.delay, timeout });
+          await locator.pressSequentially(action.text, {
+            delay: action.delay, timeout: this.recoveryTimeout(action, deadline)
+          });
           return { success: true, data: { recovered: true, strategy: 'focusAndRetry' } };
         }
       }
     ]);
 
-    // Wait action recovery strategies
-    this.errorRecoveryStrategies.set('wait', [
-      {
-        name: 'extendTimeout',
-        recover: async (page, action) => {
-          if (!action.selector) return { success: false };
-          // One more bounded window, not a doubled one: the action has already
-          // waited its full timeout, so doubling made a wait that could never
-          // resolve cost 3x what the caller asked for.
-          await this.waitForCondition(
-            page, action.selector, action.condition, this.recoveryTimeout(action)
-          );
-          return { success: true, data: { recovered: true, strategy: 'extendTimeout' } };
-        }
-      }
-    ]);
+    // No strategies for `wait`: the wait already was the retry, so a failed
+    // one has nothing to recover and gets no second window.
   }
 
   /**

package/src/core/StealthBrowserManager.js

@@ -3638,8 +3638,10 @@ export class CamoufoxAdapter extends BrowserEngine {
     // `blockWebRTC: false` is the stealthier setting behind a proxy.
     if (config.blockWebRTC) options.block_webrtc = true;
     // Native cursor humanization: camoufox moves the pointer along a plausible
-    // path rather than teleporting it.
-    if (config.humanize) options.humanize = true;
+    // path rather than teleporting it. The number caps one move in seconds;
+    // `true` means camoufox's 1.5 s default, which made a single stealth click
+    // cost 2 s or more of cursor travel alone.
+    if (config.humanize) options.humanize = 0.5;
     // Pin the persona to the installed binary's Firefox major so the UA's two
     // version tokens agree. Left unpinned, camoufox's own version rewrite
     // reaches only `rv:` and about half of all launches announce a Firefox

package/src/core/browser/actionQueue.js

@@ -0,0 +1,81 @@
+/**
+ * actionQueue — the one process-wide budget of browser slots that
+ * `scrape_with_actions` chains and `browser_session` opens share.
+ *
+ * A caller over the limit WAITS for a slot instead of being refused: the live
+ * review's fourth parallel scrape_with_actions call was rejected outright with
+ * "Maximum concurrent sessions (3) reached". It now queues for up to
+ * CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS and only then fails, with a message that
+ * names the queue depth and the limit.
+ *
+ * A slot covers the slow part of a browser job — context/page creation and
+ * navigation — not the life of whatever it produced. browser_session takes one
+ * for `open` only (see BrowserSessionTool.openSession); its session caps live
+ * in SessionStore and are policy, which still refuses.
+ */
+
+import PQueue from 'p-queue';
+
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/** A positive integer from the environment, or the default for anything else. */
+function positiveInt(value, fallback) {
+  const n = Number.parseInt(value, 10);
+  return Number.isInteger(n) && n > 0 ? n : fallback;
+}
+
+/**
+ * @param {Object} [opts]
+ * @param {number} [opts.concurrency] — slots that may run at once
+ * @param {number} [opts.timeoutMs]   — longest a caller waits for a slot
+ */
+export function createActionQueue({
+  concurrency = positiveInt(process.env.CRAWLFORGE_MAX_ACTION_SESSIONS, DEFAULT_CONCURRENCY),
+  timeoutMs = positiveInt(process.env.CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS)
+} = {}) {
+  const queue = new PQueue({ concurrency });
+
+  /**
+   * Run `fn` once a slot is free, holding the slot until `fn` settles.
+   *
+   * The timeout bounds the WAIT, never the run: the timer is cleared the moment
+   * the job starts. That matters because p-queue races a running job against
+   * its signal and frees the slot on abort while `fn` carries on — an abort
+   * after start would let a fourth browser job in beside three live ones.
+   * Before start, the abort removes the job from the queue, so a caller that
+   * timed out never runs later.
+   */
+  function run(fn, { label = 'browser job' } = {}) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => {
+      // Read before the abort removes this job: the depth the caller waited behind.
+      controller.abort(new Error(
+        `${label} timed out after ${timeoutMs / 1000} s waiting for a browser slot: ` +
+        `${queue.pending} running, ${queue.size} queued ` +
+        `(CRAWLFORGE_MAX_ACTION_SESSIONS=${concurrency}). Retry shortly, or raise ` +
+        'CRAWLFORGE_MAX_ACTION_SESSIONS / CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS if the machine has room.'
+      ));
+    }, timeoutMs);
+    timer.unref?.();
+
+    return queue.add(() => {
+      clearTimeout(timer);
+      return fn();
+    }, { signal: controller.signal });
+  }
+
+  return {
+    run,
+    concurrency,
+    timeoutMs,
+    get running() { return queue.pending; },
+    get queued() { return queue.size; }
+  };
+}
+
+/**
+ * The process-wide limiter both tools use. Tools take an `actionQueue` option
+ * so a test can hand them a small private one; production never passes it.
+ */
+export const actionQueue = createActionQueue();