crawlforge-mcp-server 6.12.1 → 6.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.12.1",
+  "version": "6.14.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,6 +113,7 @@
     "package.json"
   ],
   "dependencies": {
+    "@duckduckgo/autoconsent": "^16.43.0",
     "@modelcontextprotocol/core": "^2.0.0",
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",

package/server.js

@@ -111,7 +111,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.12.1",
+  version: "6.14.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -941,13 +941,13 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements with stable refs (@e1, @e2 ...), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
     actions: z.array(z.object({
       type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']).describe("executeJavaScript is disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; on the hosted API it is refused."),
-      selector: z.string().optional().describe("A CSS selector, or a @e1 ref from an earlier snapshot action in this chain"),
+      selector: z.string().optional().describe("A CSS selector, or a ref from an earlier snapshot action in this chain (@e1, including elements inside shadow roots and iframes)"),
       text: z.string().optional(),
       key: z.string().optional(),
       script: z.string().optional(),
@@ -1014,7 +1014,8 @@ registerToolIfEnabled("scrape_with_actions", {
       viewportHeight: z.number().min(600).max(1080).default(720),
       timeout: z.number().min(10000).max(120000).default(30000),
       stealth: z.boolean().default(false).describe("Run the action chain in the stealth browser (randomized fingerprint, WebRTC/canvas spoofing) instead of the standard browser pool. Renders JavaScript; it does not solve challenges."),
-      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium.")
+      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium."),
+      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged.")
     }).optional().describe("Browser configuration options"),
     extractionOptions: z.object({
       selectors: z.record(z.string()).optional(),
@@ -1051,7 +1052,7 @@ registerToolIfEnabled("scrape_with_actions", {
 
 // Tool: browser_session
 registerToolIfEnabled("browser_session", {
-  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ...), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
+  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ..., including elements inside open shadow roots and iframes), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
   annotations: { title: "Browser Session", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     operation: z.enum(["open", "snapshot", "act", "read", "screenshot", "close", "list"]).describe("open a session, observe it, act on it, read it, or close it"),

package/src/core/ActionExecutor.js

@@ -10,6 +10,8 @@ import { createHash } from 'node:crypto';
 import { assertUrlAllowed } from '../utils/ssrfGuard.js';
 import { browserPreflight } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
+import { settlePage } from './browser/settle.js';
+import { handleConsent } from './browser/consent.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -303,6 +305,13 @@ export class ActionExecutor extends EventEmitter {
         page = await this.initializePage(url, browserOptions);
         executionContext.page = page;
 
+        // Cookie/consent wall on the landing page (browserOptions.consent,
+        // off unless asked for). A `navigate` action does the same for its
+        // page — see executeNavigateAction.
+        if (browserOptions.consent && browserOptions.consent !== 'off') {
+          executionContext.consent = await handleConsent(page, browserOptions.consent);
+        }
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
 
@@ -408,6 +417,7 @@ export class ActionExecutor extends EventEmitter {
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
+        consent: executionContext.consent,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
         attempt: executionContext.attempt,
@@ -799,14 +809,15 @@ export class ActionExecutor extends EventEmitter {
    * Resolve a caller-supplied selector.
    *
    * A selector starting with `@` names a ref a prior snapshot action assigned,
-   * and resolves to the attribute selector that snapshot stamped on the
-   * element; anything else is already a CSS selector. Every selector a caller
+   * and resolves to the Playwright selector that snapshot recorded for it
+   * (`aria-ref=…` from the native snapshot, `[data-cf-ref=…]` from the walk
+   * fallback); anything else is already a CSS selector. Every selector a caller
    * writes goes through here, so refs work in every action type without any
-   * schema change — and, because a ref resolves to an ordinary CSS selector,
-   * in the stealth human-behaviour paths too.
+   * schema change — including the stealth human-behaviour paths, which take
+   * Playwright selectors.
    * @param {Page} page - Playwright page
    * @param {string} selector - CSS selector, or an `@e1` snapshot ref
-   * @returns {string} CSS selector
+   * @returns {string} Playwright selector
    */
   resolveSelector(page, selector) {
     return isRef(selector) ? resolveRef(page, selector) : selector;
@@ -862,23 +873,25 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Give a navigation started by the action that just ran a chance to commit.
+   * Let the page settle after the action that just ran — a navigation commit
+   * and load, or an SPA re-render that fires no load event at all.
    *
    * Playwright auto-waits on the element it acts on, but nothing waits on the
-   * *document* a click or keypress may have replaced, so the next action could
-   * run against the outgoing page. A page that never navigated is already past
-   * this state and returns immediately; a page that doesn't settle in time is
-   * not itself an action failure, hence the catch.
+   * *document* a click or keypress may have changed, so the next action could
+   * run against the outgoing page. settlePage waits for `load` and then a quiet
+   * window (no DOM mutations, no requests in flight), capped and never longer
+   * than what is left of the action's `timeout`; a page that doesn't settle in
+   * time is not itself an action failure, so it never throws.
    * @param {Page} page - Playwright page
-   * @param {number} timeout - Timeout in ms
-   * @returns {Promise<void>}
+   * @param {number} timeout - The action's timeout in ms
+   * @param {number} startedAt - When the action started (Date.now()); the settle
+   *   gets only the remainder, so a slow element wait plus a busy page cannot
+   *   outrun the backstop and turn a successful action into a timeout
+   * @returns {Promise<{waited_ms: number, settled_by: string}>}
    */
-  async settleAfterInteraction(page, timeout) {
-    try {
-      await page.waitForLoadState('domcontentloaded', { timeout });
-    } catch {
-      // Ignored on purpose — see above.
-    }
+  async settleAfterInteraction(page, timeout, startedAt) {
+    const remaining = Math.max(1, timeout - (Date.now() - startedAt));
+    return await settlePage(page, { timeout: remaining });
   }
 
   /**
@@ -960,6 +973,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Click result
    */
   async executeClickAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const locator = this.elementLocator(page, action.selector);
 
@@ -977,7 +991,8 @@ export class ActionExecutor extends EventEmitter {
         button: action.button,
         clickCount: action.clickCount,
         delay: action.delay,
-        force: action.force
+        force: action.force,
+        timeout
       });
     } else {
       // Standard click behavior
@@ -998,7 +1013,7 @@ export class ActionExecutor extends EventEmitter {
 
     // A click can follow a link or submit a form; let that navigation commit
     // before the next action runs against the outgoing document.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1050,6 +1065,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Press result
    */
   async executePressAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const keyOptions = { timeout };
     if (action.modifiers?.length > 0) {
@@ -1063,7 +1079,7 @@ export class ActionExecutor extends EventEmitter {
     }
 
     // Enter on a form field navigates as often as a click does.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       key: action.key,
@@ -1161,6 +1177,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Select result
    */
   async executeSelectAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const values = action.values?.length ? action.values : [action.value];
 
@@ -1169,7 +1186,7 @@ export class ActionExecutor extends EventEmitter {
 
     // Faceted-search dropdowns commonly submit the form on change, so let any
     // navigation commit before the next action runs.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1214,6 +1231,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Navigate result
    */
   async executeNavigateAction(page, action, executionContext) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
@@ -1225,10 +1243,18 @@ export class ActionExecutor extends EventEmitter {
     });
     page.__crawlforgeGateWarnings = gateWarnings;
 
+    // Consent handling gets at most what is left of the action's deadline, so
+    // it cannot push a navigation that succeeded past the backstop.
+    const consentMode = executionContext?.browserOptions?.consent;
+    const consent = consentMode && consentMode !== 'off'
+      ? await handleConsent(page, consentMode, { timeout: Math.min(2000, timeout - (Date.now() - startedAt)) })
+      : undefined;
+
     return {
       url: action.url,
       finalUrl: page.url(),
-      waitUntil: action.waitUntil || 'domcontentloaded'
+      waitUntil: action.waitUntil || 'domcontentloaded',
+      ...(consent ? { consent } : {})
     };
   }
 
@@ -1314,8 +1340,8 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Execute snapshot action - the page's interactive elements, each stamped
-   * with a ref later actions can target instead of a guessed CSS selector.
+   * Execute snapshot action - the page's interactive elements, each carrying
+   * a ref later actions can target instead of a guessed CSS selector.
    * @param {Page} page - Playwright page
    * @param {Object} action - Snapshot action
    * @returns {Promise<Object>} Snapshot tree with refs
@@ -1323,7 +1349,9 @@ export class ActionExecutor extends EventEmitter {
   async executeSnapshotAction(page, action) {
     return await captureSnapshot(page, {
       interactiveOnly: action.interactiveOnly,
-      maxNodes: action.maxNodes
+      maxNodes: action.maxNodes,
+      // The render wait before the walk stays inside the action's deadline.
+      timeout: this.actionTimeout(action)
     });
   }
 

package/src/core/StealthBrowserManager.js

@@ -3638,8 +3638,10 @@ export class CamoufoxAdapter extends BrowserEngine {
     // `blockWebRTC: false` is the stealthier setting behind a proxy.
     if (config.blockWebRTC) options.block_webrtc = true;
     // Native cursor humanization: camoufox moves the pointer along a plausible
-    // path rather than teleporting it.
-    if (config.humanize) options.humanize = true;
+    // path rather than teleporting it. The number caps one move in seconds;
+    // `true` means camoufox's 1.5 s default, which made a single stealth click
+    // cost 2 s or more of cursor travel alone.
+    if (config.humanize) options.humanize = 0.5;
     // Pin the persona to the installed binary's Firefox major so the UA's two
     // version tokens agree. Left unpinned, camoufox's own version rewrite
     // reaches only `rv:` and about half of all launches announce a Firefox

package/src/core/browser/actionQueue.js

@@ -0,0 +1,81 @@
+/**
+ * actionQueue — the one process-wide budget of browser slots that
+ * `scrape_with_actions` chains and `browser_session` opens share.
+ *
+ * A caller over the limit WAITS for a slot instead of being refused: the live
+ * review's fourth parallel scrape_with_actions call was rejected outright with
+ * "Maximum concurrent sessions (3) reached". It now queues for up to
+ * CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS and only then fails, with a message that
+ * names the queue depth and the limit.
+ *
+ * A slot covers the slow part of a browser job — context/page creation and
+ * navigation — not the life of whatever it produced. browser_session takes one
+ * for `open` only (see BrowserSessionTool.openSession); its session caps live
+ * in SessionStore and are policy, which still refuses.
+ */
+
+import PQueue from 'p-queue';
+
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/** A positive integer from the environment, or the default for anything else. */
+function positiveInt(value, fallback) {
+  const n = Number.parseInt(value, 10);
+  return Number.isInteger(n) && n > 0 ? n : fallback;
+}
+
+/**
+ * @param {Object} [opts]
+ * @param {number} [opts.concurrency] — slots that may run at once
+ * @param {number} [opts.timeoutMs]   — longest a caller waits for a slot
+ */
+export function createActionQueue({
+  concurrency = positiveInt(process.env.CRAWLFORGE_MAX_ACTION_SESSIONS, DEFAULT_CONCURRENCY),
+  timeoutMs = positiveInt(process.env.CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS)
+} = {}) {
+  const queue = new PQueue({ concurrency });
+
+  /**
+   * Run `fn` once a slot is free, holding the slot until `fn` settles.
+   *
+   * The timeout bounds the WAIT, never the run: the timer is cleared the moment
+   * the job starts. That matters because p-queue races a running job against
+   * its signal and frees the slot on abort while `fn` carries on — an abort
+   * after start would let a fourth browser job in beside three live ones.
+   * Before start, the abort removes the job from the queue, so a caller that
+   * timed out never runs later.
+   */
+  function run(fn, { label = 'browser job' } = {}) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => {
+      // Read before the abort removes this job: the depth the caller waited behind.
+      controller.abort(new Error(
+        `${label} timed out after ${timeoutMs / 1000} s waiting for a browser slot: ` +
+        `${queue.pending} running, ${queue.size} queued ` +
+        `(CRAWLFORGE_MAX_ACTION_SESSIONS=${concurrency}). Retry shortly, or raise ` +
+        'CRAWLFORGE_MAX_ACTION_SESSIONS / CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS if the machine has room.'
+      ));
+    }, timeoutMs);
+    timer.unref?.();
+
+    return queue.add(() => {
+      clearTimeout(timer);
+      return fn();
+    }, { signal: controller.signal });
+  }
+
+  return {
+    run,
+    concurrency,
+    timeoutMs,
+    get running() { return queue.pending; },
+    get queued() { return queue.size; }
+  };
+}
+
+/**
+ * The process-wide limiter both tools use. Tools take an `actionQueue` option
+ * so a test can hand them a small private one; production never passes it.
+ */
+export const actionQueue = createActionQueue();

package/src/core/browser/consent.js

@@ -0,0 +1,182 @@
+/**
+ * Consent — answer a cookie/consent wall with @duckduckgo/autoconsent before
+ * the chain's first action sees the page (browserOptions.consent).
+ *
+ * autoconsent ships a content script plus a rule set; the content script
+ * detects the CMP and clicks its reject (optOut) or accept (optIn) button, and
+ * reports each step as a message. It has to run in every frame: several CMPs
+ * (Sourcepoint on theguardian.com) render inside a cross-origin iframe.
+ *
+ * Messages come back through a queue in each frame that this module drains on
+ * a short poll, not through page.exposeBinding. A binding can be registered
+ * only once per page (a second navigate would throw), and on camoufox
+ * page.evaluate runs in an isolated world that a main-world binding is not
+ * visible from. Draining a queue works the same on both engines.
+ *
+ * Engines: on Chromium frame.evaluate runs in the page's main world, so the
+ * content script and its globals (window.autoconsentReceiveMessage,
+ * window.__crawlforgeConsent) are visible to page scripts. On camoufox it runs
+ * in an isolated world: DOM clicks work the same, but the few rules that read
+ * a CMP's JavaScript API (`eval` steps, e.g. Cookiebot's, OneTrust's self-test)
+ * cannot see page globals there and evaluate to false, so those rules fall
+ * back to their DOM steps or do not match.
+ *
+ * Nothing here throws and nothing fails a chain: no CMP is `action: 'none'`.
+ * The content script is not stopped at the cap — a popup it is still working
+ * on may be answered after this returns.
+ */
+
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+
+const POLL_MS = 100;
+const AUTO_ACTION = { reject: 'optOut', accept: 'optIn' };
+const QUEUE_KEY = '__crawlforgeConsent';
+const KEPT_MESSAGES = ['cmpDetected', 'popupFound', 'optOutResult', 'optInResult', 'autoconsentDone'];
+
+// Read on first use, not at import: ~860 KB that a chain without consent
+// handling never needs.
+let bundle = null;
+function loadBundle() {
+  if (!bundle) {
+    const require = createRequire(import.meta.url);
+    const rulesPath = require.resolve('@duckduckgo/autoconsent/rules/rules.json');
+    // The package's exports map does not expose dist/, so it is reached from
+    // rules/ — the two ship side by side.
+    const script = readFileSync(join(dirname(rulesPath), '..', 'dist', 'autoconsent.playwright.js'), 'utf8');
+    const rules = JSON.parse(readFileSync(rulesPath, 'utf8')).autoconsent.map((rule) => ({
+      rule,
+      pattern: rule.runContext?.urlPattern ? new RegExp(rule.runContext.urlPattern) : null
+    }));
+    bundle = { script, rules };
+  }
+  return bundle;
+}
+
+// The rules that can apply to a frame (the content script checks again; this
+// only keeps a 700 KB rule set out of every frame). Defaults per autoconsent:
+// a rule runs in the main frame unless `main: false`, in a subframe only with
+// `frame: true`.
+function rulesFor(url, mainFrame) {
+  return loadBundle().rules
+    .filter(({ rule, pattern }) => {
+      const ctx = rule.runContext || {};
+      if (mainFrame ? ctx.main === false : ctx.frame !== true) return false;
+      return !pattern || pattern.test(url);
+    })
+    .map(({ rule }) => rule);
+}
+
+function injectScript(frame, autoAction, timeout, mainFrame) {
+  const config = {
+    enabled: true,
+    autoAction,
+    disabledCmps: [],
+    // Prehiding hides the popup before it is answered and unhides it if
+    // answering fails — a snapshot taken meanwhile would lie either way.
+    enablePrehide: false,
+    enableCosmeticRules: true,
+    // One detection attempt per 500 ms; stop looking at about the cap.
+    detectRetries: Math.max(1, Math.ceil(timeout / 500)),
+    // Run eval snippets inline in whatever world evaluate() gave us instead of
+    // round-tripping them through the message queue (see the header).
+    isMainWorld: true
+  };
+  const initResp = { type: 'initResp', config, rules: { autoconsent: rulesFor(frame.url(), mainFrame) } };
+  return `window.${QUEUE_KEY} = [];
+window.autoconsentSendMessage = (m) => {
+  if (${JSON.stringify(KEPT_MESSAGES)}.includes(m.type)) window.${QUEUE_KEY}.push(m);
+  return Promise.resolve();
+};
+${loadBundle().script}
+;window.autoconsentReceiveMessage(${JSON.stringify(initResp)});
+true;`;
+}
+
+const DRAIN_SCRIPT = `(() => { const q = window.${QUEUE_KEY}; return q ? q.splice(0) : null; })()`;
+
+// This frame's new messages; injects the content script into a document that
+// has none yet (first poll, or the frame navigated since).
+async function drainFrame(frame, autoAction, timeout, mainFrame) {
+  try {
+    const messages = await frame.evaluate(DRAIN_SCRIPT);
+    if (messages) return messages;
+    await frame.evaluate(injectScript(frame, autoAction, timeout, mainFrame));
+  } catch {
+    // Detached or navigating frame: the next poll sees its new document.
+  }
+  return [];
+}
+
+/**
+ * Answer the page's consent wall, if it has one autoconsent knows.
+ *
+ * @param {Page} page - Playwright page, already navigated
+ * @param {'off'|'reject'|'accept'} mode - browserOptions.consent
+ * @param {{timeout?: number}} [options] - cap in ms (default 2000)
+ * @returns {Promise<{cmp: string|null, action: string, ms: number}|null>}
+ *   null for 'off'. `action` is 'optOut' / 'optIn' when the popup was
+ *   answered, 'hide' when a cosmetic rule hid it, 'error' when the rule ran
+ *   and failed, 'timeout' when a popup was found but not answered within the
+ *   cap, and 'none' when no popup was found.
+ */
+export async function handleConsent(page, mode, { timeout = 2000 } = {}) {
+  const autoAction = AUTO_ACTION[mode];
+  if (!autoAction) return null;
+
+  const start = Date.now();
+  const deadline = start + Math.max(0, timeout);
+  let cmp = null;
+  let action = 'none';
+
+  try {
+    const mainFrame = page.mainFrame();
+    // Frame -> true while a CMP it detected is still unanswered. One page can
+    // need two frames: on theguardian.com a top-frame rule hides Sourcepoint's
+    // container at once, and the rule in Sourcepoint's iframe then makes the
+    // actual choice — returning on the first `done` would report the hide.
+    const pending = new Map();
+    let finished = false;
+    while (!finished && Date.now() < deadline) {
+      const frames = page.frames().filter((f) => f === mainFrame || /^https?:/i.test(f.url()));
+      let timer;
+      const batches = await Promise.race([
+        Promise.all(frames.map((f) => drainFrame(f, autoAction, timeout, f === mainFrame))),
+        new Promise((resolve) => { timer = setTimeout(() => resolve([]), deadline - Date.now()); })
+      ]);
+      clearTimeout(timer);
+
+      batches.forEach((messages, i) => {
+        for (const m of messages) {
+          if (m.type === 'cmpDetected') {
+            if (!pending.has(frames[i])) pending.set(frames[i], true);
+            continue;
+          }
+          cmp = m.cmp;
+          if (m.type === 'popupFound') {
+            if (action === 'none') action = 'timeout';
+          } else if (m.type === 'autoconsentDone') {
+            action = m.isCosmetic ? 'hide' : autoAction;
+            pending.set(frames[i], false);
+          } else if (!m.result) {
+            // optOutResult/optInResult false: the rule ran and did not work.
+            action = 'error';
+            pending.set(frames[i], false);
+          }
+        }
+      });
+      // A CMP iframe removed mid-answer was closed by its own choice
+      // (Sourcepoint drops its iframe on one) — its done message goes with it.
+      for (const [frame, open] of pending) {
+        if (open && frame.isDetached()) pending.set(frame, false);
+      }
+      finished = action !== 'none' && action !== 'timeout' && ![...pending.values()].includes(true);
+      if (!finished) await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(POLL_MS, deadline - Date.now()))));
+    }
+  } catch {
+    action = 'error';
+  }
+
+  return { cmp, action, ms: Date.now() - start };
+}