crawlforge-mcp-server 6.12.1 → 6.16.0

package/src/core/StealthBrowserManager.js

@@ -17,11 +17,12 @@ import path from 'path';
 import { createRequire } from 'module';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
-import { safeGoto } from '../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { looksLikeInterstitial, detectChallengePage } from '../utils/challengeDetection.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
 import { ClearanceJar, sharedClearanceJar } from './ClearanceJar.js';
+import { readWindowState } from './browser/windowState.js';
 
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
@@ -2844,7 +2845,16 @@ export class StealthBrowserManager {
     return this._clickTurnstile(page);
   }
 
-  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
+  /**
+   * @param {object} options
+   * @param {boolean} [options.readWindowState] also read the framework globals
+   *   off `window` once the page has settled (extract_embedded_state's
+   *   escalation, src/core/browser/windowState.js); returned as `windowState`
+   * @param {(to: string) => Promise<void>} [options.onRedirect] the caller's
+   *   gate for wherever the page is redirected to (robotsGate.js
+   *   `redirectGate`); a refusal throws and nothing of the page is returned
+   */
+  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {}, readWindowState: wantWindowState = false, onRedirect } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
     const { contextId, engineFallbackWarning = null } = await this.createStealthContext({ ...stealthConfig, engine });
@@ -2870,7 +2880,8 @@ export class StealthBrowserManager {
       });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
-      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded', onRedirect });
+      const landedUrl = page.url();
       if (response && status === null) status = response.status();
       if (wait_for > 0) await page.waitForTimeout(wait_for);
       // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
@@ -2888,6 +2899,10 @@ export class StealthBrowserManager {
       // a successful scrape.
       const renderMs = await this._settleRender(page);
       const gracedMs = emptyGraceMs + renderMs;
+      // The waits above are time in which a challenge or a client-side redirect
+      // can move the page on. The document read below is the one that has to
+      // pass, so wherever the page stands now is checked like a redirect hop.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2913,6 +2928,12 @@ export class StealthBrowserManager {
       const shot = screenshot
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
+      let windowState;
+      if (wantWindowState) {
+        const read = await readWindowState(page);
+        windowState = read.state;
+        warnings.push(...read.warnings);
+      }
 
       // Still a wall after everything above: whatever clearance this context
       // carried for the site did not work, so closeContext drops it instead of
@@ -2921,7 +2942,7 @@ export class StealthBrowserManager {
         this._markBlocked(contextId, [url, page.url()]);
       }
 
-      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings };
+      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings, ...(windowState ? { windowState } : {}) };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }
@@ -3638,8 +3659,10 @@ export class CamoufoxAdapter extends BrowserEngine {
     // `blockWebRTC: false` is the stealthier setting behind a proxy.
     if (config.blockWebRTC) options.block_webrtc = true;
     // Native cursor humanization: camoufox moves the pointer along a plausible
-    // path rather than teleporting it.
-    if (config.humanize) options.humanize = true;
+    // path rather than teleporting it. The number caps one move in seconds;
+    // `true` means camoufox's 1.5 s default, which made a single stealth click
+    // cost 2 s or more of cursor travel alone.
+    if (config.humanize) options.humanize = 0.5;
     // Pin the persona to the installed binary's Firefox major so the UA's two
     // version tokens agree. Left unpinned, camoufox's own version rewrite
     // reaches only `rv:` and about half of all launches announce a Firefox

package/src/core/browser/actionQueue.js

@@ -0,0 +1,84 @@
+/**
+ * actionQueue — the one process-wide budget of browser slots that
+ * `scrape_with_actions` chains and `browser_session` opens share.
+ *
+ * A caller over the limit WAITS for a slot instead of being refused: the live
+ * review's fourth parallel scrape_with_actions call was rejected outright with
+ * "Maximum concurrent sessions (3) reached". It now queues for up to
+ * CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS and only then fails, with a message that
+ * names the queue depth and the limit.
+ *
+ * A slot covers the slow part of a browser job — context/page creation and
+ * navigation — not the life of whatever it produced. browser_session takes one
+ * for `open` only (see BrowserSessionTool.openSession); its session caps live
+ * in SessionStore and are policy, which still refuses.
+ */
+
+import PQueue from 'p-queue';
+
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/** A positive integer from the environment, or the default for anything else. */
+function positiveInt(value, fallback) {
+  const n = Number.parseInt(value, 10);
+  return Number.isInteger(n) && n > 0 ? n : fallback;
+}
+
+/**
+ * @param {Object} [opts]
+ * @param {number} [opts.concurrency] — slots that may run at once
+ * @param {number} [opts.timeoutMs]   — longest a caller waits for a slot
+ */
+export function createActionQueue({
+  concurrency = positiveInt(process.env.CRAWLFORGE_MAX_ACTION_SESSIONS, DEFAULT_CONCURRENCY),
+  timeoutMs = positiveInt(process.env.CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS)
+} = {}) {
+  const queue = new PQueue({ concurrency });
+
+  /**
+   * Run `fn` once a slot is free, holding the slot until `fn` settles.
+   *
+   * The timeout bounds the WAIT, never the run: the timer is cleared the moment
+   * the job starts. That matters because p-queue races a running job against
+   * its signal and frees the slot on abort while `fn` carries on — an abort
+   * after start would let a fourth browser job in beside three live ones.
+   * Before start, the abort removes the job from the queue, so a caller that
+   * timed out never runs later.
+   */
+  function run(fn, { label = 'browser job' } = {}) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => {
+      // Read before the abort removes this job: the depth the caller waited behind.
+      controller.abort(new Error(
+        `${label} timed out after ${timeoutMs / 1000} s waiting for a browser slot: ` +
+        `${queue.pending} running, ${queue.size} queued ` +
+        `(CRAWLFORGE_MAX_ACTION_SESSIONS=${concurrency}). Retry shortly, or raise ` +
+        'CRAWLFORGE_MAX_ACTION_SESSIONS / CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS if the machine has room.'
+      ));
+    }, timeoutMs);
+    // Not unref()'d: a caller is awaiting this wait, so it must keep the event
+    // loop alive. Unref'd, Node 22 could exit mid-wait with the promise still
+    // pending (it did, under node:test in CI). The timer never outlives the
+    // wait — it is cleared when the job starts.
+
+    return queue.add(() => {
+      clearTimeout(timer);
+      return fn();
+    }, { signal: controller.signal });
+  }
+
+  return {
+    run,
+    concurrency,
+    timeoutMs,
+    get running() { return queue.pending; },
+    get queued() { return queue.size; }
+  };
+}
+
+/**
+ * The process-wide limiter both tools use. Tools take an `actionQueue` option
+ * so a test can hand them a small private one; production never passes it.
+ */
+export const actionQueue = createActionQueue();

package/src/core/browser/consent.js

@@ -0,0 +1,182 @@
+/**
+ * Consent — answer a cookie/consent wall with @duckduckgo/autoconsent before
+ * the chain's first action sees the page (browserOptions.consent).
+ *
+ * autoconsent ships a content script plus a rule set; the content script
+ * detects the CMP and clicks its reject (optOut) or accept (optIn) button, and
+ * reports each step as a message. It has to run in every frame: several CMPs
+ * (Sourcepoint on theguardian.com) render inside a cross-origin iframe.
+ *
+ * Messages come back through a queue in each frame that this module drains on
+ * a short poll, not through page.exposeBinding. A binding can be registered
+ * only once per page (a second navigate would throw), and on camoufox
+ * page.evaluate runs in an isolated world that a main-world binding is not
+ * visible from. Draining a queue works the same on both engines.
+ *
+ * Engines: on Chromium frame.evaluate runs in the page's main world, so the
+ * content script and its globals (window.autoconsentReceiveMessage,
+ * window.__crawlforgeConsent) are visible to page scripts. On camoufox it runs
+ * in an isolated world: DOM clicks work the same, but the few rules that read
+ * a CMP's JavaScript API (`eval` steps, e.g. Cookiebot's, OneTrust's self-test)
+ * cannot see page globals there and evaluate to false, so those rules fall
+ * back to their DOM steps or do not match.
+ *
+ * Nothing here throws and nothing fails a chain: no CMP is `action: 'none'`.
+ * The content script is not stopped at the cap — a popup it is still working
+ * on may be answered after this returns.
+ */
+
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+
+const POLL_MS = 100;
+const AUTO_ACTION = { reject: 'optOut', accept: 'optIn' };
+const QUEUE_KEY = '__crawlforgeConsent';
+const KEPT_MESSAGES = ['cmpDetected', 'popupFound', 'optOutResult', 'optInResult', 'autoconsentDone'];
+
+// Read on first use, not at import: ~860 KB that a chain without consent
+// handling never needs.
+let bundle = null;
+function loadBundle() {
+  if (!bundle) {
+    const require = createRequire(import.meta.url);
+    const rulesPath = require.resolve('@duckduckgo/autoconsent/rules/rules.json');
+    // The package's exports map does not expose dist/, so it is reached from
+    // rules/ — the two ship side by side.
+    const script = readFileSync(join(dirname(rulesPath), '..', 'dist', 'autoconsent.playwright.js'), 'utf8');
+    const rules = JSON.parse(readFileSync(rulesPath, 'utf8')).autoconsent.map((rule) => ({
+      rule,
+      pattern: rule.runContext?.urlPattern ? new RegExp(rule.runContext.urlPattern) : null
+    }));
+    bundle = { script, rules };
+  }
+  return bundle;
+}
+
+// The rules that can apply to a frame (the content script checks again; this
+// only keeps a 700 KB rule set out of every frame). Defaults per autoconsent:
+// a rule runs in the main frame unless `main: false`, in a subframe only with
+// `frame: true`.
+function rulesFor(url, mainFrame) {
+  return loadBundle().rules
+    .filter(({ rule, pattern }) => {
+      const ctx = rule.runContext || {};
+      if (mainFrame ? ctx.main === false : ctx.frame !== true) return false;
+      return !pattern || pattern.test(url);
+    })
+    .map(({ rule }) => rule);
+}
+
+function injectScript(frame, autoAction, timeout, mainFrame) {
+  const config = {
+    enabled: true,
+    autoAction,
+    disabledCmps: [],
+    // Prehiding hides the popup before it is answered and unhides it if
+    // answering fails — a snapshot taken meanwhile would lie either way.
+    enablePrehide: false,
+    enableCosmeticRules: true,
+    // One detection attempt per 500 ms; stop looking at about the cap.
+    detectRetries: Math.max(1, Math.ceil(timeout / 500)),
+    // Run eval snippets inline in whatever world evaluate() gave us instead of
+    // round-tripping them through the message queue (see the header).
+    isMainWorld: true
+  };
+  const initResp = { type: 'initResp', config, rules: { autoconsent: rulesFor(frame.url(), mainFrame) } };
+  return `window.${QUEUE_KEY} = [];
+window.autoconsentSendMessage = (m) => {
+  if (${JSON.stringify(KEPT_MESSAGES)}.includes(m.type)) window.${QUEUE_KEY}.push(m);
+  return Promise.resolve();
+};
+${loadBundle().script}
+;window.autoconsentReceiveMessage(${JSON.stringify(initResp)});
+true;`;
+}
+
+const DRAIN_SCRIPT = `(() => { const q = window.${QUEUE_KEY}; return q ? q.splice(0) : null; })()`;
+
+// This frame's new messages; injects the content script into a document that
+// has none yet (first poll, or the frame navigated since).
+async function drainFrame(frame, autoAction, timeout, mainFrame) {
+  try {
+    const messages = await frame.evaluate(DRAIN_SCRIPT);
+    if (messages) return messages;
+    await frame.evaluate(injectScript(frame, autoAction, timeout, mainFrame));
+  } catch {
+    // Detached or navigating frame: the next poll sees its new document.
+  }
+  return [];
+}
+
+/**
+ * Answer the page's consent wall, if it has one autoconsent knows.
+ *
+ * @param {Page} page - Playwright page, already navigated
+ * @param {'off'|'reject'|'accept'} mode - browserOptions.consent
+ * @param {{timeout?: number}} [options] - cap in ms (default 2000)
+ * @returns {Promise<{cmp: string|null, action: string, ms: number}|null>}
+ *   null for 'off'. `action` is 'optOut' / 'optIn' when the popup was
+ *   answered, 'hide' when a cosmetic rule hid it, 'error' when the rule ran
+ *   and failed, 'timeout' when a popup was found but not answered within the
+ *   cap, and 'none' when no popup was found.
+ */
+export async function handleConsent(page, mode, { timeout = 2000 } = {}) {
+  const autoAction = AUTO_ACTION[mode];
+  if (!autoAction) return null;
+
+  const start = Date.now();
+  const deadline = start + Math.max(0, timeout);
+  let cmp = null;
+  let action = 'none';
+
+  try {
+    const mainFrame = page.mainFrame();
+    // Frame -> true while a CMP it detected is still unanswered. One page can
+    // need two frames: on theguardian.com a top-frame rule hides Sourcepoint's
+    // container at once, and the rule in Sourcepoint's iframe then makes the
+    // actual choice — returning on the first `done` would report the hide.
+    const pending = new Map();
+    let finished = false;
+    while (!finished && Date.now() < deadline) {
+      const frames = page.frames().filter((f) => f === mainFrame || /^https?:/i.test(f.url()));
+      let timer;
+      const batches = await Promise.race([
+        Promise.all(frames.map((f) => drainFrame(f, autoAction, timeout, f === mainFrame))),
+        new Promise((resolve) => { timer = setTimeout(() => resolve([]), deadline - Date.now()); })
+      ]);
+      clearTimeout(timer);
+
+      batches.forEach((messages, i) => {
+        for (const m of messages) {
+          if (m.type === 'cmpDetected') {
+            if (!pending.has(frames[i])) pending.set(frames[i], true);
+            continue;
+          }
+          cmp = m.cmp;
+          if (m.type === 'popupFound') {
+            if (action === 'none') action = 'timeout';
+          } else if (m.type === 'autoconsentDone') {
+            action = m.isCosmetic ? 'hide' : autoAction;
+            pending.set(frames[i], false);
+          } else if (!m.result) {
+            // optOutResult/optInResult false: the rule ran and did not work.
+            action = 'error';
+            pending.set(frames[i], false);
+          }
+        }
+      });
+      // A CMP iframe removed mid-answer was closed by its own choice
+      // (Sourcepoint drops its iframe on one) — its done message goes with it.
+      for (const [frame, open] of pending) {
+        if (open && frame.isDetached()) pending.set(frame, false);
+      }
+      finished = action !== 'none' && action !== 'timeout' && ![...pending.values()].includes(true);
+      if (!finished) await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(POLL_MS, deadline - Date.now()))));
+    }
+  } catch {
+    action = 'error';
+  }
+
+  return { cmp, action, ms: Date.now() - start };
+}

package/src/core/browser/settle.js

@@ -0,0 +1,141 @@
+/**
+ * Settle — wait for a page to finish rendering before a snapshot reads it, and
+ * after a click/press/select that may have changed it.
+ *
+ * `domcontentloaded` alone is not enough: a client-rendered page builds its DOM
+ * after it, and an SPA route change never fires it at all. A snapshot taken at
+ * that point describes an empty shell — the Amazon homepage came back as
+ * `[document]` with 0 nodes and a blank title. So the wait is two steps:
+ *
+ *   1. `load`, capped at LOAD_CAP_MS.
+ *   2. A quiet window: no DOM mutation for DOM_QUIET_MS AND no request in
+ *      flight for NETWORK_QUIET_MS, capped at QUIET_CAP_MS. The network window
+ *      counts from when the wait began, so a request an interaction is about to
+ *      start is not beaten to the check by a page that was idle before it.
+ *
+ * Long-polling, websockets and analytics beacons can keep a request in flight
+ * forever, which is what the cap is for: `settled_by: 'cap'` is a normal
+ * outcome, not an error. Nothing here throws — a page that navigates mid-wait
+ * restarts the DOM window, and one that closes ends the wait.
+ */
+
+export const LOAD_CAP_MS = 5000;
+export const QUIET_CAP_MS = 3000;
+export const DOM_QUIET_MS = 300;
+export const NETWORK_QUIET_MS = 500;
+const POLL_MS = 50;
+
+// Where the in-page probe keeps itself. On the document, not the window, so a
+// navigation (a new document) installs a fresh observer; non-enumerable, so it
+// does not show up in a walk of the document's keys.
+const PROBE_KEY = '__crawlforgeSettleProbe';
+
+// Page -> { inFlight: Set<Request>, idleSince }. WeakMap so a closed page's
+// tracker goes with it.
+const trackers = new WeakMap();
+
+/**
+ * Count the page's in-flight requests. Idempotent: the listeners are attached
+ * once per page, however many times the page is settled.
+ *
+ * A Set rather than a counter because requests already in flight when the
+ * listeners attach finish without ever having been counted — deleting an
+ * unknown request is a no-op, where decrementing would drive a counter negative.
+ */
+export function trackRequests(page) {
+  let tracker = trackers.get(page);
+  if (tracker) return tracker;
+
+  tracker = { inFlight: new Set(), idleSince: Date.now() };
+  const done = (request) => {
+    if (tracker.inFlight.delete(request) && tracker.inFlight.size === 0) {
+      tracker.idleSince = Date.now();
+    }
+  };
+  page.on('request', (request) => tracker.inFlight.add(request));
+  page.on('requestfinished', done);
+  page.on('requestfailed', done);
+  trackers.set(page, tracker);
+  return tracker;
+}
+
+/**
+ * Injected: ms since the document last mutated. The first call on a document
+ * installs the observer and reports 0, so a fresh document is never quiet
+ * until DOM_QUIET_MS have actually been watched.
+ */
+function mutationProbe(key) {
+  let probe = document[key];
+  if (!probe) {
+    let last = performance.now();
+    new MutationObserver(() => { last = performance.now(); })
+      .observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
+    probe = () => performance.now() - last;
+    Object.defineProperty(document, key, { value: probe });
+  }
+  return probe();
+}
+
+/** The probe's answer, or 0 (not quiet) if the page could not answer before the deadline. */
+async function sinceLastMutation(page, deadline) {
+  // A navigation destroys the execution context mid-evaluate; that document is
+  // not quiet, and the next poll probes its successor.
+  const probe = page.evaluate(mutationProbe, PROBE_KEY).catch(() => 0);
+  let timer;
+  const expiry = new Promise((resolve) => {
+    timer = setTimeout(resolve, Math.max(0, deadline - Date.now()), 0);
+  });
+  try {
+    return await Promise.race([probe, expiry]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Wait for `load`, then for a quiet window, never longer than `timeout` in all.
+ *
+ * @param {import('playwright').Page} page
+ * @param {object} [options]
+ * @param {number} [options.timeout] — overall budget in ms; default LOAD_CAP_MS + QUIET_CAP_MS
+ * @returns {Promise<{waited_ms: number, settled_by: 'quiet'|'cap'|'closed'}>}
+ *   `quiet` — both windows held; `cap` — a cap or the budget ran out first;
+ *   `closed` — the page closed during the wait.
+ */
+export async function settlePage(page, { timeout = LOAD_CAP_MS + QUIET_CAP_MS } = {}) {
+  const started = Date.now();
+  let settledBy = 'cap';
+
+  try {
+    // Attached before the load wait so the requests that hold `load` back are counted.
+    const requests = trackRequests(page);
+
+    try {
+      await page.waitForLoadState('load', { timeout: Math.max(1, Math.min(LOAD_CAP_MS, timeout)) });
+    } catch {
+      // Not loaded within the cap — the quiet window still gets its turn.
+    }
+
+    const quietFrom = Date.now();
+    const deadline = Math.min(quietFrom + QUIET_CAP_MS, started + timeout);
+    while (Date.now() < deadline) {
+      if (page.isClosed()) {
+        settledBy = 'closed';
+        break;
+      }
+      const domQuietFor = await sinceLastMutation(page, deadline);
+      const networkQuietFor = requests.inFlight.size > 0
+        ? 0
+        : Date.now() - Math.max(requests.idleSince, quietFrom);
+      if (domQuietFor >= DOM_QUIET_MS && networkQuietFor >= NETWORK_QUIET_MS) {
+        settledBy = 'quiet';
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, Math.min(POLL_MS, Math.max(0, deadline - Date.now()))));
+    }
+  } catch {
+    // A page torn down under us. Settling is never itself a failure.
+  }
+
+  return { waited_ms: Date.now() - started, settled_by: settledBy };
+}