crawlforge-mcp-server 6.12.0 → 6.14.0

package/src/core/browser/consent.js

@@ -0,0 +1,182 @@
+/**
+ * Consent — answer a cookie/consent wall with @duckduckgo/autoconsent before
+ * the chain's first action sees the page (browserOptions.consent).
+ *
+ * autoconsent ships a content script plus a rule set; the content script
+ * detects the CMP and clicks its reject (optOut) or accept (optIn) button, and
+ * reports each step as a message. It has to run in every frame: several CMPs
+ * (Sourcepoint on theguardian.com) render inside a cross-origin iframe.
+ *
+ * Messages come back through a queue in each frame that this module drains on
+ * a short poll, not through page.exposeBinding. A binding can be registered
+ * only once per page (a second navigate would throw), and on camoufox
+ * page.evaluate runs in an isolated world that a main-world binding is not
+ * visible from. Draining a queue works the same on both engines.
+ *
+ * Engines: on Chromium frame.evaluate runs in the page's main world, so the
+ * content script and its globals (window.autoconsentReceiveMessage,
+ * window.__crawlforgeConsent) are visible to page scripts. On camoufox it runs
+ * in an isolated world: DOM clicks work the same, but the few rules that read
+ * a CMP's JavaScript API (`eval` steps, e.g. Cookiebot's, OneTrust's self-test)
+ * cannot see page globals there and evaluate to false, so those rules fall
+ * back to their DOM steps or do not match.
+ *
+ * Nothing here throws and nothing fails a chain: no CMP is `action: 'none'`.
+ * The content script is not stopped at the cap — a popup it is still working
+ * on may be answered after this returns.
+ */
+
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+
+const POLL_MS = 100;
+const AUTO_ACTION = { reject: 'optOut', accept: 'optIn' };
+const QUEUE_KEY = '__crawlforgeConsent';
+const KEPT_MESSAGES = ['cmpDetected', 'popupFound', 'optOutResult', 'optInResult', 'autoconsentDone'];
+
+// Read on first use, not at import: ~860 KB that a chain without consent
+// handling never needs.
+let bundle = null;
+function loadBundle() {
+  if (!bundle) {
+    const require = createRequire(import.meta.url);
+    const rulesPath = require.resolve('@duckduckgo/autoconsent/rules/rules.json');
+    // The package's exports map does not expose dist/, so it is reached from
+    // rules/ — the two ship side by side.
+    const script = readFileSync(join(dirname(rulesPath), '..', 'dist', 'autoconsent.playwright.js'), 'utf8');
+    const rules = JSON.parse(readFileSync(rulesPath, 'utf8')).autoconsent.map((rule) => ({
+      rule,
+      pattern: rule.runContext?.urlPattern ? new RegExp(rule.runContext.urlPattern) : null
+    }));
+    bundle = { script, rules };
+  }
+  return bundle;
+}
+
+// The rules that can apply to a frame (the content script checks again; this
+// only keeps a 700 KB rule set out of every frame). Defaults per autoconsent:
+// a rule runs in the main frame unless `main: false`, in a subframe only with
+// `frame: true`.
+function rulesFor(url, mainFrame) {
+  return loadBundle().rules
+    .filter(({ rule, pattern }) => {
+      const ctx = rule.runContext || {};
+      if (mainFrame ? ctx.main === false : ctx.frame !== true) return false;
+      return !pattern || pattern.test(url);
+    })
+    .map(({ rule }) => rule);
+}
+
+function injectScript(frame, autoAction, timeout, mainFrame) {
+  const config = {
+    enabled: true,
+    autoAction,
+    disabledCmps: [],
+    // Prehiding hides the popup before it is answered and unhides it if
+    // answering fails — a snapshot taken meanwhile would lie either way.
+    enablePrehide: false,
+    enableCosmeticRules: true,
+    // One detection attempt per 500 ms; stop looking at about the cap.
+    detectRetries: Math.max(1, Math.ceil(timeout / 500)),
+    // Run eval snippets inline in whatever world evaluate() gave us instead of
+    // round-tripping them through the message queue (see the header).
+    isMainWorld: true
+  };
+  const initResp = { type: 'initResp', config, rules: { autoconsent: rulesFor(frame.url(), mainFrame) } };
+  return `window.${QUEUE_KEY} = [];
+window.autoconsentSendMessage = (m) => {
+  if (${JSON.stringify(KEPT_MESSAGES)}.includes(m.type)) window.${QUEUE_KEY}.push(m);
+  return Promise.resolve();
+};
+${loadBundle().script}
+;window.autoconsentReceiveMessage(${JSON.stringify(initResp)});
+true;`;
+}
+
+const DRAIN_SCRIPT = `(() => { const q = window.${QUEUE_KEY}; return q ? q.splice(0) : null; })()`;
+
+// This frame's new messages; injects the content script into a document that
+// has none yet (first poll, or the frame navigated since).
+async function drainFrame(frame, autoAction, timeout, mainFrame) {
+  try {
+    const messages = await frame.evaluate(DRAIN_SCRIPT);
+    if (messages) return messages;
+    await frame.evaluate(injectScript(frame, autoAction, timeout, mainFrame));
+  } catch {
+    // Detached or navigating frame: the next poll sees its new document.
+  }
+  return [];
+}
+
+/**
+ * Answer the page's consent wall, if it has one autoconsent knows.
+ *
+ * @param {Page} page - Playwright page, already navigated
+ * @param {'off'|'reject'|'accept'} mode - browserOptions.consent
+ * @param {{timeout?: number}} [options] - cap in ms (default 2000)
+ * @returns {Promise<{cmp: string|null, action: string, ms: number}|null>}
+ *   null for 'off'. `action` is 'optOut' / 'optIn' when the popup was
+ *   answered, 'hide' when a cosmetic rule hid it, 'error' when the rule ran
+ *   and failed, 'timeout' when a popup was found but not answered within the
+ *   cap, and 'none' when no popup was found.
+ */
+export async function handleConsent(page, mode, { timeout = 2000 } = {}) {
+  const autoAction = AUTO_ACTION[mode];
+  if (!autoAction) return null;
+
+  const start = Date.now();
+  const deadline = start + Math.max(0, timeout);
+  let cmp = null;
+  let action = 'none';
+
+  try {
+    const mainFrame = page.mainFrame();
+    // Frame -> true while a CMP it detected is still unanswered. One page can
+    // need two frames: on theguardian.com a top-frame rule hides Sourcepoint's
+    // container at once, and the rule in Sourcepoint's iframe then makes the
+    // actual choice — returning on the first `done` would report the hide.
+    const pending = new Map();
+    let finished = false;
+    while (!finished && Date.now() < deadline) {
+      const frames = page.frames().filter((f) => f === mainFrame || /^https?:/i.test(f.url()));
+      let timer;
+      const batches = await Promise.race([
+        Promise.all(frames.map((f) => drainFrame(f, autoAction, timeout, f === mainFrame))),
+        new Promise((resolve) => { timer = setTimeout(() => resolve([]), deadline - Date.now()); })
+      ]);
+      clearTimeout(timer);
+
+      batches.forEach((messages, i) => {
+        for (const m of messages) {
+          if (m.type === 'cmpDetected') {
+            if (!pending.has(frames[i])) pending.set(frames[i], true);
+            continue;
+          }
+          cmp = m.cmp;
+          if (m.type === 'popupFound') {
+            if (action === 'none') action = 'timeout';
+          } else if (m.type === 'autoconsentDone') {
+            action = m.isCosmetic ? 'hide' : autoAction;
+            pending.set(frames[i], false);
+          } else if (!m.result) {
+            // optOutResult/optInResult false: the rule ran and did not work.
+            action = 'error';
+            pending.set(frames[i], false);
+          }
+        }
+      });
+      // A CMP iframe removed mid-answer was closed by its own choice
+      // (Sourcepoint drops its iframe on one) — its done message goes with it.
+      for (const [frame, open] of pending) {
+        if (open && frame.isDetached()) pending.set(frame, false);
+      }
+      finished = action !== 'none' && action !== 'timeout' && ![...pending.values()].includes(true);
+      if (!finished) await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(POLL_MS, deadline - Date.now()))));
+    }
+  } catch {
+    action = 'error';
+  }
+
+  return { cmp, action, ms: Date.now() - start };
+}

package/src/core/browser/settle.js

@@ -0,0 +1,141 @@
+/**
+ * Settle — wait for a page to finish rendering before a snapshot reads it, and
+ * after a click/press/select that may have changed it.
+ *
+ * `domcontentloaded` alone is not enough: a client-rendered page builds its DOM
+ * after it, and an SPA route change never fires it at all. A snapshot taken at
+ * that point describes an empty shell — the Amazon homepage came back as
+ * `[document]` with 0 nodes and a blank title. So the wait is two steps:
+ *
+ *   1. `load`, capped at LOAD_CAP_MS.
+ *   2. A quiet window: no DOM mutation for DOM_QUIET_MS AND no request in
+ *      flight for NETWORK_QUIET_MS, capped at QUIET_CAP_MS. The network window
+ *      counts from when the wait began, so a request an interaction is about to
+ *      start is not beaten to the check by a page that was idle before it.
+ *
+ * Long-polling, websockets and analytics beacons can keep a request in flight
+ * forever, which is what the cap is for: `settled_by: 'cap'` is a normal
+ * outcome, not an error. Nothing here throws — a page that navigates mid-wait
+ * restarts the DOM window, and one that closes ends the wait.
+ */
+
+export const LOAD_CAP_MS = 5000;
+export const QUIET_CAP_MS = 3000;
+export const DOM_QUIET_MS = 300;
+export const NETWORK_QUIET_MS = 500;
+const POLL_MS = 50;
+
+// Where the in-page probe keeps itself. On the document, not the window, so a
+// navigation (a new document) installs a fresh observer; non-enumerable, so it
+// does not show up in a walk of the document's keys.
+const PROBE_KEY = '__crawlforgeSettleProbe';
+
+// Page -> { inFlight: Set<Request>, idleSince }. WeakMap so a closed page's
+// tracker goes with it.
+const trackers = new WeakMap();
+
+/**
+ * Count the page's in-flight requests. Idempotent: the listeners are attached
+ * once per page, however many times the page is settled.
+ *
+ * A Set rather than a counter because requests already in flight when the
+ * listeners attach finish without ever having been counted — deleting an
+ * unknown request is a no-op, where decrementing would drive a counter negative.
+ */
+export function trackRequests(page) {
+  let tracker = trackers.get(page);
+  if (tracker) return tracker;
+
+  tracker = { inFlight: new Set(), idleSince: Date.now() };
+  const done = (request) => {
+    if (tracker.inFlight.delete(request) && tracker.inFlight.size === 0) {
+      tracker.idleSince = Date.now();
+    }
+  };
+  page.on('request', (request) => tracker.inFlight.add(request));
+  page.on('requestfinished', done);
+  page.on('requestfailed', done);
+  trackers.set(page, tracker);
+  return tracker;
+}
+
+/**
+ * Injected: ms since the document last mutated. The first call on a document
+ * installs the observer and reports 0, so a fresh document is never quiet
+ * until DOM_QUIET_MS have actually been watched.
+ */
+function mutationProbe(key) {
+  let probe = document[key];
+  if (!probe) {
+    let last = performance.now();
+    new MutationObserver(() => { last = performance.now(); })
+      .observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
+    probe = () => performance.now() - last;
+    Object.defineProperty(document, key, { value: probe });
+  }
+  return probe();
+}
+
+/** The probe's answer, or 0 (not quiet) if the page could not answer before the deadline. */
+async function sinceLastMutation(page, deadline) {
+  // A navigation destroys the execution context mid-evaluate; that document is
+  // not quiet, and the next poll probes its successor.
+  const probe = page.evaluate(mutationProbe, PROBE_KEY).catch(() => 0);
+  let timer;
+  const expiry = new Promise((resolve) => {
+    timer = setTimeout(resolve, Math.max(0, deadline - Date.now()), 0);
+  });
+  try {
+    return await Promise.race([probe, expiry]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Wait for `load`, then for a quiet window, never longer than `timeout` in all.
+ *
+ * @param {import('playwright').Page} page
+ * @param {object} [options]
+ * @param {number} [options.timeout] — overall budget in ms; default LOAD_CAP_MS + QUIET_CAP_MS
+ * @returns {Promise<{waited_ms: number, settled_by: 'quiet'|'cap'|'closed'}>}
+ *   `quiet` — both windows held; `cap` — a cap or the budget ran out first;
+ *   `closed` — the page closed during the wait.
+ */
+export async function settlePage(page, { timeout = LOAD_CAP_MS + QUIET_CAP_MS } = {}) {
+  const started = Date.now();
+  let settledBy = 'cap';
+
+  try {
+    // Attached before the load wait so the requests that hold `load` back are counted.
+    const requests = trackRequests(page);
+
+    try {
+      await page.waitForLoadState('load', { timeout: Math.max(1, Math.min(LOAD_CAP_MS, timeout)) });
+    } catch {
+      // Not loaded within the cap — the quiet window still gets its turn.
+    }
+
+    const quietFrom = Date.now();
+    const deadline = Math.min(quietFrom + QUIET_CAP_MS, started + timeout);
+    while (Date.now() < deadline) {
+      if (page.isClosed()) {
+        settledBy = 'closed';
+        break;
+      }
+      const domQuietFor = await sinceLastMutation(page, deadline);
+      const networkQuietFor = requests.inFlight.size > 0
+        ? 0
+        : Date.now() - Math.max(requests.idleSince, quietFrom);
+      if (domQuietFor >= DOM_QUIET_MS && networkQuietFor >= NETWORK_QUIET_MS) {
+        settledBy = 'quiet';
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, Math.min(POLL_MS, Math.max(0, deadline - Date.now()))));
+    }
+  } catch {
+    // A page torn down under us. Settling is never itself a failure.
+  }
+
+  return { waited_ms: Date.now() - started, settled_by: settledBy };
+}