crawlforge-mcp-server 6.12.1 → 6.16.0

package/src/core/browser/snapshot.js

@@ -2,38 +2,47 @@
  * Snapshot — the accessibility-style page tree that hands an agent stable
  * element refs (`@e1`, `@e2`, …) to act on instead of guessed CSS selectors.
  *
- * Why ours and not Playwright's: 1.62's `locator.ariaSnapshot()` emits YAML
- * with no element refs at all, and `page._snapshotForAI()` is private API we
- * will not depend on. The walk below is injected by us and returns the tree
- * and the refs from one pass.
+ * TWO SOURCES, one public format:
  *
- * A REF LIVES IN TWO PLACES, and the halves do different jobs:
+ *   1. `page.ariaSnapshot({ mode: 'ai' })` (Playwright ≥ 1.59; verified on
+ *      1.62.1, Chromium and Camoufox). It descends open shadow roots and
+ *      iframes, and tags each visible node `[ref=e5]` (`[ref=f1e3]` in a
+ *      frame) that the public `aria-ref=` selector engine resolves — shadow
+ *      and frame refs included. Its YAML is translated into our tree below.
+ *   2. The injected walk (snapshotScript), kept as the fallback for a page
+ *      where the native call throws. It stamps `data-cf-ref="e1"` on the
+ *      elements it refs and sees neither shadow roots nor frames.
  *
- *   1. In the page — the walk stamps `data-cf-ref="e1"` onto every element it
- *      refs, so a ref resolves to an ordinary CSS selector,
- *      `[data-cf-ref="e1"]`. That is what makes refs work with every existing
- *      action path for free, including the stealth human-behaviour code that
- *      takes a raw selector string.
- *   2. In Node — a WeakMap<Page, state> cleared on every main-frame
- *      navigation. This is the half that DETECTS staleness. Without it a ref
- *      from a previous page would merely fail to match, and the caller would
- *      be told "selector not found" instead of "take a new snapshot".
+ * The result's `source` ('aria' | 'walk') says which one described the page.
  *
- * So a stale ref fails loudly, with the reason and the fix (StaleRefError),
- * and never silently hits the wrong element.
+ * Refs are OURS, not Playwright's: `@e1…@eN`, contiguous, in document order
+ * across the page, its shadow roots and its frames. Playwright numbers every
+ * visible node (the body is usually its e1), so its ids have gaps and a page's
+ * one button would be e2; each of our refs instead maps to the selector that
+ * finds its element — `aria-ref=f1e3` natively, `[data-cf-ref="e1"]` from the
+ * walk. A resolved selector is therefore NOT always CSS: a consumer must hand
+ * it to page.locator()/waitForSelector(), never to document.querySelector().
+ *
+ * STALENESS lives in Node — a WeakMap<Page, state> cleared on every main-frame
+ * navigation. Without it a ref from a previous page would merely fail to
+ * match, and the caller would be told "selector not found" instead of "take a
+ * new snapshot". So a stale ref fails loudly, with the reason and the fix
+ * (StaleRefError), and never silently hits the wrong element.
  *
  * Not to be confused with src/core/SnapshotManager.js, which is change-
  * tracking history.
  */
 
 import { randomUUID } from 'node:crypto';
+import { settlePage } from './settle.js';
 
 export const REF_ATTRIBUTE = 'data-cf-ref';
 export const DEFAULT_MAX_NODES = 200;
 export const MAX_NODES_LIMIT = 1000;
 
-// Walks of one snapshot call, including the retry when the page navigates
-// mid-walk. Two: one retry is enough for a page that settles, and a page
+// Captures of one snapshot call, including the retry when the page navigates
+// mid-capture (a fallback from the native snapshot to the walk is not one of
+// them). Two: one retry is enough for a page that settles, and a page
 // navigating repeatedly is not one a snapshot can describe.
 const MAX_WALK_ATTEMPTS = 2;
 
@@ -44,6 +53,26 @@ const MAX_NAME_LENGTH = 120;
 
 const REF_PATTERN = /^@e[1-9]\d*$/;
 
+// Shared by both sources: the walk gets them as arguments, the translation of
+// the native snapshot reads them directly.
+const INTERACTIVE_ROLES = [
+  'button', 'link', 'checkbox', 'radio', 'textbox', 'combobox', 'menuitem',
+  'menuitemcheckbox', 'menuitemradio', 'tab', 'switch', 'option', 'searchbox',
+  'slider', 'spinbutton'
+];
+const STRUCTURAL_ROLES = [
+  'heading', 'banner', 'navigation', 'contentinfo', 'complementary', 'main',
+  'form', 'region', 'search', 'iframe'
+];
+const INTERACTIVE_ROLE_SET = new Set(INTERACTIVE_ROLES);
+const STRUCTURAL_ROLE_SET = new Set(STRUCTURAL_ROLES);
+
+// An editing host (`contenteditable`) has no ARIA role, so the native snapshot
+// shows it as a plain `generic`. Only a page that has one pays for the per-node
+// check that finds it, and each check gets this long before it counts as "no".
+const EDITABLE_SELECTOR = '[contenteditable]:not([contenteditable="false"])';
+const EDITABLE_CHECK_TIMEOUT_MS = 1000;
+
 /** Thrown when a ref cannot be resolved against the page's current snapshot. */
 export class StaleRefError extends Error {
   constructor(message) {
@@ -66,17 +95,12 @@ export function isRef(selector) {
  * JavaScript, so it has nothing to do with the ALLOW_JAVASCRIPT_EXECUTION flag
  * that gates the `executeJavaScript` action. Do not put it behind that flag.
  */
-function snapshotScript({ refAttribute, interactiveOnly, maxNodes, maxIndent, maxNameLength }) {
+function snapshotScript({
+  refAttribute, interactiveOnly, maxNodes, maxIndent, maxNameLength, interactiveRoles, structuralRoles
+}) {
   const SKIP_TAGS = new Set(['script', 'style', 'noscript', 'template', 'head']);
-  const INTERACTIVE_ROLES = new Set([
-    'button', 'link', 'checkbox', 'radio', 'textbox', 'combobox', 'menuitem',
-    'menuitemcheckbox', 'menuitemradio', 'tab', 'switch', 'option', 'searchbox',
-    'slider', 'spinbutton'
-  ]);
-  const STRUCTURAL_ROLES = new Set([
-    'heading', 'banner', 'navigation', 'contentinfo', 'complementary', 'main',
-    'form', 'region', 'search'
-  ]);
+  const INTERACTIVE_ROLES = new Set(interactiveRoles);
+  const STRUCTURAL_ROLES = new Set(structuralRoles);
   const LANDMARK_TAGS = {
     main: 'main', nav: 'navigation', header: 'banner', footer: 'contentinfo',
     aside: 'complementary', form: 'form'
@@ -214,6 +238,148 @@ function snapshotScript({ refAttribute, interactiveOnly, maxNodes, maxIndent, ma
   return { title: clean(document.title), lines, refs, truncated };
 }
 
+// Same name rules as the walk's clean()/truncate(): one line, no double quotes.
+const cleanName = (value) => (value || '').replace(/\s+/g, ' ').trim().replace(/"/g, "'");
+const truncateName = (value) =>
+  (value.length > MAX_NAME_LENGTH ? `${value.slice(0, MAX_NAME_LENGTH - 1)}…` : value);
+
+/**
+ * One line of `page.ariaSnapshot({ mode: 'ai' })`:
+ *
+ *   - role "name" [checked] [level=1] [ref=e5] [cursor=pointer]: text
+ *
+ * The name is JSON-quoted, or printed bare when it is itself `/…/`. A key YAML
+ * would misread (a name holding `: ` or ` #`) comes wrapped in single quotes
+ * with `''` for `'`; a text value that needs it is double-quoted. Property
+ * lines (`- /url: …`) and bare text (`- text: …`) are not nodes and return null.
+ */
+function parseAriaLine(line) {
+  const item = /^( *)- (.*)$/.exec(line);
+  if (!item) return null;
+
+  let key = item[2];
+  let rest = '';
+  const quoted = /^'((?:[^']|'')*)'(.*)$/.exec(key);
+  if (quoted) {
+    key = quoted[1].replace(/''/g, "'");
+    rest = quoted[2];
+  } else {
+    // An unquoted key never contains `:` followed by a space or the end.
+    const colon = key.search(/:(\s|$)/);
+    if (colon >= 0) [key, rest] = [key.slice(0, colon), key.slice(colon)];
+  }
+
+  const parts = /^([a-z]+)(?: ("(?:[^"\\]|\\.)*"|\/.*\/(?= \[|$)))?(.*)$/.exec(key);
+  if (!parts || parts[1] === 'text') return null;
+
+  let text = rest.replace(/^:\s?/, '');
+  if (text.startsWith('"')) {
+    try {
+      text = JSON.parse(text.replace(/\\x([0-9a-f]{2})/gi, '\\u00$1'));
+    } catch {
+      // keep the raw value; it is only ever a fallback name
+    }
+  }
+
+  const ref = /\[ref=([^\]\s]+)\]/.exec(parts[3]);
+  return {
+    indent: item[1].length,
+    role: parts[1],
+    name: parts[2] ? (parts[2].startsWith('"') ? JSON.parse(parts[2]) : parts[2]) : '',
+    text,
+    ref: ref ? ref[1] : null,
+    pointer: parts[3].includes('[cursor=pointer]')
+  };
+}
+
+/** Playwright refs of the `generic` nodes that are contenteditable editing hosts. */
+async function editingHosts(page, nodes) {
+  const found = await Promise.all(page.frames().map((frame) =>
+    frame.locator(EDITABLE_SELECTOR).count().catch(() => 0)));
+  if (!found.some(Boolean)) return new Set();
+
+  const generics = nodes.filter((node) => node.role === 'generic');
+  const hosts = await Promise.all(generics.map((node) =>
+    page.locator(`aria-ref=${node.ref}`)
+      .evaluate(
+        (el) => el.isContentEditable && !(el.parentElement && el.parentElement.isContentEditable),
+        undefined,
+        { timeout: EDITABLE_CHECK_TIMEOUT_MS }
+      )
+      .catch(() => false)));
+  return new Set(generics.filter((_, i) => hosts[i]).map((node) => node.ref));
+}
+
+/**
+ * Describe the page with Playwright's native snapshot, in the walk's format.
+ *
+ * A node is interactive by role; or, having no such role, when it is an
+ * editing host (printed as the textbox it is typed into like) or the outermost
+ * node with a pointer cursor — the mark an onclick `<div>` usually carries.
+ * Nodes without a Playwright ref are left out: those are the invisible ones
+ * (display:none, zero-size, a closed <select>'s options). A `tabindex` or
+ * `onclick` element with neither a role nor a pointer cursor is not found —
+ * the YAML does not carry those attributes.
+ */
+async function ariaCapture(page, { interactiveOnly, maxNodes, timeout }) {
+  const yaml = await page.ariaSnapshot({ mode: 'ai', timeout });
+  const nodes = yaml.split('\n').map(parseAriaLine).filter((node) => node && node.ref);
+  const editable = await editingHosts(page, nodes);
+
+  const lines = [];
+  const refs = [];
+  let truncated = false;
+  // The emitted ancestors of the current node. Indentation follows EMITTED
+  // nesting, as in the walk, so it is this stack's depth, not the YAML's.
+  const emitted = [];
+
+  for (const node of nodes) {
+    while (emitted.length && emitted[emitted.length - 1].indent >= node.indent) emitted.pop();
+
+    const isHost = editable.has(node.ref);
+    const interactive = INTERACTIVE_ROLE_SET.has(node.role) || isHost ||
+      (node.pointer && !emitted.some((ancestor) => ancestor.interactive));
+    const structural = !interactive && !interactiveOnly && STRUCTURAL_ROLE_SET.has(node.role);
+    if (!interactive && !structural) continue;
+
+    if (lines.length >= maxNodes) {
+      truncated = true;
+      break;
+    }
+    const role = isHost ? 'textbox' : node.role;
+    // A control with no accessible name is captioned by its text, as the walk
+    // does; a landmark is named by its label only.
+    const name = truncateName(cleanName(node.name || (interactive ? node.text : '')));
+    let ref = '';
+    if (interactive) {
+      const id = `e${refs.length + 1}`;
+      refs.push({ id, role, name, selector: `aria-ref=${node.ref}` });
+      ref = `@${id} `;
+    }
+    lines.push(`${'  '.repeat(Math.min(emitted.length + 1, MAX_INDENT))}${ref}[${role}]${name ? ` "${name}"` : ''}`);
+    emitted.push({ indent: node.indent, interactive });
+  }
+
+  return { title: cleanName(await page.title()), lines, refs, truncated };
+}
+
+/** The injected walk, with each ref's selector pointing at the stamp it left. */
+async function walkCapture(page, { interactiveOnly, maxNodes }) {
+  const walk = await page.evaluate(snapshotScript, {
+    refAttribute: REF_ATTRIBUTE,
+    interactiveOnly,
+    maxNodes,
+    maxIndent: MAX_INDENT,
+    maxNameLength: MAX_NAME_LENGTH,
+    interactiveRoles: INTERACTIVE_ROLES,
+    structuralRoles: STRUCTURAL_ROLES
+  });
+  return {
+    ...walk,
+    refs: walk.refs.map(({ id, role, name }) => ({ id, role, name, selector: `[${REF_ATTRIBUTE}="${id}"]` }))
+  };
+}
+
 function stateFor(page) {
   let state = pageState.get(page);
   if (!state) {
@@ -224,12 +390,15 @@ function stateFor(page) {
 }
 
 /**
- * Walk the page and return its tree plus the refs it assigned.
+ * Describe the page and return its tree plus the refs it assigned. `source`
+ * in the result says whether the native snapshot or the walk produced it.
  *
  * @param {import('playwright').Page} page
  * @param {object} [options]
  * @param {boolean} [options.interactiveOnly=true] — false also emits headings and landmarks, unreffed
  * @param {number} [options.maxNodes=200] — cap on emitted nodes, clamped to [1, MAX_NODES_LIMIT]
+ * @param {number} [options.timeout] — budget for the render wait before the capture (see settle.js),
+ *   and for the native snapshot itself
  */
 export async function captureSnapshot(page, options = {}) {
   const interactiveOnly = options.interactiveOnly !== false;
@@ -242,27 +411,42 @@ export async function captureSnapshot(page, options = {}) {
   // refs rather than leaving them to fail as a missing selector.
   attachRefTracking(page);
 
+  // Walking before the page has rendered describes an empty shell. Both the
+  // snapshot action and browser_session's snapshot operation come through here.
+  const settle = await settlePage(page, { timeout: options.timeout });
+
   const state = stateFor(page);
   let title, lines, refs, truncated;
-
-  // A navigation that commits WHILE the walk is running would otherwise leave us
-  // holding refs for a document that has gone — the attributes were stamped on
-  // the old page, so `@e1` would match nothing and surface as a locator timeout
-  // instead of the named error D2 requires. `generation` moves on every
-  // main-frame navigation, so a change across the evaluate means exactly that.
-  // Walk the new document instead; if it navigates again, give up and leave the
-  // refs invalidated rather than publishing a tree for a page nobody is on.
-  for (let attempt = 0; ; attempt++) {
+  // The native snapshot unless this page object has none (a stub, an old
+  // Playwright) or it throws here; the walk is the fallback either way.
+  let source = typeof page.ariaSnapshot === 'function' ? 'aria' : 'walk';
+
+  // A navigation that commits WHILE the snapshot is running would otherwise
+  // leave us holding refs for a document that has gone, so `@e1` would match
+  // nothing and surface as a locator timeout instead of the named error D2
+  // requires. `generation` moves on every main-frame navigation, so a change
+  // across the capture means exactly that — whether the capture returned or
+  // threw because its context was destroyed. Capture the new document instead;
+  // if it navigates again, give up and leave the refs invalidated rather than
+  // publishing a tree for a page nobody is on.
+  for (let attempt = 0; ;) {
     const generation = state.generation;
-    ({ title, lines, refs, truncated } = await page.evaluate(snapshotScript, {
-      refAttribute: REF_ATTRIBUTE,
-      interactiveOnly,
-      maxNodes,
-      maxIndent: MAX_INDENT,
-      maxNameLength: MAX_NAME_LENGTH
-    }));
-    if (state.generation === generation) break;
-    if (attempt >= MAX_WALK_ATTEMPTS - 1) {
+    const capture = source === 'aria' ? ariaCapture : walkCapture;
+    let result;
+    try {
+      result = await capture(page, { interactiveOnly, maxNodes, timeout: options.timeout });
+    } catch (error) {
+      if (state.generation === generation) {
+        if (source === 'walk') throw error;
+        source = 'walk'; // the native snapshot failed on a page that stayed put
+        continue;
+      }
+    }
+    if (state.generation === generation) {
+      ({ title, lines, refs, truncated } = result);
+      break;
+    }
+    if (++attempt >= MAX_WALK_ATTEMPTS) {
       clearRefs(page);
       throw new StaleRefError(
         'The page navigated while the snapshot was being taken — take a new snapshot.'
@@ -272,7 +456,7 @@ export async function captureSnapshot(page, options = {}) {
 
   const snapshotId = randomUUID().slice(0, 8);
   state.snapshotId = snapshotId;
-  state.refs = new Map(refs.map(({ id, role, name, tag }) => [id, { role, name, tag }]));
+  state.refs = new Map(refs.map(({ id, role, name, selector }) => [id, { role, name, selector }]));
   state.invalidated = false;
 
   return {
@@ -283,12 +467,18 @@ export async function captureSnapshot(page, options = {}) {
     refCount: refs.length,
     nodeCount: lines.length,
     truncated,
-    interactiveOnly
+    interactiveOnly,
+    source,
+    waited_ms: settle.waited_ms,
+    settled_by: settle.settled_by
   };
 }
 
 /**
- * Turn `@e1` into the CSS selector every action path already understands.
+ * Turn `@e1` into the selector for the element it named: `aria-ref=…` from the
+ * native snapshot, `[data-cf-ref="e1"]` from the walk. Either works anywhere a
+ * Playwright selector does (locator, waitForSelector, page.type) — not in a
+ * raw document.querySelector().
  * Throws StaleRefError when the ref does not belong to the page's current
  * snapshot — it never guesses.
  */
@@ -318,7 +508,7 @@ export function resolveRef(page, selector) {
       : `Unknown element ref ${selector}: the current snapshot has ${count} refs (@e1-@e${count}) — take a new snapshot.`);
   }
 
-  return `[${REF_ATTRIBUTE}="${id}"]`;
+  return state.refs.get(id).selector;
 }
 
 /** Register the navigation listener that invalidates this page's refs. Idempotent. */

package/src/core/browser/windowState.js

@@ -0,0 +1,88 @@
+/**
+ * windowState — read a page's framework state straight off `window`, after
+ * its JavaScript ran (extract_embedded_state's escalated path, plan Phase 3.2).
+ *
+ * The served HTML is not always where the state ends up: Nuxt 2 ships
+ * __NUXT__ as an IIFE the HTML parser cannot evaluate, and YouTube's
+ * ytInitialData is assigned by a script. Inside the stealth render those are
+ * plain globals, so they are read as the page itself sees them.
+ *
+ * Engines: on Chromium page.evaluate runs in the page's main world, where the
+ * globals live. On camoufox it runs in an isolated world that cannot see them
+ * (window.__NEXT_DATA__ is undefined there), but Firefox's Xray waiver,
+ * `window.wrappedJSObject`, reaches the page's own window from that world
+ * without evaluating anything in the page's — measured against camoufox
+ * 0.1.19, 2026-09-29. So the read works on both engines and the page never
+ * sees a script of ours run in its world.
+ *
+ * Each value is serialised INSIDE the page with its own try/catch: a cyclic or
+ * otherwise unserialisable global is skipped with a warning instead of
+ * failing the read of the others.
+ */
+
+/** The globals read, in this order. Only the ones present come back. */
+export const WINDOW_STATE_GLOBALS = Object.freeze([
+  '__NEXT_DATA__',
+  '__NUXT__',
+  '__remixContext',
+  'ytInitialData',
+  'ytInitialPlayerResponse',
+  '__INITIAL_STATE__',
+  '__PRELOADED_STATE__',
+  '__APOLLO_STATE__',
+  '__TGT_DATA__',
+  '__PWS_DATA__'
+]);
+
+/** Said wherever these values are reported: they are not what the server sent. */
+export const WINDOW_STATE_NOTE = 'read from window after JavaScript ran, not from the served HTML';
+
+/**
+ * Runs in the page. `{ json }` for a global that serialised, `{ error }` for
+ * one that did not; absent (null/undefined) globals are left out.
+ * @param {string[]} names
+ */
+export function serialiseWindowGlobals(names) {
+  const target = window.wrappedJSObject || window;
+  const out = {};
+  for (const name of names) {
+    try {
+      const value = target[name];
+      if (value === undefined || value === null) continue;
+      const json = JSON.stringify(value);
+      out[name] = typeof json === 'string' ? { json } : { error: 'it has no JSON form' };
+    } catch (error) {
+      out[name] = { error: String((error && error.message) || error) };
+    }
+  }
+  return out;
+}
+
+/**
+ * @param {{ evaluate: Function }} page a Playwright page
+ * @returns {Promise<{ state: Record<string, unknown>, warnings: string[] }>}
+ */
+export async function readWindowState(page) {
+  const warnings = [];
+  let raw;
+  try {
+    raw = await page.evaluate(serialiseWindowGlobals, [...WINDOW_STATE_GLOBALS]);
+  } catch (error) {
+    warnings.push(`window_state could not be read: ${String(error.message).split('\n')[0]}`);
+    return { state: {}, warnings };
+  }
+  const state = {};
+  for (const [name, entry] of Object.entries(raw || {})) {
+    if (entry && typeof entry.json === 'string') {
+      try {
+        state[name] = JSON.parse(entry.json);
+        continue;
+      } catch (error) {
+        warnings.push(`window_state: ${name} skipped - its JSON did not parse (${error.message})`);
+        continue;
+      }
+    }
+    warnings.push(`window_state: ${name} skipped - not serialisable (${entry?.error ?? 'unknown reason'})`);
+  }
+  return { state, warnings };
+}

package/src/core/crawlers/BFSCrawler.js

@@ -8,9 +8,10 @@ import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
-import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
+import { robotsPreflight, redirectGate, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { extractMainContent, isThinMainContent } from '../../tools/scrape/_mainContent.js';
 
@@ -337,7 +338,10 @@ export class BFSCrawler {
         'Accept-Language': 'en-US,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
         'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1'
+        'Upgrade-Insecure-Requests': '1',
+        // Web Bot Auth, as on every other fetching tool. crawl_deep pages went
+        // out with the identity alone: a claim with no proof.
+        ...signRequestHeaders(url)
       };
 
       let headers = { ...defaultHeaders, ...domainRules.customHeaders };
@@ -357,7 +361,12 @@ export class BFSCrawler {
 
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers
+        headers,
+        onRedirect: redirectGate(url, {
+          respectRobots: this.respectRobots,
+          userAgent: this.userAgent,
+          tool: 'crawl_deep'
+        })
       });
 
       // Capture any cookies the server sets during the crawl

package/src/core/processing/BrowserProcessor.js

@@ -9,7 +9,7 @@ import { z } from 'zod';
 import StealthBrowserManager from '../StealthBrowserManager.js';
 import HumanBehaviorSimulator from '../../utils/HumanBehaviorSimulator.js';
 import LocalizationManager from '../LocalizationManager.js';
-import { safeGoto } from '../../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed } from '../../utils/ssrfGuard.js';
 
 const BrowserProcessorSchema = z.object({
   url: z.string().url(),
@@ -156,9 +156,12 @@ export class BrowserProcessor {
    * @param {Object} params - Processing parameters
    * @param {string} params.url - URL to process
    * @param {Object} params.options - Browser options
+   * @param {{ onRedirect?: (to: string) => Promise<void> }} [gate] - what the
+   *   calling tool's `preflightFetch` returned, so a redirect is gated the way
+   *   the URL was. A second argument because the schema strips it from `params`.
    * @returns {Promise<Object>} - Processing result with rendered content
    */
-  async processURL(params) {
+  async processURL(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -178,7 +181,7 @@ export class BrowserProcessor {
 
       try {
         // Navigate and wait for content
-        const navigationResult = await this.navigateAndWait(page, url, processingOptions);
+        const navigationResult = await this.navigateAndWait(page, url, processingOptions, gate?.onRedirect);
         
         // Extract content and metadata
         const contentResult = await this.extractContent(page, processingOptions);
@@ -635,18 +638,23 @@ export class BrowserProcessor {
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
    * @param {Object} options - Navigation options
+   * @param {(to: string) => Promise<void>} [onRedirect] - the calling tool's
+   *   redirect gate (robotsGate.js `redirectGate`)
    * @returns {Promise<Object>} - Navigation result
    */
-  async navigateAndWait(page, url, options) {
+  async navigateAndWait(page, url, options, onRedirect) {
     const startTime = Date.now();
 
     // Navigate to URL. safeGoto adds the SSRF/DNS-rebinding guard on top of the
-    // robots/blocklist preflight the calling tool already ran, and re-checks the
-    // landed URL so a redirect can't carry us into a private/metadata range.
+    // robots/blocklist preflight the calling tool already ran, and re-checks
+    // where the navigation went, so a redirect can't carry us into a
+    // private/metadata range or past the gate the URL itself went through.
     await safeGoto(page, url, {
       waitUntil: 'domcontentloaded',
-      timeout: 30000
+      timeout: 30000,
+      onRedirect
     });
+    const landedUrl = page.url();
 
     // Wait for specific selector if provided
     if (options.waitForSelector) {
@@ -687,6 +695,10 @@ export class BrowserProcessor {
       }
     }
 
+    // The waits above are time in which a client-side redirect can move the
+    // page on; the document about to be read is the one that has to pass.
+    await assertNavigationAllowed(page, landedUrl, null, onRedirect);
+
     return {
       navigationTime: Date.now() - startTime
     };

package/src/core/processing/PDFProcessor.js

@@ -10,6 +10,7 @@ import path from 'path';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 
 const PDFProcessorSchema = z.object({
   // A Buffer is the 'buffer' source: process_document hands over a body it
@@ -85,9 +86,11 @@ export class PDFProcessor {
    * @param {string} params.source - PDF source (URL, file path, or buffer)
    * @param {string} params.sourceType - Type of source ('url', 'file', 'buffer')
    * @param {Object} params.options - Processing options
+   * @param {{ headers: Record<string,string>, onRedirect: Function }} [gate] -
+   *   what `preflightFetch` returned for a URL source
    * @returns {Promise<Object>} - Processing result with text and metadata
    */
-  async processPDF(params) {
+  async processPDF(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -106,7 +109,7 @@ export class PDFProcessor {
       // Get PDF buffer based on source type
       let pdfBuffer;
       try {
-        pdfBuffer = await this.getPDFBuffer(source, sourceType);
+        pdfBuffer = await this.getPDFBuffer(source, sourceType, gate);
       } catch (error) {
         result.error = `Failed to load PDF: ${error.message}`;
         result.processingTime = Date.now() - startTime;
@@ -214,12 +217,13 @@ export class PDFProcessor {
    * Get PDF buffer from various sources
    * @param {string} source - PDF source
    * @param {string} sourceType - Source type
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async getPDFBuffer(source, sourceType) {
+  async getPDFBuffer(source, sourceType, gate) {
     switch (sourceType) {
       case 'url':
-        return await this.downloadPDFFromURL(source);
+        return await this.downloadPDFFromURL(source, gate);
       case 'file':
         return await this.readPDFFromFile(source);
       case 'buffer':
@@ -232,15 +236,20 @@ export class PDFProcessor {
   /**
    * Download PDF from URL
    * @param {string} url - PDF URL
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async downloadPDFFromURL(url) {
+  async downloadPDFFromURL(url, gate) {
     try {
       // `timeout` is not a fetch init option — undici/Node fetch silently
       // ignores unknown properties, so only `signal` actually enforces a
       // deadline here.
+      // The gate's headers are the identity the URL was cleared under plus the
+      // Web Bot Auth signature; this download used to discard them and go out
+      // unsigned. With no gate, sign as ourselves.
       const response = await safeFetch(url, {
-        headers: identityHeaders(),
+        headers: gate?.headers ?? { ...identityHeaders(), ...signRequestHeaders(url) },
+        onRedirect: gate?.onRedirect,
         signal: AbortSignal.timeout(30000)
       });