crawlforge-mcp-server 6.14.0 → 6.17.1

package/src/core/StealthBrowserManager.js

@@ -17,11 +17,12 @@ import path from 'path';
 import { createRequire } from 'module';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
-import { safeGoto } from '../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed, assertUrlAllowed } from '../utils/ssrfGuard.js';
 import { looksLikeInterstitial, detectChallengePage } from '../utils/challengeDetection.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
 import { ClearanceJar, sharedClearanceJar } from './ClearanceJar.js';
+import { readWindowState } from './browser/windowState.js';
 
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
@@ -158,6 +159,39 @@ function isChromium(page) {
   }
 }
 
+/**
+ * A proxy entry with its credentials removed. Everything before the last "@"
+ * of the authority goes, with or without a scheme: parseProxyEntry accepts a
+ * bare `user:pass@host:port`, which a rule anchored on "//" left whole.
+ * @param {string} entry
+ * @returns {string}
+ */
+export function redactProxyEntry(entry) {
+  return String(entry).replace(/^([a-z][a-z0-9+.-]*:\/\/)?[^/]*@/i, '$1');
+}
+
+/**
+ * Refuse a per-call proxy list on camoufox. camoufox takes its proxy at
+ * browser launch and that one browser serves every later camoufox call, so a
+ * caller's proxy would carry other callers' traffic, on the caller's
+ * credentials, or be ignored when a browser is already up. Chromium sets the
+ * proxy per context. Called with the engine that will actually run, so an
+ * "auto" that lands on camoufox is refused too. The operator's
+ * CRAWLFORGE_STEALTH_PROXIES is not a per-call list and is not refused.
+ * @param {string} engine - the resolved engine
+ * @param {{ proxyRotation?: { enabled?: boolean, proxies?: string[] } }} config
+ * @throws {Error}
+ */
+export function assertProxyEngineAllowed(engine, config) {
+  const requested = config?.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
+  if (engine !== 'camoufox' || !requested.length) return;
+  throw new Error(
+    'proxyRotation is refused on the camoufox engine: camoufox applies its proxy at browser launch ' +
+    'and shares that browser across calls, so a per-call proxy cannot be kept to this call. Use ' +
+    'engine:"chromium" (proxies are per context there), or set the operator list CRAWLFORGE_STEALTH_PROXIES.'
+  );
+}
+
 /**
  * Resolve a requested engine to the one that will actually launch.
  *
@@ -421,12 +455,19 @@ export class StealthBrowserManager {
    */
   async launchStealthBrowser(config = {}) {
     const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
+    // Every path to a browser comes through here, createStealthContext's
+    // included, so this is where a caller's proxy list is held to the SSRF
+    // guard before either engine is handed it.
+    await this.assertCallerProxiesAllowed(validatedConfig);
     // 'auto' has to become a concrete engine before the comparison below: an
     // unresolved 'auto' never equals _launchedEngine, so every call would park
     // a perfectly good browser and launch a second one beside it. This is the
     // call that decides the engine — every other path reaches a browser
     // through it — so it is the one that records the outcome.
     await this._resolveConfigEngine(validatedConfig, true);
+    // The tools refuse this up front; this is the backstop for every other
+    // path, decided on the engine 'auto' resolved to, before any launch.
+    assertProxyEngineAllowed(validatedConfig.engine, validatedConfig);
 
     // A Chromium that was OOM-killed or crashed doesn't error on reuse — its
     // protocol calls hang. Detect the corpse and relaunch instead.
@@ -527,7 +568,11 @@ export class StealthBrowserManager {
       // leave camoufox reporting the first proxy's city behind the second
       // proxy's exit IP, which is a worse signal than not rotating at all.
       // A rotation takes effect on the next launch, after cleanup().
-      const proxy = this.resolveProxy(validatedConfig);
+      //
+      // Only the operator's CRAWLFORGE_STEALTH_PROXIES, never a caller's list:
+      // this browser outlives the call, and every later camoufox context
+      // inherits its proxy (assertProxyEngineAllowed refuses the caller's).
+      const proxy = this.resolveProxy({ ...validatedConfig, proxyRotation: undefined });
       // The caller's locale goes to the launcher, not to the context. camoufox
       // sets language, Accept-Language and Intl together below the JS layer,
       // where a Worker reads the same answer as the document; Playwright's
@@ -2386,18 +2431,23 @@ export class StealthBrowserManager {
       // Wait for potential challenge page
       await page.waitForTimeout(2000);
       
-      // Check for CloudFlare challenge indicators
+      // Check for CloudFlare challenge indicators. Cloudflare's current
+      // interstitial reads "Performing security verification" under the title
+      // "Just a moment..." (doordash.com, 2026-10-03); none of the older
+      // phrases below appear on it, so the wait further down ended at once.
       const challengeDetected = await page.evaluate(() => {
         const indicators = [
           'cf-browser-verification',
           'cf-challenge-running',
           'Checking your browser',
+          'Performing security verification',
           'DDoS protection by Cloudflare',
           'Ray ID'
         ];
         
         const pageText = document.body.innerText;
-        return indicators.some(indicator => pageText.includes(indicator));
+        return /^just a moment/i.test(document.title.trim()) ||
+          indicators.some(indicator => pageText.includes(indicator));
       });
       
       if (challengeDetected) {
@@ -2419,16 +2469,25 @@ export class StealthBrowserManager {
           }
         }
         
-        // Wait for challenge to complete (up to 30 seconds)
+        // Wait for the challenge to complete, up to 10 s. Only waits for
+        // Cloudflare's own script to finish; nothing is clicked. A challenge
+        // that passes clears in seconds; one that will not pass (an
+        // interactive check, or this IP's reputation) never clears, and 30 s
+        // there put a walled chain past the REST window (doordash.com stealth
+        // Chromium, 3 of 3 runs walled at 39 s with the 30 s cap, 2026-10-03).
         await page.waitForFunction(() => {
           const indicators = [
             'cf-browser-verification',
             'cf-challenge-running',
-            'Checking your browser'
+            'Checking your browser',
+            'Performing security verification'
           ];
-          const pageText = document.body.innerText;
-          return !indicators.some(indicator => pageText.includes(indicator));
-        }, { timeout: 30000 }).catch(() => {});
+          const pageText = document.body ? document.body.innerText : '';
+          return !/^just a moment/i.test(document.title.trim()) &&
+            !indicators.some(indicator => pageText.includes(indicator));
+          // Options are the third argument; in the second they were the page
+          // function's argument and the wait ran Playwright's 30 s default.
+        }, undefined, { timeout: 10000 }).catch(() => {});
         
         this.performanceMetrics.successfulBypasses++;
         return true;
@@ -2549,9 +2608,34 @@ export class StealthBrowserManager {
     return proxy;
   }
 
+  /**
+   * Refuse a caller's proxy whose host the SSRF guard would refuse as a page:
+   * a proxy on loopback, link-local or a metadata address (RFC 1918 too under
+   * SSRF_STRICT, and ALLOWED_DOMAINS exempt, as everywhere else) would have
+   * the browser send the call's traffic into this server's own network. Only
+   * the per-call list is checked — CRAWLFORGE_STEALTH_PROXIES is the
+   * operator's, who may well run a proxy on localhost. The message names the
+   * host, never the entry, which carries the password.
+   * @param {{ proxyRotation?: { enabled?: boolean, proxies?: string[] } }} config
+   * @returns {Promise<void>}
+   * @throws {Error} code SSRF_BLOCKED
+   */
+  async assertCallerProxiesAllowed(config) {
+    const requested = config.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
+    for (const entry of requested) {
+      // url.host of socks4/socks5 parses the same way; the guard only takes http(s).
+      const { host } = new URL(this.parseProxyEntry(entry).server);
+      try {
+        await assertUrlAllowed(`http://${host}/`, { resolveDns: true });
+      } catch (err) {
+        throw Object.assign(new Error(`proxyRotation: proxy host ${host} refused — ${err.message}`), { code: err.code });
+      }
+    }
+  }
+
   /** A proxy entry with its credentials removed, for logs and get_stats. */
   redactProxy(entry) {
-    return String(entry).replace(/\/\/[^/@]*@/, '//');
+    return redactProxyEntry(entry);
   }
 
   /**
@@ -2844,7 +2928,16 @@ export class StealthBrowserManager {
     return this._clickTurnstile(page);
   }
 
-  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
+  /**
+   * @param {object} options
+   * @param {boolean} [options.readWindowState] also read the framework globals
+   *   off `window` once the page has settled (extract_embedded_state's
+   *   escalation, src/core/browser/windowState.js); returned as `windowState`
+   * @param {(to: string) => Promise<void>} [options.onRedirect] the caller's
+   *   gate for wherever the page is redirected to (robotsGate.js
+   *   `redirectGate`); a refusal throws and nothing of the page is returned
+   */
+  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {}, readWindowState: wantWindowState = false, onRedirect } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
     const { contextId, engineFallbackWarning = null } = await this.createStealthContext({ ...stealthConfig, engine });
@@ -2870,7 +2963,8 @@ export class StealthBrowserManager {
       });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
-      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded', onRedirect });
+      const landedUrl = page.url();
       if (response && status === null) status = response.status();
       if (wait_for > 0) await page.waitForTimeout(wait_for);
       // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
@@ -2888,6 +2982,10 @@ export class StealthBrowserManager {
       // a successful scrape.
       const renderMs = await this._settleRender(page);
       const gracedMs = emptyGraceMs + renderMs;
+      // The waits above are time in which a challenge or a client-side redirect
+      // can move the page on. The document read below is the one that has to
+      // pass, so wherever the page stands now is checked like a redirect hop.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2913,6 +3011,12 @@ export class StealthBrowserManager {
       const shot = screenshot
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
+      let windowState;
+      if (wantWindowState) {
+        const read = await readWindowState(page);
+        windowState = read.state;
+        warnings.push(...read.warnings);
+      }
 
       // Still a wall after everything above: whatever clearance this context
       // carried for the site did not work, so closeContext drops it instead of
@@ -2921,7 +3025,7 @@ export class StealthBrowserManager {
         this._markBlocked(contextId, [url, page.url()]);
       }
 
-      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings };
+      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings, ...(windowState ? { windowState } : {}) };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }

package/src/core/browser/actionQueue.js

@@ -57,7 +57,10 @@ export function createActionQueue({
         'CRAWLFORGE_MAX_ACTION_SESSIONS / CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS if the machine has room.'
       ));
     }, timeoutMs);
-    timer.unref?.();
+    // Not unref()'d: a caller is awaiting this wait, so it must keep the event
+    // loop alive. Unref'd, Node 22 could exit mid-wait with the promise still
+    // pending (it did, under node:test in CI). The timer never outlives the
+    // wait — it is cleared when the job starts.
 
     return queue.add(() => {
       clearTimeout(timer);

package/src/core/browser/windowState.js

@@ -0,0 +1,88 @@
+/**
+ * windowState — read a page's framework state straight off `window`, after
+ * its JavaScript ran (extract_embedded_state's escalated path, plan Phase 3.2).
+ *
+ * The served HTML is not always where the state ends up: Nuxt 2 ships
+ * __NUXT__ as an IIFE the HTML parser cannot evaluate, and YouTube's
+ * ytInitialData is assigned by a script. Inside the stealth render those are
+ * plain globals, so they are read as the page itself sees them.
+ *
+ * Engines: on Chromium page.evaluate runs in the page's main world, where the
+ * globals live. On camoufox it runs in an isolated world that cannot see them
+ * (window.__NEXT_DATA__ is undefined there), but Firefox's Xray waiver,
+ * `window.wrappedJSObject`, reaches the page's own window from that world
+ * without evaluating anything in the page's — measured against camoufox
+ * 0.1.19, 2026-09-29. So the read works on both engines and the page never
+ * sees a script of ours run in its world.
+ *
+ * Each value is serialised INSIDE the page with its own try/catch: a cyclic or
+ * otherwise unserialisable global is skipped with a warning instead of
+ * failing the read of the others.
+ */
+
+/** The globals read, in this order. Only the ones present come back. */
+export const WINDOW_STATE_GLOBALS = Object.freeze([
+  '__NEXT_DATA__',
+  '__NUXT__',
+  '__remixContext',
+  'ytInitialData',
+  'ytInitialPlayerResponse',
+  '__INITIAL_STATE__',
+  '__PRELOADED_STATE__',
+  '__APOLLO_STATE__',
+  '__TGT_DATA__',
+  '__PWS_DATA__'
+]);
+
+/** Said wherever these values are reported: they are not what the server sent. */
+export const WINDOW_STATE_NOTE = 'read from window after JavaScript ran, not from the served HTML';
+
+/**
+ * Runs in the page. `{ json }` for a global that serialised, `{ error }` for
+ * one that did not; absent (null/undefined) globals are left out.
+ * @param {string[]} names
+ */
+export function serialiseWindowGlobals(names) {
+  const target = window.wrappedJSObject || window;
+  const out = {};
+  for (const name of names) {
+    try {
+      const value = target[name];
+      if (value === undefined || value === null) continue;
+      const json = JSON.stringify(value);
+      out[name] = typeof json === 'string' ? { json } : { error: 'it has no JSON form' };
+    } catch (error) {
+      out[name] = { error: String((error && error.message) || error) };
+    }
+  }
+  return out;
+}
+
+/**
+ * @param {{ evaluate: Function }} page a Playwright page
+ * @returns {Promise<{ state: Record<string, unknown>, warnings: string[] }>}
+ */
+export async function readWindowState(page) {
+  const warnings = [];
+  let raw;
+  try {
+    raw = await page.evaluate(serialiseWindowGlobals, [...WINDOW_STATE_GLOBALS]);
+  } catch (error) {
+    warnings.push(`window_state could not be read: ${String(error.message).split('\n')[0]}`);
+    return { state: {}, warnings };
+  }
+  const state = {};
+  for (const [name, entry] of Object.entries(raw || {})) {
+    if (entry && typeof entry.json === 'string') {
+      try {
+        state[name] = JSON.parse(entry.json);
+        continue;
+      } catch (error) {
+        warnings.push(`window_state: ${name} skipped - its JSON did not parse (${error.message})`);
+        continue;
+      }
+    }
+    warnings.push(`window_state: ${name} skipped - not serialisable (${entry?.error ?? 'unknown reason'})`);
+  }
+  return { state, warnings };
+}

package/src/core/crawlers/BFSCrawler.js

@@ -8,9 +8,10 @@ import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
-import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
+import { robotsPreflight, redirectGate, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { extractMainContent, isThinMainContent } from '../../tools/scrape/_mainContent.js';
 
@@ -337,7 +338,10 @@ export class BFSCrawler {
         'Accept-Language': 'en-US,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
         'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1'
+        'Upgrade-Insecure-Requests': '1',
+        // Web Bot Auth, as on every other fetching tool. crawl_deep pages went
+        // out with the identity alone: a claim with no proof.
+        ...signRequestHeaders(url)
       };
 
       let headers = { ...defaultHeaders, ...domainRules.customHeaders };
@@ -357,7 +361,12 @@ export class BFSCrawler {
 
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers
+        headers,
+        onRedirect: redirectGate(url, {
+          respectRobots: this.respectRobots,
+          userAgent: this.userAgent,
+          tool: 'crawl_deep'
+        })
       });
 
       // Capture any cookies the server sets during the crawl

package/src/core/processing/BrowserProcessor.js

@@ -9,7 +9,7 @@ import { z } from 'zod';
 import StealthBrowserManager from '../StealthBrowserManager.js';
 import HumanBehaviorSimulator from '../../utils/HumanBehaviorSimulator.js';
 import LocalizationManager from '../LocalizationManager.js';
-import { safeGoto } from '../../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed } from '../../utils/ssrfGuard.js';
 
 const BrowserProcessorSchema = z.object({
   url: z.string().url(),
@@ -156,9 +156,12 @@ export class BrowserProcessor {
    * @param {Object} params - Processing parameters
    * @param {string} params.url - URL to process
    * @param {Object} params.options - Browser options
+   * @param {{ onRedirect?: (to: string) => Promise<void> }} [gate] - what the
+   *   calling tool's `preflightFetch` returned, so a redirect is gated the way
+   *   the URL was. A second argument because the schema strips it from `params`.
    * @returns {Promise<Object>} - Processing result with rendered content
    */
-  async processURL(params) {
+  async processURL(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -178,7 +181,7 @@ export class BrowserProcessor {
 
       try {
         // Navigate and wait for content
-        const navigationResult = await this.navigateAndWait(page, url, processingOptions);
+        const navigationResult = await this.navigateAndWait(page, url, processingOptions, gate?.onRedirect);
         
         // Extract content and metadata
         const contentResult = await this.extractContent(page, processingOptions);
@@ -370,6 +373,11 @@ export class BrowserProcessor {
     // createStealthContext re-validates the config it is given — so a context
     // asked for without it would relaunch on the default engine underneath.
     const engine = options.stealthMode.engine || 'chromium';
+    // The caller's own proxies (scrape_with_actions browserOptions.proxyRotation),
+    // to both calls for the same reason as the engine: camoufox takes its proxy
+    // at launch, Chromium per context. Absent unless the caller sent one, so
+    // CRAWLFORGE_STEALTH_PROXIES still applies when nobody did.
+    const proxy = options.proxyRotation ? { proxyRotation: options.proxyRotation } : {};
 
     // Launch stealth browser
     await this.stealthManager.launchStealthBrowser({
@@ -378,12 +386,14 @@ export class BrowserProcessor {
       randomizeFingerprint: options.stealthMode.randomizeFingerprint,
       hideWebDriver: options.stealthMode.hideWebDriver,
       blockWebRTC: options.stealthMode.blockWebRTC,
-      customUserAgent: options.stealthMode.customUserAgent || options.userAgent
+      customUserAgent: options.stealthMode.customUserAgent || options.userAgent,
+      ...proxy
     });
 
     // Create stealth context
     const { context, contextId } = await this.stealthManager.createStealthContext({
       engine,
+      ...proxy,
       level: options.stealthMode.level,
       customViewport: {
         width: options.viewportWidth || 1280,
@@ -635,18 +645,23 @@ export class BrowserProcessor {
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
    * @param {Object} options - Navigation options
+   * @param {(to: string) => Promise<void>} [onRedirect] - the calling tool's
+   *   redirect gate (robotsGate.js `redirectGate`)
    * @returns {Promise<Object>} - Navigation result
    */
-  async navigateAndWait(page, url, options) {
+  async navigateAndWait(page, url, options, onRedirect) {
     const startTime = Date.now();
 
     // Navigate to URL. safeGoto adds the SSRF/DNS-rebinding guard on top of the
-    // robots/blocklist preflight the calling tool already ran, and re-checks the
-    // landed URL so a redirect can't carry us into a private/metadata range.
+    // robots/blocklist preflight the calling tool already ran, and re-checks
+    // where the navigation went, so a redirect can't carry us into a
+    // private/metadata range or past the gate the URL itself went through.
     await safeGoto(page, url, {
       waitUntil: 'domcontentloaded',
-      timeout: 30000
+      timeout: 30000,
+      onRedirect
     });
+    const landedUrl = page.url();
 
     // Wait for specific selector if provided
     if (options.waitForSelector) {
@@ -687,6 +702,10 @@ export class BrowserProcessor {
       }
     }
 
+    // The waits above are time in which a client-side redirect can move the
+    // page on; the document about to be read is the one that has to pass.
+    await assertNavigationAllowed(page, landedUrl, null, onRedirect);
+
     return {
       navigationTime: Date.now() - startTime
     };

package/src/core/processing/PDFProcessor.js

@@ -10,6 +10,7 @@ import path from 'path';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 
 const PDFProcessorSchema = z.object({
   // A Buffer is the 'buffer' source: process_document hands over a body it
@@ -85,9 +86,11 @@ export class PDFProcessor {
    * @param {string} params.source - PDF source (URL, file path, or buffer)
    * @param {string} params.sourceType - Type of source ('url', 'file', 'buffer')
    * @param {Object} params.options - Processing options
+   * @param {{ headers: Record<string,string>, onRedirect: Function }} [gate] -
+   *   what `preflightFetch` returned for a URL source
    * @returns {Promise<Object>} - Processing result with text and metadata
    */
-  async processPDF(params) {
+  async processPDF(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -106,7 +109,7 @@ export class PDFProcessor {
       // Get PDF buffer based on source type
       let pdfBuffer;
       try {
-        pdfBuffer = await this.getPDFBuffer(source, sourceType);
+        pdfBuffer = await this.getPDFBuffer(source, sourceType, gate);
       } catch (error) {
         result.error = `Failed to load PDF: ${error.message}`;
         result.processingTime = Date.now() - startTime;
@@ -214,12 +217,13 @@ export class PDFProcessor {
    * Get PDF buffer from various sources
    * @param {string} source - PDF source
    * @param {string} sourceType - Source type
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async getPDFBuffer(source, sourceType) {
+  async getPDFBuffer(source, sourceType, gate) {
     switch (sourceType) {
       case 'url':
-        return await this.downloadPDFFromURL(source);
+        return await this.downloadPDFFromURL(source, gate);
       case 'file':
         return await this.readPDFFromFile(source);
       case 'buffer':
@@ -232,15 +236,20 @@ export class PDFProcessor {
   /**
    * Download PDF from URL
    * @param {string} url - PDF URL
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async downloadPDFFromURL(url) {
+  async downloadPDFFromURL(url, gate) {
     try {
       // `timeout` is not a fetch init option — undici/Node fetch silently
       // ignores unknown properties, so only `signal` actually enforces a
       // deadline here.
+      // The gate's headers are the identity the URL was cleared under plus the
+      // Web Bot Auth signature; this download used to discard them and go out
+      // unsigned. With no gate, sign as ourselves.
       const response = await safeFetch(url, {
-        headers: identityHeaders(),
+        headers: gate?.headers ?? { ...identityHeaders(), ...signRequestHeaders(url) },
+        onRedirect: gate?.onRedirect,
         signal: AbortSignal.timeout(30000)
       });
 

package/src/server/fallbackHints.js

@@ -44,8 +44,8 @@ export const FALLBACK_HINTS = Object.freeze({
 });
 
 /**
- * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
- * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * The second-stage hint (Phase 3, 3.4). A `scrape` or `extract_embedded_state`
+ * that came back `escalated: true` has ALREADY run the stealth browser on this URL, so the
  * ordinary hint would send the model back to a tool that has just failed.
  * What is left is a region-specific block, which `localization` can change,
  * or a block that is final: an Akamai-style TLS-level wall is beaten with
@@ -56,9 +56,12 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The hint for this result: the tool's, unless a scrape already escalated. */
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+
+/** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {
-  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  if (ESCALATING_TOOLS.has(toolName) && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
   return FALLBACK_HINTS[toolName];
 }