crawlforge-mcp-server 6.12.1 → 6.16.0

package/src/core/ActionExecutor.js

@@ -7,9 +7,11 @@ import { z } from 'zod';
 import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
-import { assertUrlAllowed } from '../utils/ssrfGuard.js';
-import { browserPreflight } from '../utils/robotsGate.js';
+import { assertUrlAllowed, assertNavigationAllowed } from '../utils/ssrfGuard.js';
+import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
+import { settlePage } from './browser/settle.js';
+import { handleConsent } from './browser/consent.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -303,17 +305,23 @@ export class ActionExecutor extends EventEmitter {
         page = await this.initializePage(url, browserOptions);
         executionContext.page = page;
 
+        // Cookie/consent wall on the landing page (browserOptions.consent,
+        // off unless asked for). A `navigate` action does the same for its
+        // page — see executeNavigateAction.
+        if (browserOptions.consent && browserOptions.consent !== 'off') {
+          executionContext.consent = await handleConsent(page, browserOptions.consent);
+        }
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
 
         // Capture the LIVE post-action page state before the page is closed,
         // so callers can extract final content reflecting all actions
         // (instead of re-fetching the original URL).
-        try {
-          executionContext.finalHtml = await page.content();
-          executionContext.finalUrl = page.url();
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture final page content: ' + captureErr.message);
+        const finalState = await this.captureState(page, browserOptions, 'final page content');
+        if (finalState) {
+          executionContext.finalHtml = finalState.html;
+          executionContext.finalUrl = finalState.url;
         }
 
         this.stats.successfulChains++;
@@ -327,6 +335,9 @@ export class ActionExecutor extends EventEmitter {
         // Capture error screenshot if enabled
         if ((validatedChain.screenshotOnError ?? this.enableScreenshotOnError) && page) {
           try {
+            // Not of a page the gate has not passed: the chain may have failed
+            // for another reason after the page moved.
+            await this.assertPageAllowed(page, browserOptions);
             const errorScreenshot = await this.captureScreenshot(page, {
               fullPage: true,
               description: 'Error screenshot'
@@ -408,6 +419,7 @@ export class ActionExecutor extends EventEmitter {
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
+        consent: executionContext.consent,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
         attempt: executionContext.attempt,
@@ -463,7 +475,9 @@ export class ActionExecutor extends EventEmitter {
           // (form half-filled, menu open, possibly a different URL) is not a
           // retry. Reload the starting URL so every attempt begins where the
           // first one did.
-          await this.navigateToUrl(page, executionContext.url);
+          await this.navigateToUrl(page, executionContext.url, {
+            browserOptions: executionContext.browserOptions
+          });
         }
 
         // Execute actions in sequence
@@ -476,7 +490,7 @@ export class ActionExecutor extends EventEmitter {
             ...chain.actions[i],
             timeout: Math.min(chain.actions[i].timeout || chain.timeout, chain.timeout)
           };
-          const actionResult = await this.executeActionInternal(page, action, executionContext);
+          const actionResult = await this.executeGatedAction(page, action, executionContext);
           
           executionContext.results.push(actionResult);
           this.stats.totalActions++;
@@ -500,7 +514,7 @@ export class ActionExecutor extends EventEmitter {
             
             // Handle action failure
             if (!action.continueOnError && !chain.continueOnError) {
-              throw new Error('Action failed: ' + actionResult.error);
+              throw Object.assign(new Error('Action failed: ' + actionResult.error), { code: actionResult.errorCode });
             }
           }
 
@@ -509,18 +523,16 @@ export class ActionExecutor extends EventEmitter {
           // the ALLOW_JAVASCRIPT_EXECUTION flag and doesn't add phantom
           // actions to the chain's failure/success counts.
           if (action.captureAfter) {
-            try {
-              const capturedHtml = await page.content();
+            const captured = await this.captureState(page, executionContext.browserOptions, 'intermediate state');
+            if (captured) {
               executionContext.capturedStates = executionContext.capturedStates || [];
               executionContext.capturedStates.push({
                 afterActionIndex: i,
                 afterActionId: actionResult.id,
-                url: page.url(),
-                html: capturedHtml,
+                url: captured.url,
+                html: captured.html,
                 timestamp: Date.now()
               });
-            } catch (captureErr) {
-              this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
             }
           }
 
@@ -541,7 +553,12 @@ export class ActionExecutor extends EventEmitter {
           attempt: attempt + 1, success: false, error: error.message, results: [...executionContext.results]
         });
         this.log('warn', 'Chain execution attempt ' + (attempt + 1) + ' failed: ' + error.message);
-        
+
+        // A refusal is the gate's answer, not a fault. Replaying the chain
+        // asks the same question again, after sending the site every request
+        // that led up to it a second time.
+        if (gateRefusalCode(error)) break;
+
         if (attempt < chain.retryChain) {
           // Wait before retry
           await this.delay(1000 * Math.pow(2, attempt));
@@ -567,8 +584,12 @@ export class ActionExecutor extends EventEmitter {
    *
    * Gating comes for free and must not be duplicated by callers: a `navigate`
    * action goes through executeNavigateAction, which re-runs the SSRF guard and
-   * the blocklist/robots gate on every hop (see the comment at its definition).
-   * `browserOptions.respectRobots` is what reaches that gate.
+   * the blocklist/robots gate on every hop (see the comment at its definition),
+   * and every action is run through executeGatedAction, which holds the page to
+   * the same gate wherever a click or a script has taken it. A refusal there
+   * throws rather than failing one action: the page has been emptied, and
+   * nothing after it could run. `browserOptions.respectRobots` is what reaches
+   * both gates.
    *
    * No `finalHtml` here, unlike the chain: the page is still open afterwards, so
    * content is read from it when it is asked for rather than on every call.
@@ -604,7 +625,7 @@ export class ActionExecutor extends EventEmitter {
       // per-call timeout is applied as the default for actions that set none.
       const action = timeout && !actions[i].timeout ? { ...actions[i], timeout } : actions[i];
 
-      const actionResult = await this.executeActionInternal(page, action, executionContext);
+      const actionResult = await this.executeGatedAction(page, action, executionContext);
       executionContext.results.push(actionResult);
       this.stats.totalActions++;
 
@@ -628,17 +649,15 @@ export class ActionExecutor extends EventEmitter {
       }
 
       if (action.captureAfter) {
-        try {
-          const capturedHtml = await page.content();
+        const captured = await this.captureState(page, browserOptions, 'intermediate state');
+        if (captured) {
           executionContext.capturedStates.push({
             afterActionIndex: i,
             afterActionId: actionResult.id,
-            url: page.url(),
-            html: capturedHtml,
+            url: captured.url,
+            html: captured.html,
             timestamp: Date.now()
           });
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
         }
       }
 
@@ -669,6 +688,88 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Hold the page to the gate wherever it stands now.
+   *
+   * navigateToUrl gates a load and its redirects. After that a page moves
+   * without being sent anywhere: a click follows a link, a form posts, a
+   * script redirects, a client-side router rewrites the address. Any URL the
+   * page has reached since it was last checked gets what a `navigate` to it
+   * would: the SSRF guard, then the blocklist/robots gate with this call's
+   * `respectRobots`. A refusal leaves the page empty and throws.
+   *
+   * Cheap when nothing moved (one string comparison), so it is called on both
+   * sides of everything that touches the page.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @returns {Promise<void>}
+   * @throws {Error} SSRF_BLOCKED, BlockedHostError or RobotsDisallowedError
+   */
+  async assertPageAllowed(page, browserOptions = {}) {
+    const gatedUrl = page.__crawlforgeGatedUrl;
+    if (page.url() === gatedUrl) return;
+    page.__crawlforgeGatedUrl = await assertNavigationAllowed(page, gatedUrl, null, pageMoveGate(gatedUrl, {
+      respectRobots: browserOptions?.respectRobots,
+      tool: browserOptions?.tool || 'scrape_with_actions'
+    }));
+  }
+
+  /**
+   * Read from the page with the gate held on both sides of the read, so what
+   * comes back is from a document that passed it: the check after the read
+   * discards anything read while the page was moving somewhere it may not be.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {() => Promise<T>} read - the read to make
+   * @returns {Promise<T>}
+   * @template T
+   */
+  async readGated(page, browserOptions, read) {
+    await this.assertPageAllowed(page, browserOptions);
+    const value = await read();
+    await this.assertPageAllowed(page, browserOptions);
+    return value;
+  }
+
+  /**
+   * The page's URL and HTML, read natively (no in-page JS execution, so it
+   * works regardless of ALLOW_JAVASCRIPT_EXECUTION). A page that cannot be
+   * read yields null and a warning; a page the gate refuses throws.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {string} what - names the capture in the warning
+   * @returns {Promise<{ url: string, html: string }|null>}
+   */
+  async captureState(page, browserOptions, what) {
+    return await this.readGated(page, browserOptions, async () => {
+      try {
+        return { html: await page.content(), url: page.url() };
+      } catch (captureErr) {
+        this.log('warn', 'Failed to capture ' + what + ': ' + captureErr.message);
+        return null;
+      }
+    });
+  }
+
+  /**
+   * Run one action with the page held to the gate before it and after it.
+   * Before, because an action must not read or drive a page that moved while
+   * nothing was looking; after, because the action is what usually moves it,
+   * and its result is not kept if where it led is refused.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Action to execute
+   * @param {Object} executionContext - Execution context (`browserOptions` feeds the gate)
+   * @returns {Promise<Object>} Action result
+   */
+  async executeGatedAction(page, action, executionContext) {
+    const browserOptions = executionContext?.browserOptions;
+    // A navigate leaves the page it is on and gates the URL it is sent to.
+    if (action.type !== 'navigate') await this.assertPageAllowed(page, browserOptions);
+    const actionResult = await this.executeActionInternal(page, action, executionContext);
+    await this.assertPageAllowed(page, browserOptions);
+    return actionResult;
+  }
+
   /**
    * Execute individual action (original internal method)
    * @param {Page} page - Playwright page
@@ -743,11 +844,15 @@ export class ActionExecutor extends EventEmitter {
       return actionResult;
 
     } catch (error) {
+      const refusalCode = gateRefusalCode(error);
       const actionResult = {
         id: actionId,
         type: action.type,
         success: false,
         error: error.message,
+        // Only when the gate refused (a `navigate` to a URL it would not
+        // load), so the chain can tell an answer from a fault.
+        ...(refusalCode ? { errorCode: refusalCode } : {}),
         executionTime: Date.now() - actionStartTime,
         timestamp: Date.now(),
         description: action.description
@@ -799,14 +904,15 @@ export class ActionExecutor extends EventEmitter {
    * Resolve a caller-supplied selector.
    *
    * A selector starting with `@` names a ref a prior snapshot action assigned,
-   * and resolves to the attribute selector that snapshot stamped on the
-   * element; anything else is already a CSS selector. Every selector a caller
+   * and resolves to the Playwright selector that snapshot recorded for it
+   * (`aria-ref=…` from the native snapshot, `[data-cf-ref=…]` from the walk
+   * fallback); anything else is already a CSS selector. Every selector a caller
    * writes goes through here, so refs work in every action type without any
-   * schema change — and, because a ref resolves to an ordinary CSS selector,
-   * in the stealth human-behaviour paths too.
+   * schema change — including the stealth human-behaviour paths, which take
+   * Playwright selectors.
    * @param {Page} page - Playwright page
    * @param {string} selector - CSS selector, or an `@e1` snapshot ref
-   * @returns {string} CSS selector
+   * @returns {string} Playwright selector
    */
   resolveSelector(page, selector) {
     return isRef(selector) ? resolveRef(page, selector) : selector;
@@ -862,23 +968,25 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Give a navigation started by the action that just ran a chance to commit.
+   * Let the page settle after the action that just ran — a navigation commit
+   * and load, or an SPA re-render that fires no load event at all.
    *
    * Playwright auto-waits on the element it acts on, but nothing waits on the
-   * *document* a click or keypress may have replaced, so the next action could
-   * run against the outgoing page. A page that never navigated is already past
-   * this state and returns immediately; a page that doesn't settle in time is
-   * not itself an action failure, hence the catch.
+   * *document* a click or keypress may have changed, so the next action could
+   * run against the outgoing page. settlePage waits for `load` and then a quiet
+   * window (no DOM mutations, no requests in flight), capped and never longer
+   * than what is left of the action's `timeout`; a page that doesn't settle in
+   * time is not itself an action failure, so it never throws.
    * @param {Page} page - Playwright page
-   * @param {number} timeout - Timeout in ms
-   * @returns {Promise<void>}
+   * @param {number} timeout - The action's timeout in ms
+   * @param {number} startedAt - When the action started (Date.now()); the settle
+   *   gets only the remainder, so a slow element wait plus a busy page cannot
+   *   outrun the backstop and turn a successful action into a timeout
+   * @returns {Promise<{waited_ms: number, settled_by: string}>}
    */
-  async settleAfterInteraction(page, timeout) {
-    try {
-      await page.waitForLoadState('domcontentloaded', { timeout });
-    } catch {
-      // Ignored on purpose — see above.
-    }
+  async settleAfterInteraction(page, timeout, startedAt) {
+    const remaining = Math.max(1, timeout - (Date.now() - startedAt));
+    return await settlePage(page, { timeout: remaining });
   }
 
   /**
@@ -960,6 +1068,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Click result
    */
   async executeClickAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const locator = this.elementLocator(page, action.selector);
 
@@ -977,7 +1086,8 @@ export class ActionExecutor extends EventEmitter {
         button: action.button,
         clickCount: action.clickCount,
         delay: action.delay,
-        force: action.force
+        force: action.force,
+        timeout
       });
     } else {
       // Standard click behavior
@@ -998,7 +1108,7 @@ export class ActionExecutor extends EventEmitter {
 
     // A click can follow a link or submit a form; let that navigation commit
     // before the next action runs against the outgoing document.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1050,6 +1160,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Press result
    */
   async executePressAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const keyOptions = { timeout };
     if (action.modifiers?.length > 0) {
@@ -1063,7 +1174,7 @@ export class ActionExecutor extends EventEmitter {
     }
 
     // Enter on a form field navigates as often as a click does.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       key: action.key,
@@ -1161,6 +1272,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Select result
    */
   async executeSelectAction(page, action) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
     const values = action.values?.length ? action.values : [action.value];
 
@@ -1169,7 +1281,7 @@ export class ActionExecutor extends EventEmitter {
 
     // Faceted-search dropdowns commonly submit the form on change, so let any
     // navigation commit before the next action runs.
-    await this.settleAfterInteraction(page, timeout);
+    await this.settleAfterInteraction(page, timeout, startedAt);
 
     return {
       selector: action.selector,
@@ -1214,6 +1326,7 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Navigate result
    */
   async executeNavigateAction(page, action, executionContext) {
+    const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
@@ -1221,14 +1334,23 @@ export class ActionExecutor extends EventEmitter {
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
-      timeout
+      timeout,
+      browserOptions: executionContext?.browserOptions
     });
     page.__crawlforgeGateWarnings = gateWarnings;
 
+    // Consent handling gets at most what is left of the action's deadline, so
+    // it cannot push a navigation that succeeded past the backstop.
+    const consentMode = executionContext?.browserOptions?.consent;
+    const consent = consentMode && consentMode !== 'off'
+      ? await handleConsent(page, consentMode, { timeout: Math.min(2000, timeout - (Date.now() - startedAt)) })
+      : undefined;
+
     return {
       url: action.url,
       finalUrl: page.url(),
-      waitUntil: action.waitUntil || 'domcontentloaded'
+      waitUntil: action.waitUntil || 'domcontentloaded',
+      ...(consent ? { consent } : {})
     };
   }
 
@@ -1314,8 +1436,8 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Execute snapshot action - the page's interactive elements, each stamped
-   * with a ref later actions can target instead of a guessed CSS selector.
+   * Execute snapshot action - the page's interactive elements, each carrying
+   * a ref later actions can target instead of a guessed CSS selector.
    * @param {Page} page - Playwright page
    * @param {Object} action - Snapshot action
    * @returns {Promise<Object>} Snapshot tree with refs
@@ -1323,7 +1445,9 @@ export class ActionExecutor extends EventEmitter {
   async executeSnapshotAction(page, action) {
     return await captureSnapshot(page, {
       interactiveOnly: action.interactiveOnly,
-      maxNodes: action.maxNodes
+      maxNodes: action.maxNodes,
+      // The render wait before the walk stays inside the action's deadline.
+      timeout: this.actionTimeout(action)
     });
   }
 
@@ -1364,12 +1488,14 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Navigate an existing page to a URL under the SSRF checks every load needs.
-   * Used for the initial load, again before each chain retry, and by the
-   * `navigate` action.
+   * Navigate an existing page to a URL under the SSRF checks every load needs,
+   * and put wherever it is redirected to through the same blocklist/robots
+   * gate the URL itself passed (assertRobotsAllowed). Used for the initial
+   * load, again before each chain retry, and by the `navigate` action.
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
-   * @param {{ waitUntil?: string, timeout?: number }} [options] - Navigation options
+   * @param {{ waitUntil?: string, timeout?: number, browserOptions?: Object }} [options] -
+   *   Navigation options; `browserOptions` carries `respectRobots` to the redirect gate
    * @returns {Promise<void>}
    */
   async navigateToUrl(page, url, options = {}) {
@@ -1382,12 +1508,16 @@ export class ActionExecutor extends EventEmitter {
       timeout: options.timeout || 30000
     });
 
-    // Re-validate the landed URL: a redirect during navigation could have
-    // taken us into a blocked range even though the original URL was safe.
-    const landedUrl = page.url();
-    if (/^https?:\/\//i.test(landedUrl)) {
-      await assertUrlAllowed(landedUrl, { resolveDns: true });
-    }
+    // Re-validate where the navigation went: a redirect could have taken us
+    // into a blocked range, onto a blocklisted host or into a path robots.txt
+    // disallows, even though the original URL was fine. A refusal leaves the
+    // page empty, so a session that keeps it cannot read the refused document.
+    const landedUrl = await assertNavigationAllowed(page, url, response, redirectGate(url, {
+      respectRobots: options.browserOptions?.respectRobots,
+      tool: options.browserOptions?.tool || 'scrape_with_actions'
+    }));
+    // What assertPageAllowed compares the page against from here on.
+    page.__crawlforgeGatedUrl = landedUrl;
 
     // Remembered on the page so the chain result can report the status of
     // the last navigation: tesla.com's Akamai denial ran a full action chain
@@ -1461,7 +1591,7 @@ export class ActionExecutor extends EventEmitter {
       // Navigate to URL. The pre-flight above repeats inside navigateToUrl —
       // that one is deliberately before page creation so a blocked URL never
       // launches a browser (tests/unit/phase1-ssrf-paths.test.js pins it).
-      await this.navigateToUrl(page, url);
+      await this.navigateToUrl(page, url, { browserOptions });
 
       // Handle CloudFlare challenges and reCAPTCHA if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/core/AuthManager.js

@@ -680,6 +680,13 @@ class AuthManager {
       return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_embedded_state's escalate:true (plan Phase 3.2) is scrape's
+    // second stage on the same terms: the projection carries the stealth
+    // browser's 5, and the tool reports 2 when the plain fetch sufficed.
+    if (tool === 'extract_embedded_state') {
+      return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -766,6 +773,11 @@ class AuthManager {
           'json format may incur external LLM cost (billed by your provider).';
         break;
       }
+      case 'extract_embedded_state':
+        note = params?.escalate === true
+          ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/core/LLMsTxtAnalyzer.js

@@ -474,6 +474,7 @@ export class LLMsTxtAnalyzer {
       const response = await safeFetch(url, {
         signal: controller.signal,
         headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
         ...options
       });
       clearTimeout(timeoutId);

package/src/core/ResearchOrchestrator.js

@@ -8,9 +8,9 @@ import { ResultRanker } from '../tools/search/ranking/ResultRanker.js';
 import { CacheManager } from './cache/CacheManager.js';
 import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
-import { safeFetch, safeGoto } from '../utils/ssrfGuard.js';
+import { safeFetch, safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
-import { preflightFetch, browserPreflight } from '../utils/robotsGate.js';
+import { preflightFetch, browserPreflight, redirectGate } from '../utils/robotsGate.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import {
@@ -791,6 +791,7 @@ export class ResearchOrchestrator extends EventEmitter {
                 const gate = await preflightFetch(source.link, { tool: 'deep_research' });
                 const fetchResponse = await safeFetch(source.link, {
                   headers: { ...gate.headers },
+                  onRedirect: gate.onRedirect,
                   signal: AbortSignal.timeout(10000)
                 });
                 if (fetchResponse.status === 429 || fetchResponse.status === 503) {
@@ -1107,14 +1108,19 @@ export class ResearchOrchestrator extends EventEmitter {
     }
     try {
       // browserPreflight (robots + platform blocklist) ran in _stealthFetchHtml;
-      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs.
-      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs });
+      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs,
+      // and puts wherever the page is redirected to through the same gate.
+      const onRedirect = redirectGate(url, { tool: 'deep_research' });
+      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs, onRedirect });
+      const landedUrl = page.url();
       // Do NOT bail on the initial HTTP status: anti-bot challenges (Cloudflare
       // Turnstile) return 403 on the first response and only resolve to the
       // real page after their JS runs. Let it settle, then judge by the
       // *rendered* content instead.
       await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
       await page.waitForTimeout(2500).catch(() => {});
+      // A challenge or a client-side redirect may have moved the page on since.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
       const html = await page.content();
       const title = (await page.title().catch(() => '')) || '';
       const bodyLen = await page.evaluate(() => document.body?.innerText?.trim().length || 0).catch(() => 0);