crawlforge-mcp-server 6.14.0 → 6.17.1

package/src/core/ActionExecutor.js

@@ -7,11 +7,12 @@ import { z } from 'zod';
 import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
-import { assertUrlAllowed } from '../utils/ssrfGuard.js';
-import { browserPreflight } from '../utils/robotsGate.js';
+import { assertUrlAllowed, assertNavigationAllowed } from '../utils/ssrfGuard.js';
+import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
 import { settlePage } from './browser/settle.js';
 import { handleConsent } from './browser/consent.js';
+import { stealthDocumentVerdict } from '../utils/stealthVerdict.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -31,6 +32,13 @@ const ACTION_TIMEOUT_GRACE_MS = 2000;
 // to work cost several times its stated timeout.
 const RECOVERY_TIMEOUT_MS = 3000;
 
+// How long a navigation whose document reads as a wall is watched for the wall
+// replacing itself before it is reported, re-read every NAVIGATION_WALL_POLL_MS.
+// AWS WAF's interstitial reloads into amazon.com's homepage about 0.4 s after
+// domcontentloaded (measured 2026-10-03); nothing here acts on the page.
+const NAVIGATION_WALL_GRACE_MS = 3000;
+const NAVIGATION_WALL_POLL_MS = 500;
+
 // The only states locator.waitFor()/page.waitForSelector() accept. The rest of
 // the wait-action enum (enabled/disabled/stable) are ElementHandle states and
 // have to go through waitForElementState instead — passing them here is
@@ -312,17 +320,20 @@ export class ActionExecutor extends EventEmitter {
           executionContext.consent = await handleConsent(page, browserOptions.consent);
         }
 
+        // navigations[0] is the initial load; each `navigate` action appends
+        // its own (executeNavigateAction).
+        executionContext.navigations = [await this.checkNavigation(page, url, browserOptions)];
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
 
         // Capture the LIVE post-action page state before the page is closed,
         // so callers can extract final content reflecting all actions
         // (instead of re-fetching the original URL).
-        try {
-          executionContext.finalHtml = await page.content();
-          executionContext.finalUrl = page.url();
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture final page content: ' + captureErr.message);
+        const finalState = await this.captureState(page, browserOptions, 'final page content');
+        if (finalState) {
+          executionContext.finalHtml = finalState.html;
+          executionContext.finalUrl = finalState.url;
         }
 
         this.stats.successfulChains++;
@@ -336,6 +347,9 @@ export class ActionExecutor extends EventEmitter {
         // Capture error screenshot if enabled
         if ((validatedChain.screenshotOnError ?? this.enableScreenshotOnError) && page) {
           try {
+            // Not of a page the gate has not passed: the chain may have failed
+            // for another reason after the page moved.
+            await this.assertPageAllowed(page, browserOptions);
             const errorScreenshot = await this.captureScreenshot(page, {
               fullPage: true,
               description: 'Error screenshot'
@@ -417,6 +431,7 @@ export class ActionExecutor extends EventEmitter {
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
+        navigations: executionContext.navigations || [],
         consent: executionContext.consent,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
@@ -444,6 +459,7 @@ export class ActionExecutor extends EventEmitter {
         // results, the error screenshot, and any intermediate-state
         // captures) instead of discarding them.
         results: executionContext?.results || [],
+        navigations: executionContext?.navigations || [],
         attempt: executionContext?.attempt ?? 0,
         attempts: executionContext?.attempts || [],
         screenshots: executionContext?.screenshots || [],
@@ -473,7 +489,14 @@ export class ActionExecutor extends EventEmitter {
           // (form half-filled, menu open, possibly a different URL) is not a
           // retry. Reload the starting URL so every attempt begins where the
           // first one did.
-          await this.navigateToUrl(page, executionContext.url);
+          await this.navigateToUrl(page, executionContext.url, {
+            browserOptions: executionContext.browserOptions
+          });
+          // Only the last attempt's navigations are reported, so the reload
+          // that starts this one is navigations[0].
+          executionContext.navigations = [
+            await this.checkNavigation(page, executionContext.url, executionContext.browserOptions)
+          ];
         }
 
         // Execute actions in sequence
@@ -486,7 +509,7 @@ export class ActionExecutor extends EventEmitter {
             ...chain.actions[i],
             timeout: Math.min(chain.actions[i].timeout || chain.timeout, chain.timeout)
           };
-          const actionResult = await this.executeActionInternal(page, action, executionContext);
+          const actionResult = await this.executeGatedAction(page, action, executionContext);
           
           executionContext.results.push(actionResult);
           this.stats.totalActions++;
@@ -510,7 +533,7 @@ export class ActionExecutor extends EventEmitter {
             
             // Handle action failure
             if (!action.continueOnError && !chain.continueOnError) {
-              throw new Error('Action failed: ' + actionResult.error);
+              throw Object.assign(new Error('Action failed: ' + actionResult.error), { code: actionResult.errorCode });
             }
           }
 
@@ -519,18 +542,16 @@ export class ActionExecutor extends EventEmitter {
           // the ALLOW_JAVASCRIPT_EXECUTION flag and doesn't add phantom
           // actions to the chain's failure/success counts.
           if (action.captureAfter) {
-            try {
-              const capturedHtml = await page.content();
+            const captured = await this.captureState(page, executionContext.browserOptions, 'intermediate state');
+            if (captured) {
               executionContext.capturedStates = executionContext.capturedStates || [];
               executionContext.capturedStates.push({
                 afterActionIndex: i,
                 afterActionId: actionResult.id,
-                url: page.url(),
-                html: capturedHtml,
+                url: captured.url,
+                html: captured.html,
                 timestamp: Date.now()
               });
-            } catch (captureErr) {
-              this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
             }
           }
 
@@ -551,7 +572,12 @@ export class ActionExecutor extends EventEmitter {
           attempt: attempt + 1, success: false, error: error.message, results: [...executionContext.results]
         });
         this.log('warn', 'Chain execution attempt ' + (attempt + 1) + ' failed: ' + error.message);
-        
+
+        // A refusal is the gate's answer, not a fault. Replaying the chain
+        // asks the same question again, after sending the site every request
+        // that led up to it a second time.
+        if (gateRefusalCode(error)) break;
+
         if (attempt < chain.retryChain) {
           // Wait before retry
           await this.delay(1000 * Math.pow(2, attempt));
@@ -577,8 +603,12 @@ export class ActionExecutor extends EventEmitter {
    *
    * Gating comes for free and must not be duplicated by callers: a `navigate`
    * action goes through executeNavigateAction, which re-runs the SSRF guard and
-   * the blocklist/robots gate on every hop (see the comment at its definition).
-   * `browserOptions.respectRobots` is what reaches that gate.
+   * the blocklist/robots gate on every hop (see the comment at its definition),
+   * and every action is run through executeGatedAction, which holds the page to
+   * the same gate wherever a click or a script has taken it. A refusal there
+   * throws rather than failing one action: the page has been emptied, and
+   * nothing after it could run. `browserOptions.respectRobots` is what reaches
+   * both gates.
    *
    * No `finalHtml` here, unlike the chain: the page is still open afterwards, so
    * content is read from it when it is asked for rather than on every call.
@@ -614,7 +644,7 @@ export class ActionExecutor extends EventEmitter {
       // per-call timeout is applied as the default for actions that set none.
       const action = timeout && !actions[i].timeout ? { ...actions[i], timeout } : actions[i];
 
-      const actionResult = await this.executeActionInternal(page, action, executionContext);
+      const actionResult = await this.executeGatedAction(page, action, executionContext);
       executionContext.results.push(actionResult);
       this.stats.totalActions++;
 
@@ -638,17 +668,15 @@ export class ActionExecutor extends EventEmitter {
       }
 
       if (action.captureAfter) {
-        try {
-          const capturedHtml = await page.content();
+        const captured = await this.captureState(page, browserOptions, 'intermediate state');
+        if (captured) {
           executionContext.capturedStates.push({
             afterActionIndex: i,
             afterActionId: actionResult.id,
-            url: page.url(),
-            html: capturedHtml,
+            url: captured.url,
+            html: captured.html,
             timestamp: Date.now()
           });
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
         }
       }
 
@@ -679,6 +707,88 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Hold the page to the gate wherever it stands now.
+   *
+   * navigateToUrl gates a load and its redirects. After that a page moves
+   * without being sent anywhere: a click follows a link, a form posts, a
+   * script redirects, a client-side router rewrites the address. Any URL the
+   * page has reached since it was last checked gets what a `navigate` to it
+   * would: the SSRF guard, then the blocklist/robots gate with this call's
+   * `respectRobots`. A refusal leaves the page empty and throws.
+   *
+   * Cheap when nothing moved (one string comparison), so it is called on both
+   * sides of everything that touches the page.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @returns {Promise<void>}
+   * @throws {Error} SSRF_BLOCKED, BlockedHostError or RobotsDisallowedError
+   */
+  async assertPageAllowed(page, browserOptions = {}) {
+    const gatedUrl = page.__crawlforgeGatedUrl;
+    if (page.url() === gatedUrl) return;
+    page.__crawlforgeGatedUrl = await assertNavigationAllowed(page, gatedUrl, null, pageMoveGate(gatedUrl, {
+      respectRobots: browserOptions?.respectRobots,
+      tool: browserOptions?.tool || 'scrape_with_actions'
+    }));
+  }
+
+  /**
+   * Read from the page with the gate held on both sides of the read, so what
+   * comes back is from a document that passed it: the check after the read
+   * discards anything read while the page was moving somewhere it may not be.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {() => Promise<T>} read - the read to make
+   * @returns {Promise<T>}
+   * @template T
+   */
+  async readGated(page, browserOptions, read) {
+    await this.assertPageAllowed(page, browserOptions);
+    const value = await read();
+    await this.assertPageAllowed(page, browserOptions);
+    return value;
+  }
+
+  /**
+   * The page's URL and HTML, read natively (no in-page JS execution, so it
+   * works regardless of ALLOW_JAVASCRIPT_EXECUTION). A page that cannot be
+   * read yields null and a warning; a page the gate refuses throws.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {string} what - names the capture in the warning
+   * @returns {Promise<{ url: string, html: string }|null>}
+   */
+  async captureState(page, browserOptions, what) {
+    return await this.readGated(page, browserOptions, async () => {
+      try {
+        return { html: await page.content(), url: page.url() };
+      } catch (captureErr) {
+        this.log('warn', 'Failed to capture ' + what + ': ' + captureErr.message);
+        return null;
+      }
+    });
+  }
+
+  /**
+   * Run one action with the page held to the gate before it and after it.
+   * Before, because an action must not read or drive a page that moved while
+   * nothing was looking; after, because the action is what usually moves it,
+   * and its result is not kept if where it led is refused.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Action to execute
+   * @param {Object} executionContext - Execution context (`browserOptions` feeds the gate)
+   * @returns {Promise<Object>} Action result
+   */
+  async executeGatedAction(page, action, executionContext) {
+    const browserOptions = executionContext?.browserOptions;
+    // A navigate leaves the page it is on and gates the URL it is sent to.
+    if (action.type !== 'navigate') await this.assertPageAllowed(page, browserOptions);
+    const actionResult = await this.executeActionInternal(page, action, executionContext);
+    await this.assertPageAllowed(page, browserOptions);
+    return actionResult;
+  }
+
   /**
    * Execute individual action (original internal method)
    * @param {Page} page - Playwright page
@@ -753,11 +863,15 @@ export class ActionExecutor extends EventEmitter {
       return actionResult;
 
     } catch (error) {
+      const refusalCode = gateRefusalCode(error);
       const actionResult = {
         id: actionId,
         type: action.type,
         success: false,
         error: error.message,
+        // Only when the gate refused (a `navigate` to a URL it would not
+        // load), so the chain can tell an answer from a fault.
+        ...(refusalCode ? { errorCode: refusalCode } : {}),
         executionTime: Date.now() - actionStartTime,
         timestamp: Date.now(),
         description: action.description
@@ -1239,7 +1353,8 @@ export class ActionExecutor extends EventEmitter {
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
-      timeout
+      timeout,
+      browserOptions: executionContext?.browserOptions
     });
     page.__crawlforgeGateWarnings = gateWarnings;
 
@@ -1250,14 +1365,73 @@ export class ActionExecutor extends EventEmitter {
       ? await handleConsent(page, consentMode, { timeout: Math.min(2000, timeout - (Date.now() - startedAt)) })
       : undefined;
 
+    // Reporting only: the navigation happened, so the action succeeds whether
+    // or not the document it reached is a wall. The wait for a wall to clear
+    // gets what is left of the deadline, like consent above.
+    const navigation = await this.checkNavigation(page, action.url, executionContext?.browserOptions, {
+      graceMs: Math.min(NAVIGATION_WALL_GRACE_MS, timeout - (Date.now() - startedAt))
+    });
+    executionContext?.navigations?.push(navigation);
+
     return {
       url: action.url,
-      finalUrl: page.url(),
+      finalUrl: navigation.finalUrl,
       waitUntil: action.waitUntil || 'domcontentloaded',
+      httpStatus: navigation.httpStatus,
+      ...(navigation.blocked ? { blocked: navigation.blocked } : {}),
       ...(consent ? { consent } : {})
     };
   }
 
+  /**
+   * What the navigation that just finished reached: the page, or a wall. The
+   * same verdict the chain's final document gets, run on this document with
+   * the status of this navigation. A document that reads as a wall is re-read
+   * for up to `graceMs` first, because some walls replace themselves (AWS WAF's
+   * interstitial reloads into the page); one that cannot be read at all is
+   * reported without a verdict rather than as a wall.
+   * @param {Page} page - Playwright page, just navigated
+   * @param {string} url - the URL the navigation was sent to
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` for the read's gate)
+   * @param {{ graceMs?: number }} [options]
+   * @returns {Promise<{ url: string, finalUrl: string, httpStatus: number|null, blocked?: { vendor: string, evidence: string } }>}
+   */
+  async checkNavigation(page, url, browserOptions, { graceMs = NAVIGATION_WALL_GRACE_MS } = {}) {
+    const httpStatus = page.__crawlforgeNavigation?.status ?? null;
+    const read = () => this.readGated(page, browserOptions, async () => {
+      try {
+        const [title, html, text] = await Promise.all([
+          page.title(),
+          page.content(),
+          page.evaluate(() => (document.body ? document.body.innerText : ''))
+        ]);
+        return stealthDocumentVerdict(
+          { url: page.url(), title, html, text, status: httpStatus },
+          { allowEmpty: true, fetcher: browserOptions?.stealthMode?.enabled ? 'the stealth browser' : 'the browser' }
+        );
+      } catch {
+        return null; // mid-navigation, or a page that cannot be read
+      }
+    });
+
+    let verdict = await read();
+    const deadline = Date.now() + graceMs;
+    while (verdict?.blocked && Date.now() + NAVIGATION_WALL_POLL_MS <= deadline) {
+      await this.delay(NAVIGATION_WALL_POLL_MS);
+      const next = await read();
+      // A read that fails mid-reload says nothing yet; keep the wall until a
+      // document answers.
+      if (next) verdict = next;
+    }
+
+    return {
+      url,
+      finalUrl: page.url(),
+      httpStatus,
+      ...(verdict?.blocked ? { blocked: verdict.blocked } : {})
+    };
+  }
+
   /**
    * Execute screenshot action
    * @param {Page} page - Playwright page
@@ -1392,12 +1566,14 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Navigate an existing page to a URL under the SSRF checks every load needs.
-   * Used for the initial load, again before each chain retry, and by the
-   * `navigate` action.
+   * Navigate an existing page to a URL under the SSRF checks every load needs,
+   * and put wherever it is redirected to through the same blocklist/robots
+   * gate the URL itself passed (assertRobotsAllowed). Used for the initial
+   * load, again before each chain retry, and by the `navigate` action.
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
-   * @param {{ waitUntil?: string, timeout?: number }} [options] - Navigation options
+   * @param {{ waitUntil?: string, timeout?: number, browserOptions?: Object }} [options] -
+   *   Navigation options; `browserOptions` carries `respectRobots` to the redirect gate
    * @returns {Promise<void>}
    */
   async navigateToUrl(page, url, options = {}) {
@@ -1410,12 +1586,16 @@ export class ActionExecutor extends EventEmitter {
       timeout: options.timeout || 30000
     });
 
-    // Re-validate the landed URL: a redirect during navigation could have
-    // taken us into a blocked range even though the original URL was safe.
-    const landedUrl = page.url();
-    if (/^https?:\/\//i.test(landedUrl)) {
-      await assertUrlAllowed(landedUrl, { resolveDns: true });
-    }
+    // Re-validate where the navigation went: a redirect could have taken us
+    // into a blocked range, onto a blocklisted host or into a path robots.txt
+    // disallows, even though the original URL was fine. A refusal leaves the
+    // page empty, so a session that keeps it cannot read the refused document.
+    const landedUrl = await assertNavigationAllowed(page, url, response, redirectGate(url, {
+      respectRobots: options.browserOptions?.respectRobots,
+      tool: options.browserOptions?.tool || 'scrape_with_actions'
+    }));
+    // What assertPageAllowed compares the page against from here on.
+    page.__crawlforgeGatedUrl = landedUrl;
 
     // Remembered on the page so the chain result can report the status of
     // the last navigation: tesla.com's Akamai denial ran a full action chain
@@ -1489,7 +1669,7 @@ export class ActionExecutor extends EventEmitter {
       // Navigate to URL. The pre-flight above repeats inside navigateToUrl —
       // that one is deliberately before page creation so a blocked URL never
       // launches a browser (tests/unit/phase1-ssrf-paths.test.js pins it).
-      await this.navigateToUrl(page, url);
+      await this.navigateToUrl(page, url, { browserOptions });
 
       // Handle CloudFlare challenges and reCAPTCHA if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/core/AuthManager.js

@@ -680,6 +680,13 @@ class AuthManager {
       return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_embedded_state's escalate:true (plan Phase 3.2) is scrape's
+    // second stage on the same terms: the projection carries the stealth
+    // browser's 5, and the tool reports 2 when the plain fetch sufficed.
+    if (tool === 'extract_embedded_state') {
+      return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -766,6 +773,11 @@ class AuthManager {
           'json format may incur external LLM cost (billed by your provider).';
         break;
       }
+      case 'extract_embedded_state':
+        note = params?.escalate === true
+          ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/core/LLMsTxtAnalyzer.js

@@ -474,6 +474,7 @@ export class LLMsTxtAnalyzer {
       const response = await safeFetch(url, {
         signal: controller.signal,
         headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
         ...options
       });
       clearTimeout(timeoutId);

package/src/core/ResearchOrchestrator.js

@@ -8,9 +8,9 @@ import { ResultRanker } from '../tools/search/ranking/ResultRanker.js';
 import { CacheManager } from './cache/CacheManager.js';
 import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
-import { safeFetch, safeGoto } from '../utils/ssrfGuard.js';
+import { safeFetch, safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
-import { preflightFetch, browserPreflight } from '../utils/robotsGate.js';
+import { preflightFetch, browserPreflight, redirectGate } from '../utils/robotsGate.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import {
@@ -791,6 +791,7 @@ export class ResearchOrchestrator extends EventEmitter {
                 const gate = await preflightFetch(source.link, { tool: 'deep_research' });
                 const fetchResponse = await safeFetch(source.link, {
                   headers: { ...gate.headers },
+                  onRedirect: gate.onRedirect,
                   signal: AbortSignal.timeout(10000)
                 });
                 if (fetchResponse.status === 429 || fetchResponse.status === 503) {
@@ -1107,14 +1108,19 @@ export class ResearchOrchestrator extends EventEmitter {
     }
     try {
       // browserPreflight (robots + platform blocklist) ran in _stealthFetchHtml;
-      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs.
-      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs });
+      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs,
+      // and puts wherever the page is redirected to through the same gate.
+      const onRedirect = redirectGate(url, { tool: 'deep_research' });
+      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs, onRedirect });
+      const landedUrl = page.url();
       // Do NOT bail on the initial HTTP status: anti-bot challenges (Cloudflare
       // Turnstile) return 403 on the first response and only resolve to the
       // real page after their JS runs. Let it settle, then judge by the
       // *rendered* content instead.
       await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
       await page.waitForTimeout(2500).catch(() => {});
+      // A challenge or a client-side redirect may have moved the page on since.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
       const html = await page.content();
       const title = (await page.title().catch(() => '')) || '';
       const bodyLen = await page.evaluate(() => document.body?.innerText?.trim().length || 0).catch(() => 0);