crawlforge-mcp-server 6.14.0 → 6.16.0

package/README.md

@@ -187,7 +187,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `read_result` | 1 | Search, slice, read lines or a JSON path from a result a tool returned with `truncated: true` and a `result_handle` (kept 1 hour on your own machine) — never fetch the page again |
 | `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings. `escalate:true` retries a blocked plain fetch once in the stealth browser and returns the page instead of the block (+5 as the projected ceiling; the charge drops back to 2 when the plain fetch succeeded and no escalation ran) |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
-| `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
+| `extract_embedded_state` | 2 (7 projected with `escalate: true`, charged 2 when the plain fetch works) | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads (references between rows resolved, plus a `data_rows` index), Nuxt (Nuxt 3's `__NUXT_DATA__` decoded), SvelteKit, Apollo, Redux, `ytInitialData`, Inertia, Shopify, JSON `data-*` attributes, `<script type="application/json">` — with a `path` to scope the result, `keys_only: true` to see its first two levels of keys, or `find: "<key>"` to get every path where a field of that name lives. `escalate: true` re-reads a walled page in the stealth browser, runs the same parser and also reads the framework globals off `window` (`window_state`). No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |
 | `map_site` | 2 | Discover and map website structure (optional `search=` ranks the discovered URLs) |
 | `process_document` | 2 | Multi-format document processing |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.14.0",
+  "version": "6.16.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -118,17 +118,17 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@mozilla/readability": "^0.6.0",
-    "mammoth": "^1.12.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.10.0",
+    "crawlforge-extractors": "^1.11.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
     "isomorphic-dompurify": "^3.9.0",
     "jsdom": "^29.0.2",
     "lru-cache": "^11.1.0",
+    "mammoth": "^1.12.2",
     "p-queue": "^9.3.3",
     "pdf-parse": "2.4.5",
     "playwright": "^1.54.2",

package/server.js

@@ -36,7 +36,8 @@ import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
 import { detectChallengePage } from "./src/utils/challengeDetection.js";
 import { stealthDocumentVerdict } from "./src/utils/stealthVerdict.js";
-import { browserPreflight } from "./src/utils/robotsGate.js";
+import { browserPreflight, redirectGate } from "./src/utils/robotsGate.js";
+import { safeGoto } from "./src/utils/ssrfGuard.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 import AuthManager from "./src/core/AuthManager.js";
@@ -55,12 +56,12 @@ import { extractTextHandler } from "./src/tools/basic/extractText.js";
 import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
-import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
+import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
 import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
 import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
-import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
+import { markPreflightRefusal, preflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
 import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
 import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
@@ -111,7 +112,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.14.0",
+  version: "6.16.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -246,16 +247,18 @@ const deepResearchTool = new DeepResearchTool();
 const trackChangesTool = new TrackChangesTool();
 const generateLLMsTxtTool = new GenerateLLMsTxtTool();
 const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
-// Phase 3: the escalation stage for `scrape`'s escalate:true and the agent's
-// automatic stealth retry (stealth review Phase 3) — one function, so both
-// go through the same gate, engine resolver and server-level proxy list.
+// Phase 3: the escalation stage for `scrape`'s escalate:true, the agent's
+// automatic stealth retry (stealth review Phase 3) and extract_embedded_state's
+// escalate:true (plan Phase 3.2) — one function, so all three go through the
+// same gate, engine resolver and server-level proxy list.
 // Injected so the tool modules never import StealthBrowserManager (that would pull a
 // browser dependency into every unit test that loads `scrape`). Same gate
 // and same browser the stealth_mode tool drives, plus the impit TLS try, and the
 // engine name is resolved here, beside its sibling, by the one resolver
 // every stealth entry point shares.
-const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
+const stealthEscalation = async ({ url, engine, respectRobots, tool, waitFor = 0, readWindowState = false }) => {
   const warnings = await stealthComplianceGate(url, respectRobots);
+  const onRedirect = stealthRedirectGate(url, respectRobots);
   const auditIdentity = { apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken() };
   // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
   // honest User-Agent is tried before any browser launches. A caller who
@@ -263,7 +266,7 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
   if (engine === 'auto' && impitEnabled() && await loadImpit()) {
     recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
-    const page = await impitFetchPage(url);
+    const page = await impitFetchPage(url, { onRedirect });
     if (page) return { ...page, warnings };
   }
   // "auto" prefers camoufox and falls back to chromium when its binary is
@@ -275,9 +278,14 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // Audit row: the request is about to go out under a browser identity. After
   // the gate (a refusal presents none) and the resolver (the engine that runs).
   recordStealthEscalation({ url, tool, engine: resolved.engine, ...auditIdentity });
+  // waitFor and readWindowState come from extract_embedded_state only; scrape
+  // and agent pass neither, so their stage is unchanged.
   const scraped = await stealthBrowserManager.scrapeWithStealth({
     url,
-    engine: resolved.engine
+    engine: resolved.engine,
+    wait_for: waitFor,
+    readWindowState,
+    onRedirect
   });
   // The RESOLVED engine: `scrape` reports it as stealth.engine, where the
   // requested "auto" would tell the caller nothing about what ran.
@@ -404,6 +412,18 @@ const VERIFY_NUMBERS_PARAM = {
 };
 
 
+/**
+ * Whether a browser tool's result is a chain the compliance gate refused.
+ *
+ * scrape_with_actions and browser_session report a failed chain as
+ * `{ success: false, error }` rather than by throwing, so the actions that did
+ * run are still in the result. withAuth waives the charge for a refusal only
+ * on an error result, so without `isError` a caller paid in full for a chain
+ * we refused to finish: a load redirected into a disallowed path, or a click
+ * that landed on one.
+ */
+const refusedByGate = (result) => result?.success === false && preflightRefusal() !== null;
+
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
   description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
@@ -456,15 +476,16 @@ registerToolIfEnabled("extract_metadata", {
 
 // Tool: extract_embedded_state
 registerToolIfEnabled("extract_embedded_state", {
-  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree instead of the whole blob. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__, and React Server Component payloads with references between rows resolved plus a data_rows index of the rows carrying data), Nuxt (Nuxt 3's __NUXT_DATA__ decoded), SvelteKit, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), ytInitialData and ytInitialPlayerResponse, Inertia, Shopify, JSON data-* attributes and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree, keys_only:true to see the first two levels of keys before choosing one, or find:\"<key>\" to get every path where a field of that name lives, each ready to pass back as `path`. A result over max_inline_chars comes back as a preview (whole lines of the JSON), data_keys and a result_handle; read the rest with read_result json_path, e.g. \"data.next_data.props\". Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page, run the same parser and also read the globals off window (window_state: __NEXT_DATA__, __NUXT__, ytInitialData and others, read after JavaScript ran) - projected at 7, charged 2 when the plain fetch worked. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
   annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
-    url: z.string().url().describe("The URL to read embedded state from"),
-    path: z.string().optional().describe("Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. \"next_data.props.pageProps\" or \"next_f[0].f\" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them."),
+    ...EMBEDDED_STATE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...MAX_INLINE_CHARS_PARAM
   }
-}, withAuth("extract_embedded_state", extractEmbeddedStateHandler));
+}, withAuth("extract_embedded_state", createExtractEmbeddedStateHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_embedded_state' })
+})));
 
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
@@ -1044,7 +1065,10 @@ registerToolIfEnabled("scrape_with_actions", {
     // array (Phase 0, 0.4).
     stripScreenshotData(result, (actionId, data) => resourceRegistry.storeScreenshot(actionId, data));
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Scrape with actions failed: ${error.message}` }], isError: true };
   }
@@ -1110,7 +1134,10 @@ registerToolIfEnabled("browser_session", {
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Browser session failed: ${error.message}` }], isError: true };
   }
@@ -1317,6 +1344,17 @@ async function stealthComplianceGate(url, respectRobots) {
   return browserPreflight(url, { respectRobots, tool: 'stealth_mode' });
 }
 
+/**
+ * The same gate for wherever a navigation to `url` is redirected to. The
+ * pre-flight above decides about the URL the caller named; a redirect can lead
+ * anywhere, so each stealth entry point hands this to its navigation.
+ *
+ * @returns {(to: string) => Promise<void>} refuses a hop by throwing
+ */
+function stealthRedirectGate(url, respectRobots) {
+  return redirectGate(url, { respectRobots, tool: 'stealth_mode' });
+}
+
 /**
  * Build the requested formats from one stealth render. The browser already
  * returned the rendered HTML and visible text, so nothing here refetches;
@@ -1441,7 +1479,8 @@ registerToolIfEnabled("stealth_mode", {
           engine: resolvedEngine.engine,
           wait_for: wait_for || 0,
           screenshot: wantsScreenshot,
-          stealthConfig
+          stealthConfig,
+          onRedirect: stealthRedirectGate(url, respect_robots)
         });
 
         // A bot-wall interstitial arrives as HTTP 200 with a title and prose
@@ -1532,7 +1571,13 @@ registerToolIfEnabled("stealth_mode", {
             // JSON-serializable — extract just the useful navigation details.
             // Explicit timeout keeps navigation inside every caller's window
             // (Playwright's default is 30s, longer than some proxy budgets).
-            const response = await page.goto(urlToTest, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            // safeGoto, not page.goto: the SSRF guard on the URL and on where
+            // it lands, and the gate above again for every redirect on the way.
+            const response = await safeGoto(page, urlToTest, {
+              waitUntil: 'domcontentloaded',
+              timeout: 20000,
+              onRedirect: stealthRedirectGate(urlToTest, respect_robots)
+            });
             navigation = {
               requestedUrl: urlToTest,
               finalUrl: page.url(),

package/src/cli/commands/stealth.js

@@ -2,7 +2,7 @@
  * stealth command — scrape a URL using stealth mode.
  */
 import { StealthBrowserManager } from '../../core/StealthBrowserManager.js';
-import { browserPreflight } from '../../utils/robotsGate.js';
+import { browserPreflight, redirectGate } from '../../utils/robotsGate.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
 
@@ -22,7 +22,7 @@ export function register(program) {
         // a different door, not by a different set of rules.
         execute: async (p) => {
           await browserPreflight(p.url, { tool: 'stealth_mode' });
-          return mgr.scrapeWithStealth(p);
+          return mgr.scrapeWithStealth({ ...p, onRedirect: redirectGate(p.url, { tool: 'stealth_mode' }) });
         }
       };
       await runTool(wrapperTool, {

package/src/core/ActionExecutor.js

@@ -7,8 +7,8 @@ import { z } from 'zod';
 import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
-import { assertUrlAllowed } from '../utils/ssrfGuard.js';
-import { browserPreflight } from '../utils/robotsGate.js';
+import { assertUrlAllowed, assertNavigationAllowed } from '../utils/ssrfGuard.js';
+import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
 import { settlePage } from './browser/settle.js';
 import { handleConsent } from './browser/consent.js';
@@ -318,11 +318,10 @@ export class ActionExecutor extends EventEmitter {
         // Capture the LIVE post-action page state before the page is closed,
         // so callers can extract final content reflecting all actions
         // (instead of re-fetching the original URL).
-        try {
-          executionContext.finalHtml = await page.content();
-          executionContext.finalUrl = page.url();
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture final page content: ' + captureErr.message);
+        const finalState = await this.captureState(page, browserOptions, 'final page content');
+        if (finalState) {
+          executionContext.finalHtml = finalState.html;
+          executionContext.finalUrl = finalState.url;
         }
 
         this.stats.successfulChains++;
@@ -336,6 +335,9 @@ export class ActionExecutor extends EventEmitter {
         // Capture error screenshot if enabled
         if ((validatedChain.screenshotOnError ?? this.enableScreenshotOnError) && page) {
           try {
+            // Not of a page the gate has not passed: the chain may have failed
+            // for another reason after the page moved.
+            await this.assertPageAllowed(page, browserOptions);
             const errorScreenshot = await this.captureScreenshot(page, {
               fullPage: true,
               description: 'Error screenshot'
@@ -473,7 +475,9 @@ export class ActionExecutor extends EventEmitter {
           // (form half-filled, menu open, possibly a different URL) is not a
           // retry. Reload the starting URL so every attempt begins where the
           // first one did.
-          await this.navigateToUrl(page, executionContext.url);
+          await this.navigateToUrl(page, executionContext.url, {
+            browserOptions: executionContext.browserOptions
+          });
         }
 
         // Execute actions in sequence
@@ -486,7 +490,7 @@ export class ActionExecutor extends EventEmitter {
             ...chain.actions[i],
             timeout: Math.min(chain.actions[i].timeout || chain.timeout, chain.timeout)
           };
-          const actionResult = await this.executeActionInternal(page, action, executionContext);
+          const actionResult = await this.executeGatedAction(page, action, executionContext);
           
           executionContext.results.push(actionResult);
           this.stats.totalActions++;
@@ -510,7 +514,7 @@ export class ActionExecutor extends EventEmitter {
             
             // Handle action failure
             if (!action.continueOnError && !chain.continueOnError) {
-              throw new Error('Action failed: ' + actionResult.error);
+              throw Object.assign(new Error('Action failed: ' + actionResult.error), { code: actionResult.errorCode });
             }
           }
 
@@ -519,18 +523,16 @@ export class ActionExecutor extends EventEmitter {
           // the ALLOW_JAVASCRIPT_EXECUTION flag and doesn't add phantom
           // actions to the chain's failure/success counts.
           if (action.captureAfter) {
-            try {
-              const capturedHtml = await page.content();
+            const captured = await this.captureState(page, executionContext.browserOptions, 'intermediate state');
+            if (captured) {
               executionContext.capturedStates = executionContext.capturedStates || [];
               executionContext.capturedStates.push({
                 afterActionIndex: i,
                 afterActionId: actionResult.id,
-                url: page.url(),
-                html: capturedHtml,
+                url: captured.url,
+                html: captured.html,
                 timestamp: Date.now()
               });
-            } catch (captureErr) {
-              this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
             }
           }
 
@@ -551,7 +553,12 @@ export class ActionExecutor extends EventEmitter {
           attempt: attempt + 1, success: false, error: error.message, results: [...executionContext.results]
         });
         this.log('warn', 'Chain execution attempt ' + (attempt + 1) + ' failed: ' + error.message);
-        
+
+        // A refusal is the gate's answer, not a fault. Replaying the chain
+        // asks the same question again, after sending the site every request
+        // that led up to it a second time.
+        if (gateRefusalCode(error)) break;
+
         if (attempt < chain.retryChain) {
           // Wait before retry
           await this.delay(1000 * Math.pow(2, attempt));
@@ -577,8 +584,12 @@ export class ActionExecutor extends EventEmitter {
    *
    * Gating comes for free and must not be duplicated by callers: a `navigate`
    * action goes through executeNavigateAction, which re-runs the SSRF guard and
-   * the blocklist/robots gate on every hop (see the comment at its definition).
-   * `browserOptions.respectRobots` is what reaches that gate.
+   * the blocklist/robots gate on every hop (see the comment at its definition),
+   * and every action is run through executeGatedAction, which holds the page to
+   * the same gate wherever a click or a script has taken it. A refusal there
+   * throws rather than failing one action: the page has been emptied, and
+   * nothing after it could run. `browserOptions.respectRobots` is what reaches
+   * both gates.
    *
    * No `finalHtml` here, unlike the chain: the page is still open afterwards, so
    * content is read from it when it is asked for rather than on every call.
@@ -614,7 +625,7 @@ export class ActionExecutor extends EventEmitter {
       // per-call timeout is applied as the default for actions that set none.
       const action = timeout && !actions[i].timeout ? { ...actions[i], timeout } : actions[i];
 
-      const actionResult = await this.executeActionInternal(page, action, executionContext);
+      const actionResult = await this.executeGatedAction(page, action, executionContext);
       executionContext.results.push(actionResult);
       this.stats.totalActions++;
 
@@ -638,17 +649,15 @@ export class ActionExecutor extends EventEmitter {
       }
 
       if (action.captureAfter) {
-        try {
-          const capturedHtml = await page.content();
+        const captured = await this.captureState(page, browserOptions, 'intermediate state');
+        if (captured) {
           executionContext.capturedStates.push({
             afterActionIndex: i,
             afterActionId: actionResult.id,
-            url: page.url(),
-            html: capturedHtml,
+            url: captured.url,
+            html: captured.html,
             timestamp: Date.now()
           });
-        } catch (captureErr) {
-          this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
         }
       }
 
@@ -679,6 +688,88 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Hold the page to the gate wherever it stands now.
+   *
+   * navigateToUrl gates a load and its redirects. After that a page moves
+   * without being sent anywhere: a click follows a link, a form posts, a
+   * script redirects, a client-side router rewrites the address. Any URL the
+   * page has reached since it was last checked gets what a `navigate` to it
+   * would: the SSRF guard, then the blocklist/robots gate with this call's
+   * `respectRobots`. A refusal leaves the page empty and throws.
+   *
+   * Cheap when nothing moved (one string comparison), so it is called on both
+   * sides of everything that touches the page.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @returns {Promise<void>}
+   * @throws {Error} SSRF_BLOCKED, BlockedHostError or RobotsDisallowedError
+   */
+  async assertPageAllowed(page, browserOptions = {}) {
+    const gatedUrl = page.__crawlforgeGatedUrl;
+    if (page.url() === gatedUrl) return;
+    page.__crawlforgeGatedUrl = await assertNavigationAllowed(page, gatedUrl, null, pageMoveGate(gatedUrl, {
+      respectRobots: browserOptions?.respectRobots,
+      tool: browserOptions?.tool || 'scrape_with_actions'
+    }));
+  }
+
+  /**
+   * Read from the page with the gate held on both sides of the read, so what
+   * comes back is from a document that passed it: the check after the read
+   * discards anything read while the page was moving somewhere it may not be.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {() => Promise<T>} read - the read to make
+   * @returns {Promise<T>}
+   * @template T
+   */
+  async readGated(page, browserOptions, read) {
+    await this.assertPageAllowed(page, browserOptions);
+    const value = await read();
+    await this.assertPageAllowed(page, browserOptions);
+    return value;
+  }
+
+  /**
+   * The page's URL and HTML, read natively (no in-page JS execution, so it
+   * works regardless of ALLOW_JAVASCRIPT_EXECUTION). A page that cannot be
+   * read yields null and a warning; a page the gate refuses throws.
+   * @param {Page} page - Playwright page
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @param {string} what - names the capture in the warning
+   * @returns {Promise<{ url: string, html: string }|null>}
+   */
+  async captureState(page, browserOptions, what) {
+    return await this.readGated(page, browserOptions, async () => {
+      try {
+        return { html: await page.content(), url: page.url() };
+      } catch (captureErr) {
+        this.log('warn', 'Failed to capture ' + what + ': ' + captureErr.message);
+        return null;
+      }
+    });
+  }
+
+  /**
+   * Run one action with the page held to the gate before it and after it.
+   * Before, because an action must not read or drive a page that moved while
+   * nothing was looking; after, because the action is what usually moves it,
+   * and its result is not kept if where it led is refused.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Action to execute
+   * @param {Object} executionContext - Execution context (`browserOptions` feeds the gate)
+   * @returns {Promise<Object>} Action result
+   */
+  async executeGatedAction(page, action, executionContext) {
+    const browserOptions = executionContext?.browserOptions;
+    // A navigate leaves the page it is on and gates the URL it is sent to.
+    if (action.type !== 'navigate') await this.assertPageAllowed(page, browserOptions);
+    const actionResult = await this.executeActionInternal(page, action, executionContext);
+    await this.assertPageAllowed(page, browserOptions);
+    return actionResult;
+  }
+
   /**
    * Execute individual action (original internal method)
    * @param {Page} page - Playwright page
@@ -753,11 +844,15 @@ export class ActionExecutor extends EventEmitter {
       return actionResult;
 
     } catch (error) {
+      const refusalCode = gateRefusalCode(error);
       const actionResult = {
         id: actionId,
         type: action.type,
         success: false,
         error: error.message,
+        // Only when the gate refused (a `navigate` to a URL it would not
+        // load), so the chain can tell an answer from a fault.
+        ...(refusalCode ? { errorCode: refusalCode } : {}),
         executionTime: Date.now() - actionStartTime,
         timestamp: Date.now(),
         description: action.description
@@ -1239,7 +1334,8 @@ export class ActionExecutor extends EventEmitter {
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
-      timeout
+      timeout,
+      browserOptions: executionContext?.browserOptions
     });
     page.__crawlforgeGateWarnings = gateWarnings;
 
@@ -1392,12 +1488,14 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Navigate an existing page to a URL under the SSRF checks every load needs.
-   * Used for the initial load, again before each chain retry, and by the
-   * `navigate` action.
+   * Navigate an existing page to a URL under the SSRF checks every load needs,
+   * and put wherever it is redirected to through the same blocklist/robots
+   * gate the URL itself passed (assertRobotsAllowed). Used for the initial
+   * load, again before each chain retry, and by the `navigate` action.
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
-   * @param {{ waitUntil?: string, timeout?: number }} [options] - Navigation options
+   * @param {{ waitUntil?: string, timeout?: number, browserOptions?: Object }} [options] -
+   *   Navigation options; `browserOptions` carries `respectRobots` to the redirect gate
    * @returns {Promise<void>}
    */
   async navigateToUrl(page, url, options = {}) {
@@ -1410,12 +1508,16 @@ export class ActionExecutor extends EventEmitter {
       timeout: options.timeout || 30000
     });
 
-    // Re-validate the landed URL: a redirect during navigation could have
-    // taken us into a blocked range even though the original URL was safe.
-    const landedUrl = page.url();
-    if (/^https?:\/\//i.test(landedUrl)) {
-      await assertUrlAllowed(landedUrl, { resolveDns: true });
-    }
+    // Re-validate where the navigation went: a redirect could have taken us
+    // into a blocked range, onto a blocklisted host or into a path robots.txt
+    // disallows, even though the original URL was fine. A refusal leaves the
+    // page empty, so a session that keeps it cannot read the refused document.
+    const landedUrl = await assertNavigationAllowed(page, url, response, redirectGate(url, {
+      respectRobots: options.browserOptions?.respectRobots,
+      tool: options.browserOptions?.tool || 'scrape_with_actions'
+    }));
+    // What assertPageAllowed compares the page against from here on.
+    page.__crawlforgeGatedUrl = landedUrl;
 
     // Remembered on the page so the chain result can report the status of
     // the last navigation: tesla.com's Akamai denial ran a full action chain
@@ -1489,7 +1591,7 @@ export class ActionExecutor extends EventEmitter {
       // Navigate to URL. The pre-flight above repeats inside navigateToUrl —
       // that one is deliberately before page creation so a blocked URL never
       // launches a browser (tests/unit/phase1-ssrf-paths.test.js pins it).
-      await this.navigateToUrl(page, url);
+      await this.navigateToUrl(page, url, { browserOptions });
 
       // Handle CloudFlare challenges and reCAPTCHA if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/core/AuthManager.js

@@ -680,6 +680,13 @@ class AuthManager {
       return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_embedded_state's escalate:true (plan Phase 3.2) is scrape's
+    // second stage on the same terms: the projection carries the stealth
+    // browser's 5, and the tool reports 2 when the plain fetch sufficed.
+    if (tool === 'extract_embedded_state') {
+      return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -766,6 +773,11 @@ class AuthManager {
           'json format may incur external LLM cost (billed by your provider).';
         break;
       }
+      case 'extract_embedded_state':
+        note = params?.escalate === true
+          ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/core/LLMsTxtAnalyzer.js

@@ -474,6 +474,7 @@ export class LLMsTxtAnalyzer {
       const response = await safeFetch(url, {
         signal: controller.signal,
         headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
         ...options
       });
       clearTimeout(timeoutId);

package/src/core/ResearchOrchestrator.js

@@ -8,9 +8,9 @@ import { ResultRanker } from '../tools/search/ranking/ResultRanker.js';
 import { CacheManager } from './cache/CacheManager.js';
 import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
-import { safeFetch, safeGoto } from '../utils/ssrfGuard.js';
+import { safeFetch, safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
-import { preflightFetch, browserPreflight } from '../utils/robotsGate.js';
+import { preflightFetch, browserPreflight, redirectGate } from '../utils/robotsGate.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import {
@@ -791,6 +791,7 @@ export class ResearchOrchestrator extends EventEmitter {
                 const gate = await preflightFetch(source.link, { tool: 'deep_research' });
                 const fetchResponse = await safeFetch(source.link, {
                   headers: { ...gate.headers },
+                  onRedirect: gate.onRedirect,
                   signal: AbortSignal.timeout(10000)
                 });
                 if (fetchResponse.status === 429 || fetchResponse.status === 503) {
@@ -1107,14 +1108,19 @@ export class ResearchOrchestrator extends EventEmitter {
     }
     try {
       // browserPreflight (robots + platform blocklist) ran in _stealthFetchHtml;
-      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs.
-      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs });
+      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs,
+      // and puts wherever the page is redirected to through the same gate.
+      const onRedirect = redirectGate(url, { tool: 'deep_research' });
+      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs, onRedirect });
+      const landedUrl = page.url();
       // Do NOT bail on the initial HTTP status: anti-bot challenges (Cloudflare
       // Turnstile) return 403 on the first response and only resolve to the
       // real page after their JS runs. Let it settle, then judge by the
       // *rendered* content instead.
       await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
       await page.waitForTimeout(2500).catch(() => {});
+      // A challenge or a client-side redirect may have moved the page on since.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
       const html = await page.content();
       const title = (await page.title().catch(() => '')) || '';
       const bodyLen = await page.evaluate(() => document.body?.innerText?.trim().length || 0).catch(() => 0);