crawlforge-mcp-server 6.12.1 → 6.16.0

package/README.md

@@ -187,7 +187,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `read_result` | 1 | Search, slice, read lines or a JSON path from a result a tool returned with `truncated: true` and a `result_handle` (kept 1 hour on your own machine) — never fetch the page again |
 | `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings. `escalate:true` retries a blocked plain fetch once in the stealth browser and returns the page instead of the block (+5 as the projected ceiling; the charge drops back to 2 when the plain fetch succeeded and no escalation ran) |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
-| `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
+| `extract_embedded_state` | 2 (7 projected with `escalate: true`, charged 2 when the plain fetch works) | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads (references between rows resolved, plus a `data_rows` index), Nuxt (Nuxt 3's `__NUXT_DATA__` decoded), SvelteKit, Apollo, Redux, `ytInitialData`, Inertia, Shopify, JSON `data-*` attributes, `<script type="application/json">` — with a `path` to scope the result, `keys_only: true` to see its first two levels of keys, or `find: "<key>"` to get every path where a field of that name lives. `escalate: true` re-reads a walled page in the stealth browser, runs the same parser and also reads the framework globals off `window` (`window_state`). No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |
 | `map_site` | 2 | Discover and map website structure (optional `search=` ranks the discovered URLs) |
 | `process_document` | 2 | Multi-format document processing |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.12.1",
+  "version": "6.16.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,21 +113,22 @@
     "package.json"
   ],
   "dependencies": {
+    "@duckduckgo/autoconsent": "^16.43.0",
     "@modelcontextprotocol/core": "^2.0.0",
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@mozilla/readability": "^0.6.0",
-    "mammoth": "^1.12.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.10.0",
+    "crawlforge-extractors": "^1.11.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
     "isomorphic-dompurify": "^3.9.0",
     "jsdom": "^29.0.2",
     "lru-cache": "^11.1.0",
+    "mammoth": "^1.12.2",
     "p-queue": "^9.3.3",
     "pdf-parse": "2.4.5",
     "playwright": "^1.54.2",

package/server.js

@@ -36,7 +36,8 @@ import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
 import { detectChallengePage } from "./src/utils/challengeDetection.js";
 import { stealthDocumentVerdict } from "./src/utils/stealthVerdict.js";
-import { browserPreflight } from "./src/utils/robotsGate.js";
+import { browserPreflight, redirectGate } from "./src/utils/robotsGate.js";
+import { safeGoto } from "./src/utils/ssrfGuard.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 import AuthManager from "./src/core/AuthManager.js";
@@ -55,12 +56,12 @@ import { extractTextHandler } from "./src/tools/basic/extractText.js";
 import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
-import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
+import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
 import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
 import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
-import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
+import { markPreflightRefusal, preflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
 import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
 import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
@@ -111,7 +112,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.12.1",
+  version: "6.16.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -246,16 +247,18 @@ const deepResearchTool = new DeepResearchTool();
 const trackChangesTool = new TrackChangesTool();
 const generateLLMsTxtTool = new GenerateLLMsTxtTool();
 const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
-// Phase 3: the escalation stage for `scrape`'s escalate:true and the agent's
-// automatic stealth retry (stealth review Phase 3) — one function, so both
-// go through the same gate, engine resolver and server-level proxy list.
+// Phase 3: the escalation stage for `scrape`'s escalate:true, the agent's
+// automatic stealth retry (stealth review Phase 3) and extract_embedded_state's
+// escalate:true (plan Phase 3.2) — one function, so all three go through the
+// same gate, engine resolver and server-level proxy list.
 // Injected so the tool modules never import StealthBrowserManager (that would pull a
 // browser dependency into every unit test that loads `scrape`). Same gate
 // and same browser the stealth_mode tool drives, plus the impit TLS try, and the
 // engine name is resolved here, beside its sibling, by the one resolver
 // every stealth entry point shares.
-const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
+const stealthEscalation = async ({ url, engine, respectRobots, tool, waitFor = 0, readWindowState = false }) => {
   const warnings = await stealthComplianceGate(url, respectRobots);
+  const onRedirect = stealthRedirectGate(url, respectRobots);
   const auditIdentity = { apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken() };
   // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
   // honest User-Agent is tried before any browser launches. A caller who
@@ -263,7 +266,7 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
   if (engine === 'auto' && impitEnabled() && await loadImpit()) {
     recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
-    const page = await impitFetchPage(url);
+    const page = await impitFetchPage(url, { onRedirect });
     if (page) return { ...page, warnings };
   }
   // "auto" prefers camoufox and falls back to chromium when its binary is
@@ -275,9 +278,14 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // Audit row: the request is about to go out under a browser identity. After
   // the gate (a refusal presents none) and the resolver (the engine that runs).
   recordStealthEscalation({ url, tool, engine: resolved.engine, ...auditIdentity });
+  // waitFor and readWindowState come from extract_embedded_state only; scrape
+  // and agent pass neither, so their stage is unchanged.
   const scraped = await stealthBrowserManager.scrapeWithStealth({
     url,
-    engine: resolved.engine
+    engine: resolved.engine,
+    wait_for: waitFor,
+    readWindowState,
+    onRedirect
   });
   // The RESOLVED engine: `scrape` reports it as stealth.engine, where the
   // requested "auto" would tell the caller nothing about what ran.
@@ -404,6 +412,18 @@ const VERIFY_NUMBERS_PARAM = {
 };
 
 
+/**
+ * Whether a browser tool's result is a chain the compliance gate refused.
+ *
+ * scrape_with_actions and browser_session report a failed chain as
+ * `{ success: false, error }` rather than by throwing, so the actions that did
+ * run are still in the result. withAuth waives the charge for a refusal only
+ * on an error result, so without `isError` a caller paid in full for a chain
+ * we refused to finish: a load redirected into a disallowed path, or a click
+ * that landed on one.
+ */
+const refusedByGate = (result) => result?.success === false && preflightRefusal() !== null;
+
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
   description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
@@ -456,15 +476,16 @@ registerToolIfEnabled("extract_metadata", {
 
 // Tool: extract_embedded_state
 registerToolIfEnabled("extract_embedded_state", {
-  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree instead of the whole blob. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__, and React Server Component payloads with references between rows resolved plus a data_rows index of the rows carrying data), Nuxt (Nuxt 3's __NUXT_DATA__ decoded), SvelteKit, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), ytInitialData and ytInitialPlayerResponse, Inertia, Shopify, JSON data-* attributes and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree, keys_only:true to see the first two levels of keys before choosing one, or find:\"<key>\" to get every path where a field of that name lives, each ready to pass back as `path`. A result over max_inline_chars comes back as a preview (whole lines of the JSON), data_keys and a result_handle; read the rest with read_result json_path, e.g. \"data.next_data.props\". Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page, run the same parser and also read the globals off window (window_state: __NEXT_DATA__, __NUXT__, ytInitialData and others, read after JavaScript ran) - projected at 7, charged 2 when the plain fetch worked. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
   annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
-    url: z.string().url().describe("The URL to read embedded state from"),
-    path: z.string().optional().describe("Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. \"next_data.props.pageProps\" or \"next_f[0].f\" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them."),
+    ...EMBEDDED_STATE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...MAX_INLINE_CHARS_PARAM
   }
-}, withAuth("extract_embedded_state", extractEmbeddedStateHandler));
+}, withAuth("extract_embedded_state", createExtractEmbeddedStateHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_embedded_state' })
+})));
 
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
@@ -941,13 +962,13 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements with stable refs (@e1, @e2 ...), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
     actions: z.array(z.object({
       type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']).describe("executeJavaScript is disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; on the hosted API it is refused."),
-      selector: z.string().optional().describe("A CSS selector, or a @e1 ref from an earlier snapshot action in this chain"),
+      selector: z.string().optional().describe("A CSS selector, or a ref from an earlier snapshot action in this chain (@e1, including elements inside shadow roots and iframes)"),
       text: z.string().optional(),
       key: z.string().optional(),
       script: z.string().optional(),
@@ -1014,7 +1035,8 @@ registerToolIfEnabled("scrape_with_actions", {
       viewportHeight: z.number().min(600).max(1080).default(720),
       timeout: z.number().min(10000).max(120000).default(30000),
       stealth: z.boolean().default(false).describe("Run the action chain in the stealth browser (randomized fingerprint, WebRTC/canvas spoofing) instead of the standard browser pool. Renders JavaScript; it does not solve challenges."),
-      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium.")
+      engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium."),
+      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged.")
     }).optional().describe("Browser configuration options"),
     extractionOptions: z.object({
       selectors: z.record(z.string()).optional(),
@@ -1043,7 +1065,10 @@ registerToolIfEnabled("scrape_with_actions", {
     // array (Phase 0, 0.4).
     stripScreenshotData(result, (actionId, data) => resourceRegistry.storeScreenshot(actionId, data));
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Scrape with actions failed: ${error.message}` }], isError: true };
   }
@@ -1051,7 +1076,7 @@ registerToolIfEnabled("scrape_with_actions", {
 
 // Tool: browser_session
 registerToolIfEnabled("browser_session", {
-  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ...), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
+  description: "Use this to drive a browser across several calls, keeping the page, its cookies and its login in between. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2 ..., including elements inside open shadow roots and iframes), act on those refs, read the content, close. Because the page stays open you can look before each step instead of committing to a whole chain up front, so a wrong selector costs one call rather than all of them. Operations: open (url, stealth, engine, ttl, activity_ttl, viewport), snapshot, act (the same action array as scrape_with_actions), read (formats), screenshot, close, list. Navigation invalidates refs, so snapshot again after one. robots.txt is respected on every navigation, and screenshots are stored as crawlforge://screenshot/{actionId} resources. A session expires 600s after it opens or 300s after its last use, whichever comes first, so close it when you are done. Not for a page that renders without interaction (scrape), and not for an interaction you can write out in advance - that is one scrape_with_actions call for 5. Cost: 3 credits to open; read 2; snapshot, act, screenshot, close and list 1 each. Example: browser_session({operation:\"open\", url:\"https://app.com/login\"}), then browser_session({operation:\"snapshot\", session_id:\"...\"})",
   annotations: { title: "Browser Session", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     operation: z.enum(["open", "snapshot", "act", "read", "screenshot", "close", "list"]).describe("open a session, observe it, act on it, read it, or close it"),
@@ -1109,7 +1134,10 @@ registerToolIfEnabled("browser_session", {
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Browser session failed: ${error.message}` }], isError: true };
   }
@@ -1316,6 +1344,17 @@ async function stealthComplianceGate(url, respectRobots) {
   return browserPreflight(url, { respectRobots, tool: 'stealth_mode' });
 }
 
+/**
+ * The same gate for wherever a navigation to `url` is redirected to. The
+ * pre-flight above decides about the URL the caller named; a redirect can lead
+ * anywhere, so each stealth entry point hands this to its navigation.
+ *
+ * @returns {(to: string) => Promise<void>} refuses a hop by throwing
+ */
+function stealthRedirectGate(url, respectRobots) {
+  return redirectGate(url, { respectRobots, tool: 'stealth_mode' });
+}
+
 /**
  * Build the requested formats from one stealth render. The browser already
  * returned the rendered HTML and visible text, so nothing here refetches;
@@ -1440,7 +1479,8 @@ registerToolIfEnabled("stealth_mode", {
           engine: resolvedEngine.engine,
           wait_for: wait_for || 0,
           screenshot: wantsScreenshot,
-          stealthConfig
+          stealthConfig,
+          onRedirect: stealthRedirectGate(url, respect_robots)
         });
 
         // A bot-wall interstitial arrives as HTTP 200 with a title and prose
@@ -1531,7 +1571,13 @@ registerToolIfEnabled("stealth_mode", {
             // JSON-serializable — extract just the useful navigation details.
             // Explicit timeout keeps navigation inside every caller's window
             // (Playwright's default is 30s, longer than some proxy budgets).
-            const response = await page.goto(urlToTest, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            // safeGoto, not page.goto: the SSRF guard on the URL and on where
+            // it lands, and the gate above again for every redirect on the way.
+            const response = await safeGoto(page, urlToTest, {
+              waitUntil: 'domcontentloaded',
+              timeout: 20000,
+              onRedirect: stealthRedirectGate(urlToTest, respect_robots)
+            });
             navigation = {
               requestedUrl: urlToTest,
               finalUrl: page.url(),

package/src/cli/commands/stealth.js

@@ -2,7 +2,7 @@
  * stealth command — scrape a URL using stealth mode.
  */
 import { StealthBrowserManager } from '../../core/StealthBrowserManager.js';
-import { browserPreflight } from '../../utils/robotsGate.js';
+import { browserPreflight, redirectGate } from '../../utils/robotsGate.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
 
@@ -22,7 +22,7 @@ export function register(program) {
         // a different door, not by a different set of rules.
         execute: async (p) => {
           await browserPreflight(p.url, { tool: 'stealth_mode' });
-          return mgr.scrapeWithStealth(p);
+          return mgr.scrapeWithStealth({ ...p, onRedirect: redirectGate(p.url, { tool: 'stealth_mode' }) });
         }
       };
       await runTool(wrapperTool, {