crawlforge-mcp-server 6.14.0 → 6.17.1

package/README.md

@@ -187,7 +187,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `read_result` | 1 | Search, slice, read lines or a JSON path from a result a tool returned with `truncated: true` and a `result_handle` (kept 1 hour on your own machine) — never fetch the page again |
 | `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings. `escalate:true` retries a blocked plain fetch once in the stealth browser and returns the page instead of the block (+5 as the projected ceiling; the charge drops back to 2 when the plain fetch succeeded and no escalation ran) |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
-| `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
+| `extract_embedded_state` | 2 (7 projected with `escalate: true`, charged 2 when the plain fetch works) | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads (references between rows resolved, plus a `data_rows` index), Nuxt (Nuxt 3's `__NUXT_DATA__` decoded), SvelteKit, Apollo, Redux, `ytInitialData`, Inertia, Shopify, JSON `data-*` attributes, `<script type="application/json">` — with a `path` to scope the result, `keys_only: true` to see its first two levels of keys, or `find: "<key>"` to get every path where a field of that name lives. `escalate: true` re-reads a walled page in the stealth browser, runs the same parser and also reads the framework globals off `window` (`window_state`). No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |
 | `map_site` | 2 | Discover and map website structure (optional `search=` ranks the discovered URLs) |
 | `process_document` | 2 | Multi-format document processing |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.14.0",
+  "version": "6.17.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -118,17 +118,17 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@mozilla/readability": "^0.6.0",
-    "mammoth": "^1.12.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.10.0",
+    "crawlforge-extractors": "^1.12.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
     "isomorphic-dompurify": "^3.9.0",
     "jsdom": "^29.0.2",
     "lru-cache": "^11.1.0",
+    "mammoth": "^1.12.2",
     "p-queue": "^9.3.3",
     "pdf-parse": "2.4.5",
     "playwright": "^1.54.2",

package/server.js

@@ -29,14 +29,15 @@ import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
 import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
 import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
 import { AgentTool } from "./src/tools/agent/agent.js"; // D4 D2
-import { StealthBrowserManager, resolveStealthEngine } from "./src/core/StealthBrowserManager.js";
+import { StealthBrowserManager, resolveStealthEngine, assertProxyEngineAllowed } from "./src/core/StealthBrowserManager.js";
 import { LocalizationManager } from "./src/core/LocalizationManager.js";
 // Stealth scrape: format conversion + the pre-fetch compliance gate (G5/G6/G7)
 import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
 import { detectChallengePage } from "./src/utils/challengeDetection.js";
 import { stealthDocumentVerdict } from "./src/utils/stealthVerdict.js";
-import { browserPreflight } from "./src/utils/robotsGate.js";
+import { browserPreflight, redirectGate } from "./src/utils/robotsGate.js";
+import { safeGoto } from "./src/utils/ssrfGuard.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 import AuthManager from "./src/core/AuthManager.js";
@@ -55,12 +56,12 @@ import { extractTextHandler } from "./src/tools/basic/extractText.js";
 import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
-import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
+import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
 import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
 import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
-import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
+import { markPreflightRefusal, preflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
 import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
 import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
@@ -111,7 +112,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.14.0",
+  version: "6.17.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -246,16 +247,18 @@ const deepResearchTool = new DeepResearchTool();
 const trackChangesTool = new TrackChangesTool();
 const generateLLMsTxtTool = new GenerateLLMsTxtTool();
 const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
-// Phase 3: the escalation stage for `scrape`'s escalate:true and the agent's
-// automatic stealth retry (stealth review Phase 3) — one function, so both
-// go through the same gate, engine resolver and server-level proxy list.
+// Phase 3: the escalation stage for `scrape`'s escalate:true, the agent's
+// automatic stealth retry (stealth review Phase 3) and extract_embedded_state's
+// escalate:true (plan Phase 3.2) — one function, so all three go through the
+// same gate, engine resolver and server-level proxy list.
 // Injected so the tool modules never import StealthBrowserManager (that would pull a
 // browser dependency into every unit test that loads `scrape`). Same gate
 // and same browser the stealth_mode tool drives, plus the impit TLS try, and the
 // engine name is resolved here, beside its sibling, by the one resolver
 // every stealth entry point shares.
-const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
+const stealthEscalation = async ({ url, engine, respectRobots, tool, waitFor = 0, readWindowState = false }) => {
   const warnings = await stealthComplianceGate(url, respectRobots);
+  const onRedirect = stealthRedirectGate(url, respectRobots);
   const auditIdentity = { apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken() };
   // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
   // honest User-Agent is tried before any browser launches. A caller who
@@ -263,7 +266,7 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
   if (engine === 'auto' && impitEnabled() && await loadImpit()) {
     recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
-    const page = await impitFetchPage(url);
+    const page = await impitFetchPage(url, { onRedirect });
     if (page) return { ...page, warnings };
   }
   // "auto" prefers camoufox and falls back to chromium when its binary is
@@ -275,9 +278,14 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // Audit row: the request is about to go out under a browser identity. After
   // the gate (a refusal presents none) and the resolver (the engine that runs).
   recordStealthEscalation({ url, tool, engine: resolved.engine, ...auditIdentity });
+  // waitFor and readWindowState come from extract_embedded_state only; scrape
+  // and agent pass neither, so their stage is unchanged.
   const scraped = await stealthBrowserManager.scrapeWithStealth({
     url,
-    engine: resolved.engine
+    engine: resolved.engine,
+    wait_for: waitFor,
+    readWindowState,
+    onRedirect
   });
   // The RESOLVED engine: `scrape` reports it as stealth.engine, where the
   // requested "auto" would tell the caller nothing about what ran.
@@ -404,6 +412,18 @@ const VERIFY_NUMBERS_PARAM = {
 };
 
 
+/**
+ * Whether a browser tool's result is a chain the compliance gate refused.
+ *
+ * scrape_with_actions and browser_session report a failed chain as
+ * `{ success: false, error }` rather than by throwing, so the actions that did
+ * run are still in the result. withAuth waives the charge for a refusal only
+ * on an error result, so without `isError` a caller paid in full for a chain
+ * we refused to finish: a load redirected into a disallowed path, or a click
+ * that landed on one.
+ */
+const refusedByGate = (result) => result?.success === false && preflightRefusal() !== null;
+
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
   description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
@@ -456,15 +476,16 @@ registerToolIfEnabled("extract_metadata", {
 
 // Tool: extract_embedded_state
 registerToolIfEnabled("extract_embedded_state", {
-  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree instead of the whole blob. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__, and React Server Component payloads with references between rows resolved plus a data_rows index of the rows carrying data), Nuxt (Nuxt 3's __NUXT_DATA__ decoded), SvelteKit, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), ytInitialData and ytInitialPlayerResponse, Inertia, Shopify, JSON data-* attributes and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree, keys_only:true to see the first two levels of keys before choosing one, or find:\"<key>\" to get every path where a field of that name lives, each ready to pass back as `path`. A result over max_inline_chars comes back as a preview (whole lines of the JSON), data_keys and a result_handle; read the rest with read_result json_path, e.g. \"data.next_data.props\". Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page, run the same parser and also read the globals off window (window_state: __NEXT_DATA__, __NUXT__, ytInitialData and others, read after JavaScript ran) - projected at 7, charged 2 when the plain fetch worked. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
   annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
-    url: z.string().url().describe("The URL to read embedded state from"),
-    path: z.string().optional().describe("Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. \"next_data.props.pageProps\" or \"next_f[0].f\" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them."),
+    ...EMBEDDED_STATE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...MAX_INLINE_CHARS_PARAM
   }
-}, withAuth("extract_embedded_state", extractEmbeddedStateHandler));
+}, withAuth("extract_embedded_state", createExtractEmbeddedStateHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_embedded_state' })
+})));
 
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
@@ -941,7 +962,7 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation, and each navigation is checked: a bot wall is reported as `blocked` with the vendor named, per navigation in `navigations` and for the final page at the top level. browserOptions.proxyRotation routes a stealth chain through your own proxies. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
@@ -1015,7 +1036,12 @@ registerToolIfEnabled("scrape_with_actions", {
       timeout: z.number().min(10000).max(120000).default(30000),
       stealth: z.boolean().default(false).describe("Run the action chain in the stealth browser (randomized fingerprint, WebRTC/canvas spoofing) instead of the standard browser pool. Renders JavaScript; it does not solve challenges."),
       engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium."),
-      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged.")
+      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged."),
+      proxyRotation: z.object({
+        enabled: z.boolean().default(false),
+        proxies: z.array(z.string()).optional(),
+        rotationInterval: z.number().default(300000)
+      }).optional().describe("Route a stealth chain through your own proxies; requires stealth:true and the Chromium engine, and is refused without stealth or on camoufox (including \"auto\" when it resolves to camoufox), which shares one launch-time proxy across calls. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. A proxy on a loopback, link-local or cloud-metadata address is refused, and credentials are removed from the result. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. CrawlForge supplies no proxies.")
     }).optional().describe("Browser configuration options"),
     extractionOptions: z.object({
       selectors: z.record(z.string()).optional(),
@@ -1044,7 +1070,10 @@ registerToolIfEnabled("scrape_with_actions", {
     // array (Phase 0, 0.4).
     stripScreenshotData(result, (actionId, data) => resourceRegistry.storeScreenshot(actionId, data));
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Scrape with actions failed: ${error.message}` }], isError: true };
   }
@@ -1110,7 +1139,10 @@ registerToolIfEnabled("browser_session", {
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);
 
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+      ...(refusedByGate(result) ? { isError: true } : {})
+    };
   } catch (error) {
     return { content: [{ type: "text", text: `Browser session failed: ${error.message}` }], isError: true };
   }
@@ -1317,6 +1349,17 @@ async function stealthComplianceGate(url, respectRobots) {
   return browserPreflight(url, { respectRobots, tool: 'stealth_mode' });
 }
 
+/**
+ * The same gate for wherever a navigation to `url` is redirected to. The
+ * pre-flight above decides about the URL the caller named; a redirect can lead
+ * anywhere, so each stealth entry point hands this to its navigation.
+ *
+ * @returns {(to: string) => Promise<void>} refuses a hop by throwing
+ */
+function stealthRedirectGate(url, respectRobots) {
+  return redirectGate(url, { respectRobots, tool: 'stealth_mode' });
+}
+
 /**
  * Build the requested formats from one stealth render. The browser already
  * returned the rendered HTML and visible text, so nothing here refetches;
@@ -1388,7 +1431,7 @@ registerToolIfEnabled("stealth_mode", {
         enabled: z.boolean().default(false),
         proxies: z.array(z.string()).optional(),
         rotationInterval: z.number().default(300000)
-      }).optional().describe("Route the browser through your own proxies. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. CrawlForge supplies no proxies."),
+      }).optional().describe("Route the browser through your own proxies. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. Requires the Chromium engine: refused on camoufox (including \"auto\" when it resolves to camoufox), which shares one launch-time proxy across calls. CrawlForge supplies no proxies."),
       antiDetection: z.object({
         cloudflareBypass: z.boolean().default(true),
         recaptchaHandling: z.boolean().default(true),
@@ -1428,6 +1471,7 @@ registerToolIfEnabled("stealth_mode", {
         // is missing; "playwright" is this tool's old public name for chromium.
         // The resolver owns both, and reports a fallback the caller can see.
         const resolvedEngine = await resolveStealthEngine(engine);
+        assertProxyEngineAllowed(resolvedEngine.engine, stealthConfig);
         if (resolvedEngine.fallbackWarning) warnings.push(resolvedEngine.fallbackWarning);
 
         recordStealthEscalation({
@@ -1441,7 +1485,8 @@ registerToolIfEnabled("stealth_mode", {
           engine: resolvedEngine.engine,
           wait_for: wait_for || 0,
           screenshot: wantsScreenshot,
-          stealthConfig
+          stealthConfig,
+          onRedirect: stealthRedirectGate(url, respect_robots)
         });
 
         // A bot-wall interstitial arrives as HTTP 200 with a title and prose
@@ -1494,6 +1539,7 @@ registerToolIfEnabled("stealth_mode", {
         // without it, create_context always ran on chromium whatever the caller
         // asked for. Resolved, so "auto" reaches the manager as a real engine.
         const contextEngine = await resolveStealthEngine(engine);
+        assertProxyEngineAllowed(contextEngine.engine, stealthConfig);
         const contextData = await stealthBrowserManager.createStealthContext({
           ...(stealthConfig || {}),
           engine: contextEngine.engine
@@ -1532,7 +1578,13 @@ registerToolIfEnabled("stealth_mode", {
             // JSON-serializable — extract just the useful navigation details.
             // Explicit timeout keeps navigation inside every caller's window
             // (Playwright's default is 30s, longer than some proxy budgets).
-            const response = await page.goto(urlToTest, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            // safeGoto, not page.goto: the SSRF guard on the URL and on where
+            // it lands, and the gate above again for every redirect on the way.
+            const response = await safeGoto(page, urlToTest, {
+              waitUntil: 'domcontentloaded',
+              timeout: 20000,
+              onRedirect: stealthRedirectGate(urlToTest, respect_robots)
+            });
             navigation = {
               requestedUrl: urlToTest,
               finalUrl: page.url(),

package/src/cli/commands/stealth.js

@@ -2,7 +2,7 @@
  * stealth command — scrape a URL using stealth mode.
  */
 import { StealthBrowserManager } from '../../core/StealthBrowserManager.js';
-import { browserPreflight } from '../../utils/robotsGate.js';
+import { browserPreflight, redirectGate } from '../../utils/robotsGate.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
 
@@ -22,7 +22,7 @@ export function register(program) {
         // a different door, not by a different set of rules.
         execute: async (p) => {
           await browserPreflight(p.url, { tool: 'stealth_mode' });
-          return mgr.scrapeWithStealth(p);
+          return mgr.scrapeWithStealth({ ...p, onRedirect: redirectGate(p.url, { tool: 'stealth_mode' }) });
         }
       };
       await runTool(wrapperTool, {