crawlforge-mcp-server 6.16.0 → 6.18.0

package/README.md

@@ -178,8 +178,8 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | Tool | Credits | What it does |
 |------|---------|--------------|
 | `fetch_url` | 1 | Fetch content from any URL |
-| `extract_text` | 1 | Extract clean text from web pages |
-| `extract_links` | 1 | Get all links from a page |
+| `extract_text` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Extract clean text from web pages |
+| `extract_links` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Get all links from a page |
 | `extract_metadata` | 1 | Extract page metadata (title, OG tags, schema.org) |
 | `scrape_template` | 1 | Structured data from well-known sites (Amazon, GitHub, LinkedIn, YouTube, Reddit, Hacker News, npm, and more) without writing selectors |
 | `list_ollama_models` | 1 | List the Ollama models installed locally (helps you pick a `model` for `extract_with_llm`) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.16.0",
+  "version": "6.18.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.11.0",
+    "crawlforge-extractors": "^1.13.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -29,7 +29,7 @@ import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
 import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
 import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
 import { AgentTool } from "./src/tools/agent/agent.js"; // D4 D2
-import { StealthBrowserManager, resolveStealthEngine } from "./src/core/StealthBrowserManager.js";
+import { StealthBrowserManager, resolveStealthEngine, assertProxyEngineAllowed } from "./src/core/StealthBrowserManager.js";
 import { LocalizationManager } from "./src/core/LocalizationManager.js";
 // Stealth scrape: format conversion + the pre-fetch compliance gate (G5/G6/G7)
 import * as cheerio from "cheerio";
@@ -52,8 +52,9 @@ import { createOAuthProvider } from "./src/server/auth/oauth.js";
 import { createMetricsRegistry } from "./src/observability/metrics.js";
 // Basic tool handlers (extracted from server.js)
 import { fetchUrlHandler } from "./src/tools/basic/fetchUrl.js";
-import { extractTextHandler } from "./src/tools/basic/extractText.js";
-import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { createExtractTextHandler } from "./src/tools/basic/extractText.js";
+import { createExtractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { EXTRACT_ESCALATION_SHAPE } from "./src/utils/fetchLadder.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
 import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
@@ -112,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.16.0",
+  version: "6.18.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -439,29 +440,37 @@ registerToolIfEnabled("fetch_url", {
 
 // Tool: extract_text
 registerToolIfEnabled("extract_text", {
-  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
+  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG; selector reads only the matched elements, max_length caps the returned text. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
   annotations: { title: "Extract Text", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract text from"),
     remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
     remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
     output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
+    selector: z.string().optional().describe("CSS selector: read only the matched elements (nav/header/footer are then kept). No match is an error"),
+    max_length: z.number().int().min(1).max(1000000).optional().describe("Maximum characters of text or markdown to return; a longer result is cut and ends with \"...\""),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...REDACT_PII_PARAM
   }
-}, withAuth("extract_text", extractTextHandler));
+}, withAuth("extract_text", createExtractTextHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_text' })
+})));
 
 // Tool: extract_links
 registerToolIfEnabled("extract_links", {
-  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true returns only outbound links. Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
+  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true drops internal links; mailto:, tel: and javascript: links are type \"other\". Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
   annotations: { title: "Extract Links", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract links from"),
-    filter_external: z.boolean().optional().default(false).describe("Only return external links"),
+    filter_external: z.boolean().optional().default(false).describe("Drop internal (same-host) links"),
     base_url: z.string().url().optional().describe("Base URL for resolving relative links"),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("extract_links", extractLinksHandler));
+}, withAuth("extract_links", createExtractLinksHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_links' })
+})));
 
 // Tool: extract_metadata
 registerToolIfEnabled("extract_metadata", {
@@ -962,7 +971,7 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements, including those in open shadow roots and iframes, with stable refs (@e1, @e2 ... in document order), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.consent:\"reject\" (or \"accept\") to answer a cookie/consent banner before the first action; it is off by default. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation, and each navigation is checked: a bot wall is reported as `blocked` with the vendor named, per navigation in `navigations` and for the final page at the top level. browserOptions.proxyRotation routes a stealth chain through your own proxies. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
@@ -1036,7 +1045,12 @@ registerToolIfEnabled("scrape_with_actions", {
       timeout: z.number().min(10000).max(120000).default(30000),
       stealth: z.boolean().default(false).describe("Run the action chain in the stealth browser (randomized fingerprint, WebRTC/canvas spoofing) instead of the standard browser pool. Renders JavaScript; it does not solve challenges."),
       engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).default("auto").describe("Stealth engine for the chain, with stealth:true. \"auto\" (default) runs camoufox when it is installed and Chromium otherwise; the result's `engine` says which ran. \"camoufox\" is Firefox-based with a higher anti-detect score; \"chromium\" (= \"playwright\") forces Chromium. Refused without stealth:true, where the browser is always Chromium."),
-      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged.")
+      consent: z.enum(["off", "reject", "accept"]).default("off").describe("Cookie/consent banner handling, using DuckDuckGo autoconsent's rules for known consent platforms (OneTrust, Sourcepoint, Didomi ...). \"reject\" declines non-essential cookies and \"accept\" accepts them, once after the initial page load and again after each navigate action, for at most 2s each; the result's `consent` ({cmp, action, ms}) and each navigate result say what was found and done. A banner no rule matches is left in place (action \"none\") and never fails the chain. Default \"off\": the page is left as it loads, so a chain that clicks the banner itself works unchanged."),
+      proxyRotation: z.object({
+        enabled: z.boolean().default(false),
+        proxies: z.array(z.string()).optional(),
+        rotationInterval: z.number().default(300000)
+      }).optional().describe("Route a stealth chain through your own proxies; requires stealth:true and the Chromium engine, and is refused without stealth or on camoufox (including \"auto\" when it resolves to camoufox), which shares one launch-time proxy across calls. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. A proxy on a loopback, link-local or cloud-metadata address is refused, and credentials are removed from the result. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. CrawlForge supplies no proxies.")
     }).optional().describe("Browser configuration options"),
     extractionOptions: z.object({
       selectors: z.record(z.string()).optional(),
@@ -1426,7 +1440,7 @@ registerToolIfEnabled("stealth_mode", {
         enabled: z.boolean().default(false),
         proxies: z.array(z.string()).optional(),
         rotationInterval: z.number().default(300000)
-      }).optional().describe("Route the browser through your own proxies. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. CrawlForge supplies no proxies."),
+      }).optional().describe("Route the browser through your own proxies. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. Requires the Chromium engine: refused on camoufox (including \"auto\" when it resolves to camoufox), which shares one launch-time proxy across calls. CrawlForge supplies no proxies."),
       antiDetection: z.object({
         cloudflareBypass: z.boolean().default(true),
         recaptchaHandling: z.boolean().default(true),
@@ -1466,6 +1480,7 @@ registerToolIfEnabled("stealth_mode", {
         // is missing; "playwright" is this tool's old public name for chromium.
         // The resolver owns both, and reports a fallback the caller can see.
         const resolvedEngine = await resolveStealthEngine(engine);
+        assertProxyEngineAllowed(resolvedEngine.engine, stealthConfig);
         if (resolvedEngine.fallbackWarning) warnings.push(resolvedEngine.fallbackWarning);
 
         recordStealthEscalation({
@@ -1533,6 +1548,7 @@ registerToolIfEnabled("stealth_mode", {
         // without it, create_context always ran on chromium whatever the caller
         // asked for. Resolved, so "auto" reaches the manager as a real engine.
         const contextEngine = await resolveStealthEngine(engine);
+        assertProxyEngineAllowed(contextEngine.engine, stealthConfig);
         const contextData = await stealthBrowserManager.createStealthContext({
           ...(stealthConfig || {}),
           engine: contextEngine.engine

package/src/core/ActionExecutor.js

@@ -12,6 +12,7 @@ import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode } from '.
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
 import { settlePage } from './browser/settle.js';
 import { handleConsent } from './browser/consent.js';
+import { stealthDocumentVerdict } from '../utils/stealthVerdict.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -31,6 +32,13 @@ const ACTION_TIMEOUT_GRACE_MS = 2000;
 // to work cost several times its stated timeout.
 const RECOVERY_TIMEOUT_MS = 3000;
 
+// How long a navigation whose document reads as a wall is watched for the wall
+// replacing itself before it is reported, re-read every NAVIGATION_WALL_POLL_MS.
+// AWS WAF's interstitial reloads into amazon.com's homepage about 0.4 s after
+// domcontentloaded (measured 2026-10-03); nothing here acts on the page.
+const NAVIGATION_WALL_GRACE_MS = 3000;
+const NAVIGATION_WALL_POLL_MS = 500;
+
 // The only states locator.waitFor()/page.waitForSelector() accept. The rest of
 // the wait-action enum (enabled/disabled/stable) are ElementHandle states and
 // have to go through waitForElementState instead — passing them here is
@@ -312,6 +320,10 @@ export class ActionExecutor extends EventEmitter {
           executionContext.consent = await handleConsent(page, browserOptions.consent);
         }
 
+        // navigations[0] is the initial load; each `navigate` action appends
+        // its own (executeNavigateAction).
+        executionContext.navigations = [await this.checkNavigation(page, url, browserOptions)];
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
 
@@ -419,6 +431,7 @@ export class ActionExecutor extends EventEmitter {
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
+        navigations: executionContext.navigations || [],
         consent: executionContext.consent,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
@@ -446,6 +459,7 @@ export class ActionExecutor extends EventEmitter {
         // results, the error screenshot, and any intermediate-state
         // captures) instead of discarding them.
         results: executionContext?.results || [],
+        navigations: executionContext?.navigations || [],
         attempt: executionContext?.attempt ?? 0,
         attempts: executionContext?.attempts || [],
         screenshots: executionContext?.screenshots || [],
@@ -478,6 +492,11 @@ export class ActionExecutor extends EventEmitter {
           await this.navigateToUrl(page, executionContext.url, {
             browserOptions: executionContext.browserOptions
           });
+          // Only the last attempt's navigations are reported, so the reload
+          // that starts this one is navigations[0].
+          executionContext.navigations = [
+            await this.checkNavigation(page, executionContext.url, executionContext.browserOptions)
+          ];
         }
 
         // Execute actions in sequence
@@ -1346,14 +1365,73 @@ export class ActionExecutor extends EventEmitter {
       ? await handleConsent(page, consentMode, { timeout: Math.min(2000, timeout - (Date.now() - startedAt)) })
       : undefined;
 
+    // Reporting only: the navigation happened, so the action succeeds whether
+    // or not the document it reached is a wall. The wait for a wall to clear
+    // gets what is left of the deadline, like consent above.
+    const navigation = await this.checkNavigation(page, action.url, executionContext?.browserOptions, {
+      graceMs: Math.min(NAVIGATION_WALL_GRACE_MS, timeout - (Date.now() - startedAt))
+    });
+    executionContext?.navigations?.push(navigation);
+
     return {
       url: action.url,
-      finalUrl: page.url(),
+      finalUrl: navigation.finalUrl,
       waitUntil: action.waitUntil || 'domcontentloaded',
+      httpStatus: navigation.httpStatus,
+      ...(navigation.blocked ? { blocked: navigation.blocked } : {}),
       ...(consent ? { consent } : {})
     };
   }
 
+  /**
+   * What the navigation that just finished reached: the page, or a wall. The
+   * same verdict the chain's final document gets, run on this document with
+   * the status of this navigation. A document that reads as a wall is re-read
+   * for up to `graceMs` first, because some walls replace themselves (AWS WAF's
+   * interstitial reloads into the page); one that cannot be read at all is
+   * reported without a verdict rather than as a wall.
+   * @param {Page} page - Playwright page, just navigated
+   * @param {string} url - the URL the navigation was sent to
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` for the read's gate)
+   * @param {{ graceMs?: number }} [options]
+   * @returns {Promise<{ url: string, finalUrl: string, httpStatus: number|null, blocked?: { vendor: string, evidence: string } }>}
+   */
+  async checkNavigation(page, url, browserOptions, { graceMs = NAVIGATION_WALL_GRACE_MS } = {}) {
+    const httpStatus = page.__crawlforgeNavigation?.status ?? null;
+    const read = () => this.readGated(page, browserOptions, async () => {
+      try {
+        const [title, html, text] = await Promise.all([
+          page.title(),
+          page.content(),
+          page.evaluate(() => (document.body ? document.body.innerText : ''))
+        ]);
+        return stealthDocumentVerdict(
+          { url: page.url(), title, html, text, status: httpStatus },
+          { allowEmpty: true, fetcher: browserOptions?.stealthMode?.enabled ? 'the stealth browser' : 'the browser' }
+        );
+      } catch {
+        return null; // mid-navigation, or a page that cannot be read
+      }
+    });
+
+    let verdict = await read();
+    const deadline = Date.now() + graceMs;
+    while (verdict?.blocked && Date.now() + NAVIGATION_WALL_POLL_MS <= deadline) {
+      await this.delay(NAVIGATION_WALL_POLL_MS);
+      const next = await read();
+      // A read that fails mid-reload says nothing yet; keep the wall until a
+      // document answers.
+      if (next) verdict = next;
+    }
+
+    return {
+      url,
+      finalUrl: page.url(),
+      httpStatus,
+      ...(verdict?.blocked ? { blocked: verdict.blocked } : {})
+    };
+  }
+
   /**
    * Execute screenshot action
    * @param {Page} page - Playwright page
@@ -1503,9 +1581,12 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
+    // The initial load and a retry's reload pass no timeout of their own, so
+    // browserOptions.timeout bounds them as it bounds the actions (0.2); a
+    // navigate action's own timeout comes first.
     const response = await page.goto(url, {
       waitUntil: options.waitUntil || 'domcontentloaded',
-      timeout: options.timeout || 30000
+      timeout: options.timeout || options.browserOptions?.timeout || 30000
     });
 
     // Re-validate where the navigation went: a redirect could have taken us

package/src/core/AuthManager.js

@@ -687,6 +687,12 @@ class AuthManager {
       return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_text and extract_links (fix plan Phase E2) run the same ladder:
+    // projected 1+5 when escalating, charged 1 when the plain fetch sufficed.
+    if (tool === 'extract_text' || tool === 'extract_links') {
+      return costs[tool] + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -778,6 +784,12 @@ class AuthManager {
           ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
           : 'Fixed cost per invocation.';
         break;
+      case 'extract_text':
+      case 'extract_links':
+        note = params?.escalate === true
+          ? 'Base 1; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/core/StealthBrowserManager.js

@@ -17,7 +17,7 @@ import path from 'path';
 import { createRequire } from 'module';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
-import { safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed, assertUrlAllowed } from '../utils/ssrfGuard.js';
 import { looksLikeInterstitial, detectChallengePage } from '../utils/challengeDetection.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
@@ -159,6 +159,39 @@ function isChromium(page) {
   }
 }
 
+/**
+ * A proxy entry with its credentials removed. Everything before the last "@"
+ * of the authority goes, with or without a scheme: parseProxyEntry accepts a
+ * bare `user:pass@host:port`, which a rule anchored on "//" left whole.
+ * @param {string} entry
+ * @returns {string}
+ */
+export function redactProxyEntry(entry) {
+  return String(entry).replace(/^([a-z][a-z0-9+.-]*:\/\/)?[^/]*@/i, '$1');
+}
+
+/**
+ * Refuse a per-call proxy list on camoufox. camoufox takes its proxy at
+ * browser launch and that one browser serves every later camoufox call, so a
+ * caller's proxy would carry other callers' traffic, on the caller's
+ * credentials, or be ignored when a browser is already up. Chromium sets the
+ * proxy per context. Called with the engine that will actually run, so an
+ * "auto" that lands on camoufox is refused too. The operator's
+ * CRAWLFORGE_STEALTH_PROXIES is not a per-call list and is not refused.
+ * @param {string} engine - the resolved engine
+ * @param {{ proxyRotation?: { enabled?: boolean, proxies?: string[] } }} config
+ * @throws {Error}
+ */
+export function assertProxyEngineAllowed(engine, config) {
+  const requested = config?.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
+  if (engine !== 'camoufox' || !requested.length) return;
+  throw new Error(
+    'proxyRotation is refused on the camoufox engine: camoufox applies its proxy at browser launch ' +
+    'and shares that browser across calls, so a per-call proxy cannot be kept to this call. Use ' +
+    'engine:"chromium" (proxies are per context there), or set the operator list CRAWLFORGE_STEALTH_PROXIES.'
+  );
+}
+
 /**
  * Resolve a requested engine to the one that will actually launch.
  *
@@ -422,12 +455,19 @@ export class StealthBrowserManager {
    */
   async launchStealthBrowser(config = {}) {
     const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
+    // Every path to a browser comes through here, createStealthContext's
+    // included, so this is where a caller's proxy list is held to the SSRF
+    // guard before either engine is handed it.
+    await this.assertCallerProxiesAllowed(validatedConfig);
     // 'auto' has to become a concrete engine before the comparison below: an
     // unresolved 'auto' never equals _launchedEngine, so every call would park
     // a perfectly good browser and launch a second one beside it. This is the
     // call that decides the engine — every other path reaches a browser
     // through it — so it is the one that records the outcome.
     await this._resolveConfigEngine(validatedConfig, true);
+    // The tools refuse this up front; this is the backstop for every other
+    // path, decided on the engine 'auto' resolved to, before any launch.
+    assertProxyEngineAllowed(validatedConfig.engine, validatedConfig);
 
     // A Chromium that was OOM-killed or crashed doesn't error on reuse — its
     // protocol calls hang. Detect the corpse and relaunch instead.
@@ -528,7 +568,11 @@ export class StealthBrowserManager {
       // leave camoufox reporting the first proxy's city behind the second
       // proxy's exit IP, which is a worse signal than not rotating at all.
       // A rotation takes effect on the next launch, after cleanup().
-      const proxy = this.resolveProxy(validatedConfig);
+      //
+      // Only the operator's CRAWLFORGE_STEALTH_PROXIES, never a caller's list:
+      // this browser outlives the call, and every later camoufox context
+      // inherits its proxy (assertProxyEngineAllowed refuses the caller's).
+      const proxy = this.resolveProxy({ ...validatedConfig, proxyRotation: undefined });
       // The caller's locale goes to the launcher, not to the context. camoufox
       // sets language, Accept-Language and Intl together below the JS layer,
       // where a Worker reads the same answer as the document; Playwright's
@@ -2387,18 +2431,23 @@ export class StealthBrowserManager {
       // Wait for potential challenge page
       await page.waitForTimeout(2000);
       
-      // Check for CloudFlare challenge indicators
+      // Check for CloudFlare challenge indicators. Cloudflare's current
+      // interstitial reads "Performing security verification" under the title
+      // "Just a moment..." (doordash.com, 2026-10-03); none of the older
+      // phrases below appear on it, so the wait further down ended at once.
       const challengeDetected = await page.evaluate(() => {
         const indicators = [
           'cf-browser-verification',
           'cf-challenge-running',
           'Checking your browser',
+          'Performing security verification',
           'DDoS protection by Cloudflare',
           'Ray ID'
         ];
         
         const pageText = document.body.innerText;
-        return indicators.some(indicator => pageText.includes(indicator));
+        return /^just a moment/i.test(document.title.trim()) ||
+          indicators.some(indicator => pageText.includes(indicator));
       });
       
       if (challengeDetected) {
@@ -2420,16 +2469,25 @@ export class StealthBrowserManager {
           }
         }
         
-        // Wait for challenge to complete (up to 30 seconds)
+        // Wait for the challenge to complete, up to 10 s. Only waits for
+        // Cloudflare's own script to finish; nothing is clicked. A challenge
+        // that passes clears in seconds; one that will not pass (an
+        // interactive check, or this IP's reputation) never clears, and 30 s
+        // there put a walled chain past the REST window (doordash.com stealth
+        // Chromium, 3 of 3 runs walled at 39 s with the 30 s cap, 2026-10-03).
         await page.waitForFunction(() => {
           const indicators = [
             'cf-browser-verification',
             'cf-challenge-running',
-            'Checking your browser'
+            'Checking your browser',
+            'Performing security verification'
           ];
-          const pageText = document.body.innerText;
-          return !indicators.some(indicator => pageText.includes(indicator));
-        }, { timeout: 30000 }).catch(() => {});
+          const pageText = document.body ? document.body.innerText : '';
+          return !/^just a moment/i.test(document.title.trim()) &&
+            !indicators.some(indicator => pageText.includes(indicator));
+          // Options are the third argument; in the second they were the page
+          // function's argument and the wait ran Playwright's 30 s default.
+        }, undefined, { timeout: 10000 }).catch(() => {});
         
         this.performanceMetrics.successfulBypasses++;
         return true;
@@ -2550,9 +2608,34 @@ export class StealthBrowserManager {
     return proxy;
   }
 
+  /**
+   * Refuse a caller's proxy whose host the SSRF guard would refuse as a page:
+   * a proxy on loopback, link-local or a metadata address (RFC 1918 too under
+   * SSRF_STRICT, and ALLOWED_DOMAINS exempt, as everywhere else) would have
+   * the browser send the call's traffic into this server's own network. Only
+   * the per-call list is checked — CRAWLFORGE_STEALTH_PROXIES is the
+   * operator's, who may well run a proxy on localhost. The message names the
+   * host, never the entry, which carries the password.
+   * @param {{ proxyRotation?: { enabled?: boolean, proxies?: string[] } }} config
+   * @returns {Promise<void>}
+   * @throws {Error} code SSRF_BLOCKED
+   */
+  async assertCallerProxiesAllowed(config) {
+    const requested = config.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
+    for (const entry of requested) {
+      // url.host of socks4/socks5 parses the same way; the guard only takes http(s).
+      const { host } = new URL(this.parseProxyEntry(entry).server);
+      try {
+        await assertUrlAllowed(`http://${host}/`, { resolveDns: true });
+      } catch (err) {
+        throw Object.assign(new Error(`proxyRotation: proxy host ${host} refused — ${err.message}`), { code: err.code });
+      }
+    }
+  }
+
   /** A proxy entry with its credentials removed, for logs and get_stats. */
   redactProxy(entry) {
-    return String(entry).replace(/\/\/[^/@]*@/, '//');
+    return redactProxyEntry(entry);
   }
 
   /**

package/src/core/processing/BrowserProcessor.js

@@ -373,6 +373,11 @@ export class BrowserProcessor {
     // createStealthContext re-validates the config it is given — so a context
     // asked for without it would relaunch on the default engine underneath.
     const engine = options.stealthMode.engine || 'chromium';
+    // The caller's own proxies (scrape_with_actions browserOptions.proxyRotation),
+    // to both calls for the same reason as the engine: camoufox takes its proxy
+    // at launch, Chromium per context. Absent unless the caller sent one, so
+    // CRAWLFORGE_STEALTH_PROXIES still applies when nobody did.
+    const proxy = options.proxyRotation ? { proxyRotation: options.proxyRotation } : {};
 
     // Launch stealth browser
     await this.stealthManager.launchStealthBrowser({
@@ -381,12 +386,14 @@ export class BrowserProcessor {
       randomizeFingerprint: options.stealthMode.randomizeFingerprint,
       hideWebDriver: options.stealthMode.hideWebDriver,
       blockWebRTC: options.stealthMode.blockWebRTC,
-      customUserAgent: options.stealthMode.customUserAgent || options.userAgent
+      customUserAgent: options.stealthMode.customUserAgent || options.userAgent,
+      ...proxy
     });
 
     // Create stealth context
     const { context, contextId } = await this.stealthManager.createStealthContext({
       engine,
+      ...proxy,
       level: options.stealthMode.level,
       customViewport: {
         width: options.viewportWidth || 1280,

package/src/server/fallbackHints.js

@@ -11,8 +11,8 @@
 
 export const FALLBACK_HINTS = Object.freeze({
   fetch_url: 'For a JS-rendered page or an empty shell use scrape; after a 403/429/CAPTCHA/challenge page use stealth_mode operation:"scrape". Do not repeat the same fetch_url call.',
-  extract_text: 'Use scrape formats:["markdown"] (renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
-  extract_links: 'Use scrape formats:["links"]; for a whole site use map_site.',
+  extract_text: 'After a 403/429/444/challenge page call extract_text again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["markdown"]; for a PDF use process_document.',
+  extract_links: 'After a 403/429/444/challenge page call extract_links again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["links"]; for a whole site use map_site.',
   extract_metadata: 'Use scrape formats:["metadata"]; after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_content: 'Use scrape formats:["markdown"] (same clean output, renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_embedded_state: 'Call again without `path` to see the top-level keys, or use scrape formats:["markdown"] if the page has no framework payload.',
@@ -56,8 +56,8 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
-const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2; extract_text and extract_links since Phase E2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state', 'extract_text', 'extract_links']);
 
 /** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {