crawlforge-mcp-server 5.8.0 → 5.9.0

package/README.md

@@ -179,7 +179,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `list_ollama_models` | 1 | List the Ollama models installed locally (helps you pick a `model` for `extract_with_llm`) |
 | `get_batch_results` | 1 | Retrieve paginated results for a `batch_scrape` job by `batchId` |
 | `read_result` | 1 | Search, slice, read lines or a JSON path from a result a tool returned with `truncated: true` and a `result_handle` (kept 1 hour on your own machine) — never fetch the page again |
-| `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings |
+| `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings. `escalate:true` retries a blocked plain fetch once in the stealth browser and returns the page instead of the block (+5 as the projected ceiling; the charge drops back to 2 when the plain fetch succeeded and no escalation ran) |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
 | `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.8.0",
+  "version": "5.9.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -109,7 +109,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.8.0",
+  version: "5.9.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -121,7 +121,7 @@ const server = new McpServer({
     "- Read one page whose URL you have -> scrape (2). Ask for every format you need in that call: markdown, links, metadata, html, screenshot, json. Default for any page.",
     "- Raw JSON/XML/API body, headers or status -> fetch_url (1). Not for HTML you will read: scrape returns markdown from the same fetch.",
     "- Find pages for a query -> search_web (5); snippets often answer without a scrape. Google organic position -> serp_rank (5). Reddit -> reddit_search (5); reddit.com blocks direct scraping.",
-    "- Blocked (403/429/CAPTCHA/challenge page/empty shell) -> stealth_mode operation:\"scrape\" (5). Needs a click, login or scroll -> scrape_with_actions (5). Never start with these.",
+    "- Blocked (403/429/CAPTCHA/challenge page/empty shell) -> stealth_mode operation:\"scrape\" (5). scrape with escalate:true when a site is known to block (projected 7, charged 2 when the plain fetch works); still never stealth_mode first. Needs a click, login or scroll -> scrape_with_actions (5). Never start with these.",
     "- 2-50 known URLs -> one batch_scrape (5), never a loop of scrape calls.",
     "- A site's URL list -> map_site (2); many pages of one site -> crawl_deep (4).",
     "- Exact values from a Next.js/Nuxt/Redux payload -> extract_embedded_state (2) with a path.",
@@ -165,6 +165,7 @@ server.registerPrompt("getting-started", {
           "\n" +
           "Hard pages\n" +
           "- stealth_mode (5): after a 403/429/CAPTCHA/challenge page or an empty shell - never first.\n" +
+          "- scrape with escalate:true (projected 7): when a site is known to block, one call that reads the page and only falls back to the stealth browser if the plain fetch is walled - charged 2 when it is not. Still never stealth_mode first.\n" +
           "- scrape_with_actions (5): click, log in, scroll or wait, then scrape.\n" +
           "- localization (2): country and locale context for geo-specific content.\n" +
           "\n" +
@@ -236,7 +237,22 @@ const deepResearchTool = new DeepResearchTool();
 const trackChangesTool = new TrackChangesTool();
 const generateLLMsTxtTool = new GenerateLLMsTxtTool();
 const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
-const unifiedScrapeTool = new UnifiedScrapeTool({ actionExecutor: scrapeWithActionsTool.actionExecutor }); // D4 D1 (+v4.8 screenshot reuses the shared browser pool)
+const unifiedScrapeTool = new UnifiedScrapeTool({
+  actionExecutor: scrapeWithActionsTool.actionExecutor, // D4 D1 (+v4.8 screenshot reuses the shared browser pool)
+  // Phase 3: the escalation stage for `scrape`'s escalate:true, injected so
+  // the tool module never imports StealthBrowserManager (that would pull a
+  // browser dependency into every unit test that loads `scrape`). Same gate
+  // and same browser the stealth_mode tool drives — no new evasion, and the
+  // "playwright" → chromium engine mapping stays here with its sibling.
+  escalateScrape: async ({ url, engine, respectRobots }) => {
+    const warnings = await stealthComplianceGate(url, respectRobots);
+    const scraped = await stealthBrowserManager.scrapeWithStealth({
+      url,
+      engine: engine === 'camoufox' ? 'camoufox' : 'chromium'
+    });
+    return { ...scraped, engine, warnings };
+  }
+});
 const agentTool = new AgentTool(); // D4 D2
 const stealthBrowserManager = new StealthBrowserManager();
 const localizationManager = new LocalizationManager();
@@ -1066,7 +1082,7 @@ if (toolFilter.isEnabled("deep_research")) {
 
 // Tool: scrape (D4 D1 — unified multi-format single-fetch)
 registerToolIfEnabled("scrape", {
-  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Not for raw API/JSON bodies (fetch_url), a blocked site (stealth_mode), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
+  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it comes back walled does the stealth browser retry and return the page - projected at 7, charged 2 when the plain fetch worked. Not for raw API/JSON bodies (fetch_url), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
   annotations: { title: "Scrape (Multi-Format)", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   // Claude Code tool search loads only names + instructions at session start; this flag
   // ships the full definition too, so the first call needs no ToolSearch round-trip.

package/src/core/AuthManager.js

@@ -13,6 +13,7 @@ import { resolveApiEndpoint } from './endpointGuard.js';
 import { logger } from '../utils/Logger.js';
 import { maskSecrets } from '../utils/secretMask.js';
 import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
+import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
 // Stamped on every usage report so support can tell which client version
 // made a call; it read '3.0.3' from 3.0.3 to 5.6.10 (the website stores it
 // per record from its Phase 1.3 on).
@@ -626,11 +627,13 @@ class AuthManager {
     }
 
     // The query-scoped formats (highlights, question) add 1 once per call,
-    // and mode:"model" adds 3 once; the base price in the table is what the
-    // backend sees. The rule lives with the format schemas.
+    // and mode:"model" adds 3 once; escalate:true adds the stealth browser's
+    // own 5, since that is the ceiling the call may reach. The base price in
+    // the table is what the backend sees. Both rules live with the schemas
+    // that declare the params.
     if (tool === 'scrape') {
       const { query, model } = scrapeFormatSurcharge(params?.formats);
-      return costs.scrape + query + model;
+      return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate);
     }
 
     return costs[tool] ?? 1;
@@ -689,7 +692,9 @@ class AuthManager {
         break;
       case 'scrape': {
         projected = base;
-        note = 'Base 2; a highlights or question format adds 1 once per call, mode:"model" adds 3 once (charged at the extractive price when no LLM route exists). json format may incur external LLM cost (billed by your provider).';
+        note = 'Base 2; a highlights or question format adds 1 once per call, mode:"model" adds 3 once (charged at the extractive price when no LLM route exists). ' +
+          'escalate:true adds 5 for the stealth browser it may need — the projection is the ceiling, and the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. ' +
+          'json format may incur external LLM cost (billed by your provider).';
         break;
       }
       case 'agent': {

package/src/schemas/toolOutputSchemas.js

@@ -124,6 +124,11 @@ const scrapeShape = {
     vendor: z.string().optional(),
     evidence: z.string().optional()
   }).passthrough().optional().describe('Present when a bot-defence vendor served a challenge page; the fallback hint names the tool to try next'),
+  escalated: z.boolean().optional().describe('Present only when escalate:true was passed: whether the blocked plain fetch was retried in the stealth browser. False means the plain fetch sufficed and the call is charged at the base price'),
+  stealth: z.object({
+    engine: z.string().optional(),
+    vendor_detected: z.string().nullable().optional()
+  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
   content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
   ...resultHandleShape,

package/src/server/fallbackHints.js

@@ -42,26 +42,51 @@ export const FALLBACK_HINTS = Object.freeze({
   scrape_template: 'Use template:"list" to see valid template ids, template:"auto" to pick from the URL, or scrape for a site without a template.'
 });
 
+/**
+ * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
+ * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * ordinary hint would send the model back to a tool that has just failed.
+ * What is left is a region-specific block, which `localization` can change,
+ * or a block that is final: an Akamai-style TLS-level wall is beaten with
+ * residential proxies, which CrawlForge does not offer.
+ */
+export const SCRAPE_ESCALATED_HINT =
+  'The stealth browser has already run on this URL (escalate:true) - do not call stealth_mode or repeat this call. ' +
+  'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
+  'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
+
+/** The hint for this result: the tool's, unless a scrape already escalated. */
+function hintFor(toolName, parsed) {
+  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  return FALLBACK_HINTS[toolName];
+}
+
 /**
  * Append the hint for `toolName` to an error result in place. No-op for
  * success results, unknown tools, non-text content, or a hint already present.
  */
 export function appendFallbackHint(toolName, result) {
-  const hint = FALLBACK_HINTS[toolName];
-  if (!hint || result?.isError !== true || !Array.isArray(result.content)) return result;
+  if (result?.isError !== true || !Array.isArray(result.content)) return result;
   const first = result.content[0];
   if (first?.type !== 'text' || typeof first.text !== 'string') return result;
-  if (first.text.includes(hint)) return result;
+
+  // Parsed first: which hint a `scrape` failure gets depends on whether the
+  // body says the escalation already ran.
+  let parsed = null;
   try {
-    const parsed = JSON.parse(first.text);
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
-      parsed.next_step = hint;
-      first.text = JSON.stringify(parsed, null, 2);
-      return result;
-    }
+    const candidate = JSON.parse(first.text);
+    if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) parsed = candidate;
   } catch {
     // plain text
   }
+
+  const hint = hintFor(toolName, parsed);
+  if (!hint || first.text.includes(hint)) return result;
+  if (parsed) {
+    parsed.next_step = hint;
+    first.text = JSON.stringify(parsed, null, 2);
+    return result;
+  }
   first.text = `${first.text}\nNext step: ${hint}`;
   return result;
 }

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -90,6 +90,7 @@ Do not suggest adding API keys — local Ollama is the intended zero-cost defaul
 | Symptom | Try next |
 |---------|----------|
 | `scrape` / `fetch_url` returns 403, 429, CAPTCHA, or empty JS shell | `stealth_mode` (crawlforge-stealth-browsing) |
+| A site you already know blocks | `scrape` with `escalate: true` — plain fetch first, stealth browser only if it is walled; never `stealth_mode` first |
 | No template for a known site | `scrape_structured` → `extract_structured` → `extract_with_llm` |
 | LLM extraction unavailable (no Ollama/keys) | `scrape_structured` with CSS selectors |
 | Single page too slow / many pages | `batch_scrape` (async + webhook) |

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -20,7 +20,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 
 | Tool | Notes |
 |------|-------|
-| `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. |
+| `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. `escalate:true` adds 5 as the projected ceiling; the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. |
 | `scrape_structured` | CSS-selector extraction. |
 | `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
 | `extract_content` | Readability-cleaned article. |
@@ -77,4 +77,8 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 - `get_batch_results` (1) is cheap — submit a batch once, page through results.
 - A result over `max_inline_chars` costs nothing extra; read the stored copy with
   `read_result` (1) instead of fetching again.
+- On a site that blocks, one `scrape` with `escalate:true` costs at most 7 and
+  only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
+  way. Do not set it on a site that reads fine — it raises the projection your
+  balance is checked against.
 - Errors are charged at half the tool cost; creator mode is unlimited.

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -24,6 +24,11 @@ Escalate from a normal `scrape` / `fetch_url` (see crawlforge-web-scraping) when
 `stealth_mode` drives a real browser with randomized fingerprints, human
 behavior simulation, and WebRTC/canvas/WebGL spoofing.
 
+When you already know the site blocks, `scrape` with `escalate: true` does both
+steps in one call: the plain fetch first, then the same stealth browser only if
+that fetch is walled. Projected at 7, charged 2 when the plain fetch worked.
+Still never reach for `stealth_mode` first.
+
 ## stealth_mode (cost: 5)
 
 `stealth_mode` is operation-based. Typical flow: create a context, then create a

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -36,7 +36,9 @@ site discovery (sitemaps and URL maps), and whole-site crawling.
 5. **`crawl_deep`** to walk an entire site and (optionally) extract content.
 
 If a page returns 403/429, a CAPTCHA, or empty "enable JavaScript" content,
-switch to the **crawlforge-stealth-browsing** skill (`stealth_mode`).
+switch to the **crawlforge-stealth-browsing** skill (`stealth_mode`). When the
+site is *known* to block, use `scrape` with `escalate: true` instead of two
+calls — still never `stealth_mode` first.
 
 ## scrape — unified multi-format (cost: 2)
 
@@ -59,6 +61,33 @@ Get markdown + links + metadata in a single call:
 a failing format adds a `warnings[]` entry instead of failing the whole call.
 `onlyMainContent` (default `true`) strips boilerplate via Readability.
 
+### escalate — one call for a site that blocks (projected 7, charged 2 when it doesn't)
+
+`escalate: true` keeps the plain fetch first and retries in the stealth browser
+only if that fetch comes back walled (403/429, a challenge page, an empty
+shell). The escalated page goes through the same formats, so `markdown`,
+`highlights` and the rest work exactly as they do on a plain read.
+
+```json
+{
+  "tool": "scrape",
+  "params": {
+    "url": "https://www.producthunt.com/",
+    "formats": ["markdown"],
+    "escalate": true,
+    "escalate_engine": "playwright"
+  }
+}
+```
+
+The result adds `escalated: true|false`, and `stealth: { engine,
+vendor_detected }` when the browser ran. The projection is `2 + 5`; the charge
+falls back to the base when the plain fetch succeeded and nothing escalated.
+`escalate_engine` is `"playwright"` (default) or `"camoufox"`. robots.txt is
+respected on the escalated path too, and a second call to a host that blocked
+within the last 24 hours skips the doomed plain fetch and says so in
+`warnings[]`.
+
 Query-scoped formats return only the parts of the page that match, verbatim,
 with offsets into the `markdown` of the same call:
 

package/src/tools/scrape/escalation.js

@@ -0,0 +1,62 @@
+/**
+ * escalation.js — the one declaration of `scrape`'s opt-in auto-escalation.
+ *
+ * `escalate: true` asks the tool to run the stealth browser itself when the
+ * plain fetch comes back walled (Phase 3). The plain fetch always runs first
+ * (G1: never start with stealth), so the escalation is a second stage, not a
+ * different tool: it reuses the same compliance gate and the same
+ * StealthBrowserManager `stealth_mode` uses, and adds no new evasion.
+ *
+ * The two fields and the price rule live here so `server.js`, the tool
+ * module, `AuthManager.getToolCost` and the output schema read one
+ * declaration (G5) — the same arrangement `formats.js` already has for the
+ * query-format surcharge.
+ */
+
+import { z } from 'zod';
+
+/** What an escalated call adds to `scrape`'s price: the stealth browser's own. */
+export const SCRAPE_ESCALATION_CREDITS = 5;
+
+/** The two public fields, in tools/list order, with the text the client sees. */
+export const SCRAPE_ESCALATION_SHAPE = {
+  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/challenge page/empty shell), retry once in the stealth browser and return its content instead of the block. Projected at 2+5; the actual charge stays at the base price when the plain fetch succeeded. Default: false'),
+  escalate_engine: z.enum(['playwright', 'camoufox']).optional().default('playwright').describe('Stealth engine for the escalated retry (default: "playwright")')
+};
+
+/**
+ * What escalation adds to `scrape`'s base price: 5 when the caller asked for
+ * it, nothing otherwise. Reads the RAW param — getToolCost runs before
+ * validation, so anything that is not exactly `true` prices as 0.
+ *
+ * @param {unknown} escalate the raw `escalate` param
+ * @returns {0 | 5}
+ */
+export function scrapeEscalationSurcharge(escalate) {
+  return escalate === true ? SCRAPE_ESCALATION_CREDITS : 0;
+}
+
+/**
+ * Whether a second, browser-backed attempt could plausibly change this
+ * outcome.
+ *
+ * A named vendor's wall qualifies, and so does a bare 403 or 429 — an
+ * IP-reputation, WAF or User-Agent block, which a real browser and fingerprint
+ * often do pass (travel.state.gov answers some networks with exactly that). So
+ * does any failure that arrived with a 2xx: an empty shell, a JavaScript-only
+ * document or a soft-error placeholder is precisely what rendering fixes.
+ *
+ * A 404 is not a wall and a 5xx is the server's own failure. Escalating either
+ * would spend the browser's 5 credits to be told the same thing, so `escalate`
+ * leaves them as the plain fetch reported them (G4 — never charge for work
+ * that cannot help).
+ *
+ * @param {{ blocked?: unknown } | null | undefined} verdict
+ * @param {number | null | undefined} status
+ * @returns {boolean}
+ */
+export function aBrowserMightPass(verdict, status) {
+  if (verdict && verdict.blocked) return true;
+  if (status === 403 || status === 429) return true;
+  return status === null || status === undefined || (status >= 200 && status < 300);
+}

package/src/tools/scrape/unifiedScrape.js

@@ -20,6 +20,7 @@ import {
   SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema,
   scrapeFormatSurcharge
 } from './formats.js';
+import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from './escalation.js';
 import { toPublicUnit, parseChosenIndexes, groundingCheck } from './_highlights.js';
 import { setActualCost } from '../../server/requestContext.js';
 import { fenceUntrusted } from '../../utils/untrustedContent.js';
@@ -29,11 +30,15 @@ import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
-import { noteHostBlocked, clearHostBlocked } from '../../utils/hostRateLimiter.js';
+import { noteHostBlocked, clearHostBlocked, getHostBlock } from '../../utils/hostRateLimiter.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
 export { SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema };
+export { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS };
+
+/** scrape's base price, as AuthManager's table spells it. */
+const SCRAPE_BASE_CREDITS = 2;
 
 // Above this, a markdown result is the whole page in the context window;
 // the warning names the format that returns only the matching units (1.4).
@@ -61,7 +66,9 @@ export const SCRAPE_INPUT_SHAPE = {
     fullPage: z.boolean().optional().default(false).describe('Capture the full scrollable page'),
     format: z.enum(['png', 'jpeg']).optional().default('png'),
     quality: z.number().min(0).max(100).optional().describe('JPEG quality (jpeg only)')
-  }).optional().describe('Options for the "screenshot" format')
+  }).optional().describe('Options for the "screenshot" format'),
+  // Opt-in second stage (Phase 3): the plain fetch still runs first.
+  ...SCRAPE_ESCALATION_SHAPE
 };
 
 export const UnifiedScrapeSchema = z.object({
@@ -194,6 +201,15 @@ export class UnifiedScrapeTool {
     // Optional shared ActionExecutor (injected from server.js so we reuse the
     // existing browser pool rather than spinning up a second one).
     this._actionExecutor = options.actionExecutor || null;
+    // Optional escalation stage (Phase 3), injected from server.js the same
+    // way. The tool module must never import StealthBrowserManager itself:
+    // that would pull a browser dependency into every unit test that loads
+    // `scrape`. Signature:
+    //   ({ url, engine, respectRobots }) =>
+    //     { html, url, title, text, status, engine, warnings? }
+    // The injected function owns the compliance gate (robots + blocklist) and
+    // the engine-name mapping, exactly as the stealth_mode tool does.
+    this._escalateScrape = options.escalateScrape || null;
     this._mcpServer = null;
   }
 
@@ -278,59 +294,183 @@ export class UnifiedScrapeTool {
     const validated = UnifiedScrapeSchema.parse(params);
     const { url, formats, onlyMainContent, timeoutMs, brandingOptions, screenshotOptions, resolveHiddenContent } = validated;
 
+    // The gate's warnings (e.g. a respect_robots override) travel with the
+    // per-format ones, so the caller sees the decision in the response.
+    const warnings = [];
+    // Whether a model completed for a mode:"model" format, and whether the
+    // stealth stage ran; the surcharge for each is charged only then.
+    let modelUsed = false;
+    let escalationRan = false;
+    let stealthEngine = null;
+
+    // One charge computation for every return path (3.2). The projection is
+    // the ceiling a caller saw before the call, so each stage that did not
+    // run only ever lowers the bill (G4): the model surcharge when nothing
+    // completed, the escalation surcharge when the plain fetch sufficed.
+    const surcharge = scrapeFormatSurcharge(formats);
+    const reportCost = () => setActualCost(
+      SCRAPE_BASE_CREDITS +
+      surcharge.query +
+      (modelUsed ? surcharge.model : 0) +
+      (escalationRan ? SCRAPE_ESCALATION_CREDITS : 0)
+    );
+
+    // Opt-in second stage (Phase 3): the plain fetch runs first and the
+    // stealth browser only follows a blocked verdict.
+    const escalate = validated.escalate === true;
+    // 3.3: a host that walled us within the last 24 hours (0.6's memory) has
+    // a doomed plain fetch ahead of it. Read ONLY when escalating — an
+    // ordinary scrape must behave exactly as it did before.
+    const remembered = escalate ? getHostBlock(url) : null;
+
     // Single fetch
-    let html, $, finalUrl, fetchWarnings, status;
-    try {
-      ({ html, $, finalUrl, warnings: fetchWarnings, status } = await fetchAndParse(url, {
-        timeoutMs,
-        userAgent: validated.user_agent,
-        respectRobots: validated.respect_robots,
-        tool: 'scrape',
-        stripTags: [], // we handle boilerplate ourselves
-        // A real wall reaches a plain fetch as a 403 with the challenge in
-        // the body; the verdict below needs that body and the status.
-        errorDocuments: true
-      }));
-    } catch (err) {
-      throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
+    let html, $, fetchWarnings, status, title, verdict;
+    let finalUrl = url;
+    if (remembered) {
+      warnings.push(
+        `escalate: ${remembered.vendor || 'a bot wall'} walled this host within the last 24 hours; the plain fetch was skipped and the stealth browser ran first`
+      );
+    } else {
+      try {
+        ({ html, $, finalUrl, warnings: fetchWarnings, status } = await fetchAndParse(url, {
+          timeoutMs,
+          userAgent: validated.user_agent,
+          respectRobots: validated.respect_robots,
+          tool: 'scrape',
+          stripTags: [], // we handle boilerplate ourselves
+          // A real wall reaches a plain fetch as a 403 with the challenge in
+          // the body; the verdict below needs that body and the status.
+          errorDocuments: true
+        }));
+      } catch (err) {
+        throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
+      }
+      warnings.push(...fetchWarnings);
+
+      // A bot wall, an HTTP error page, an empty shell or an error placeholder
+      // arrives with a title and prose of its own; reported as a successful
+      // scrape it hid the block for three rounds (producthunt.com, R10 Q1 →
+      // R15). The stealth path has named these since 5.6.2; this path never
+      // looked, and threw away every non-2xx body before it could (Phase 0,
+      // 0.1). Scripts stay in $ for the rawHtml and metadata formats,
+      // so the text is measured on a copy without them — the vendor tables
+      // count the characters a reader would see, as the browser path does.
+      // Nothing below runs on a failed verdict: a screenshot would launch a
+      // browser on the wall and branding would fetch its stylesheets.
+      title = pageTitle($);
+      const $visible = load(html);
+      $visible('script, style, noscript, template').remove();
+      verdict = documentVerdict(
+        { url: finalUrl, title, text: $visible('body').text(), html, status },
+        { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+      );
+      // Host memory (0.6): remember the vendor that walled this host, keyed by
+      // the URL the caller passed as well as the final one when a redirect
+      // moved hosts, since a later call looks it up by what it was given.
+      const hosts = [finalUrl, url];
+      if (verdict.blocked) {
+        for (const h of hosts) noteHostBlocked(h, verdict.blocked.vendor);
+      } else if (verdict.success) {
+        for (const h of hosts) clearHostBlocked(h);
+      }
     }
 
-    // A bot wall, an HTTP error page, an empty shell or an error placeholder
-    // arrives with a title and prose of its own; reported as a successful
-    // scrape it hid the block for three rounds (producthunt.com, R10 Q1 →
-    // R15). The stealth path has named these since 5.6.2; this path never
-    // looked, and threw away every non-2xx body before it could (Phase 0,
-    // 0.1). Scripts stay in $ for the rawHtml and metadata formats,
-    // so the text is measured on a copy without them — the vendor tables
-    // count the characters a reader would see, as the browser path does.
-    // Nothing below runs on a failed verdict: a screenshot would launch a
-    // browser on the wall and branding would fetch its stylesheets.
-    const title = pageTitle($);
-    const $visible = load(html);
-    $visible('script, style, noscript, template').remove();
-    const verdict = documentVerdict(
-      { url: finalUrl, title, text: $visible('body').text(), html, status },
-      { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
-    );
-    // Host memory (0.6): remember the vendor that walled this host, keyed by
-    // the URL the caller passed as well as the final one when a redirect
-    // moved hosts, since a later call looks it up by what it was given.
-    const hosts = [finalUrl, url];
-    if (verdict.blocked) {
-      for (const h of hosts) noteHostBlocked(h, verdict.blocked.vendor);
-    } else if (verdict.success) {
-      for (const h of hosts) clearHostBlocked(h);
+    // ── Escalation (Phase 3) ─────────────────────────────────────────────
+    // Opt-in, and second by construction: the plain fetch above has already
+    // run and only a blocked verdict gets here (G1 — never start with
+    // stealth). The stage reuses the compliance gate and the browser
+    // `stealth_mode` already drives, so it adds no evasion of its own.
+    // The vendor the plain fetch hit, or the one this host is remembered for
+    // when that fetch was skipped.
+    const vendorDetected = remembered ? (remembered.vendor ?? null) : (verdict?.blocked?.vendor ?? null);
+    // Set when the stage was asked for and could not run at all; it is the
+    // only thing left to report when the plain fetch was skipped as well.
+    let escalationError = null;
+
+    // Only a failure a browser could plausibly change is worth the second
+    // stage: a wall or a bare 403/429, or a page that failed with a 2xx (an
+    // empty shell renders). A 404 or a 5xx escalates nothing — see
+    // aBrowserMightPass.
+    if (escalate && (remembered || (!verdict.success && aBrowserMightPass(verdict, status)))) {
+      if (!this._escalateScrape) {
+        escalationError = 'no stealth stage is wired into this server build';
+        warnings.push(`escalate: ${escalationError}`);
+      } else {
+        try {
+          const stealth = await this._escalateScrape({
+            url: remembered ? url : finalUrl,
+            engine: validated.escalate_engine,
+            respectRobots: validated.respect_robots
+          });
+          escalationRan = true;
+          stealthEngine = stealth.engine || validated.escalate_engine;
+          if (Array.isArray(stealth.warnings)) warnings.push(...stealth.warnings);
+
+          // The stealth render replaces the document every format below is
+          // built from, so an escalated page goes through the SAME formats
+          // loop a plain fetch does — highlights and question included.
+          html = stealth.html || '';
+          finalUrl = stealth.url || finalUrl;
+          status = stealth.status ?? null;
+          $ = load(html);
+          title = stealth.title || pageTitle($);
+          const $rendered = load(html);
+          $rendered('script, style, noscript, template').remove();
+          // Re-run the verdict on what the browser rendered: a wall that
+          // survives the stealth pass must not come back as a success.
+          verdict = documentVerdict(
+            { url: finalUrl, title, text: stealth.text || $rendered('body').text(), html, status },
+            { waitedMs: stealth.gracedMs || 0, fetcher: 'the stealth browser', rendered: true, contentReturned: false }
+          );
+          // Escalation fires on ANY failed verdict, not only a named vendor —
+          // an empty shell and an error placeholder are exactly the cases a
+          // browser fixes — so the wording must not call every one a block.
+          warnings.push(
+            `escalate: the plain fetch ${vendorDetected ? `was blocked by ${vendorDetected}` : 'did not return the page'}; ` +
+            `the ${stealthEngine} stealth browser ${verdict.success ? 'returned it' : 'did not get it either'}`
+          );
+          // The host memory is deliberately left alone here. It records what
+          // a PLAIN fetch met, which is what 3.3 reads before deciding to
+          // skip one; a stealth success says nothing about that, and only a
+          // clean plain fetch clears the entry (0.6).
+        } catch (err) {
+          // The gate refuses a disallowed URL by throwing, after stamping
+          // markPreflightRefusal('ROBOTS_DISALLOWED') on the request context.
+          // That flag makes withAuth bill the WHOLE call zero — plain fetch
+          // included — which is the intended outcome: we never rendered the
+          // page we were refused. Caught here so a refusal is a warning on
+          // the verdict we already have rather than a throw out of the tool.
+          escalationError = err.message;
+          warnings.push(`escalate: the stealth retry did not run — ${escalationError}`);
+        }
+      }
     }
-    if (!verdict.success) {
+
+    // `escalated` is reported only to a caller who asked to escalate: a call
+    // that never asked keeps exactly the result shape it had before.
+    const escalationFields = escalate
+      ? {
+        escalated: escalationRan,
+        ...(escalationRan ? { stealth: { engine: stealthEngine, vendor_detected: vendorDetected } } : {})
+      }
+      : {};
+
+    if (!verdict || !verdict.success) {
+      reportCost();
       return {
         success: false,
         url: finalUrl,
-        status,
+        // A stealth render reports no status when nothing navigated, and a
+        // skipped plain fetch reports none at all.
+        ...(typeof status === 'number' ? { status } : {}),
         title,
-        error: verdict.error,
-        ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
+        error: verdict
+          ? verdict.error
+          : `The plain fetch was skipped because ${vendorDetected || 'a bot wall'} walled this host within the last 24 hours, and the stealth retry did not run: ${escalationError}`,
+        ...(verdict?.blocked ? { blocked: verdict.blocked } : {}),
+        ...escalationFields,
         content: {},
-        warnings: fetchWarnings.length > 0 ? fetchWarnings : undefined
+        warnings: warnings.length > 0 ? warnings : undefined
       };
     }
 
@@ -380,14 +520,7 @@ export class UnifiedScrapeTool {
       if (units === null) units = segmentUnits(getMarkdown());
       return units;
     }
-    // Whether a model completed for a mode:"model" format; the model
-    // surcharge is charged only then.
-    let modelUsed = false;
-
     const content = {};
-    // The gate's warnings (e.g. a respect_robots override) travel with the
-    // per-format ones, so the caller sees the decision in the response.
-    const warnings = [...fetchWarnings];
 
     // Kept for the rawHtml format, which must survive the strip below.
     const pristineHtml = html;
@@ -652,15 +785,15 @@ export class UnifiedScrapeTool {
     if (!queryScoped && typeof content.markdown === 'string' && content.markdown.length > OVERSIZED_MARKDOWN_CHARS) {
       warnings.push(`markdown: ${content.markdown.length} characters; ask for {type:"highlights", query} (1 extra credit, no model) to get only the matching sentences, table rows and code blocks`);
     }
-    // The projection charged the model step; nothing ran, so the charge
-    // drops to the extractive price (2 is scrape's base in AuthManager;
-    // withAuth clamps to the projection either way).
-    const surcharge = scrapeFormatSurcharge(formats);
-    if (surcharge.model > 0 && !modelUsed) setActualCost(2 + surcharge.query);
+    // What this call actually spent: the model step and the escalation each
+    // drop out when they did not run (withAuth clamps to the projection
+    // either way, so this can only lower the charge).
+    reportCost();
 
     return {
       success: true,
       url: finalUrl,
+      ...escalationFields,
       content,
       warnings: warnings.length > 0 ? warnings : undefined
     };