crawlforge-mcp-server 6.17.1 → 6.18.0

package/README.md

@@ -178,8 +178,8 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | Tool | Credits | What it does |
 |------|---------|--------------|
 | `fetch_url` | 1 | Fetch content from any URL |
-| `extract_text` | 1 | Extract clean text from web pages |
-| `extract_links` | 1 | Get all links from a page |
+| `extract_text` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Extract clean text from web pages |
+| `extract_links` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Get all links from a page |
 | `extract_metadata` | 1 | Extract page metadata (title, OG tags, schema.org) |
 | `scrape_template` | 1 | Structured data from well-known sites (Amazon, GitHub, LinkedIn, YouTube, Reddit, Hacker News, npm, and more) without writing selectors |
 | `list_ollama_models` | 1 | List the Ollama models installed locally (helps you pick a `model` for `extract_with_llm`) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.17.1",
+  "version": "6.18.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.12.0",
+    "crawlforge-extractors": "^1.13.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -52,8 +52,9 @@ import { createOAuthProvider } from "./src/server/auth/oauth.js";
 import { createMetricsRegistry } from "./src/observability/metrics.js";
 // Basic tool handlers (extracted from server.js)
 import { fetchUrlHandler } from "./src/tools/basic/fetchUrl.js";
-import { extractTextHandler } from "./src/tools/basic/extractText.js";
-import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { createExtractTextHandler } from "./src/tools/basic/extractText.js";
+import { createExtractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { EXTRACT_ESCALATION_SHAPE } from "./src/utils/fetchLadder.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
 import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
@@ -112,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.17.1",
+  version: "6.18.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -439,29 +440,37 @@ registerToolIfEnabled("fetch_url", {
 
 // Tool: extract_text
 registerToolIfEnabled("extract_text", {
-  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
+  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG; selector reads only the matched elements, max_length caps the returned text. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
   annotations: { title: "Extract Text", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract text from"),
     remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
     remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
     output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
+    selector: z.string().optional().describe("CSS selector: read only the matched elements (nav/header/footer are then kept). No match is an error"),
+    max_length: z.number().int().min(1).max(1000000).optional().describe("Maximum characters of text or markdown to return; a longer result is cut and ends with \"...\""),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...REDACT_PII_PARAM
   }
-}, withAuth("extract_text", extractTextHandler));
+}, withAuth("extract_text", createExtractTextHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_text' })
+})));
 
 // Tool: extract_links
 registerToolIfEnabled("extract_links", {
-  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true returns only outbound links. Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
+  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true drops internal links; mailto:, tel: and javascript: links are type \"other\". Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
   annotations: { title: "Extract Links", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract links from"),
-    filter_external: z.boolean().optional().default(false).describe("Only return external links"),
+    filter_external: z.boolean().optional().default(false).describe("Drop internal (same-host) links"),
     base_url: z.string().url().optional().describe("Base URL for resolving relative links"),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("extract_links", extractLinksHandler));
+}, withAuth("extract_links", createExtractLinksHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_links' })
+})));
 
 // Tool: extract_metadata
 registerToolIfEnabled("extract_metadata", {

package/src/core/ActionExecutor.js

@@ -1581,9 +1581,12 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
+    // The initial load and a retry's reload pass no timeout of their own, so
+    // browserOptions.timeout bounds them as it bounds the actions (0.2); a
+    // navigate action's own timeout comes first.
     const response = await page.goto(url, {
       waitUntil: options.waitUntil || 'domcontentloaded',
-      timeout: options.timeout || 30000
+      timeout: options.timeout || options.browserOptions?.timeout || 30000
     });
 
     // Re-validate where the navigation went: a redirect could have taken us

package/src/core/AuthManager.js

@@ -687,6 +687,12 @@ class AuthManager {
       return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_text and extract_links (fix plan Phase E2) run the same ladder:
+    // projected 1+5 when escalating, charged 1 when the plain fetch sufficed.
+    if (tool === 'extract_text' || tool === 'extract_links') {
+      return costs[tool] + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -778,6 +784,12 @@ class AuthManager {
           ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
           : 'Fixed cost per invocation.';
         break;
+      case 'extract_text':
+      case 'extract_links':
+        note = params?.escalate === true
+          ? 'Base 1; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/server/fallbackHints.js

@@ -11,8 +11,8 @@
 
 export const FALLBACK_HINTS = Object.freeze({
   fetch_url: 'For a JS-rendered page or an empty shell use scrape; after a 403/429/CAPTCHA/challenge page use stealth_mode operation:"scrape". Do not repeat the same fetch_url call.',
-  extract_text: 'Use scrape formats:["markdown"] (renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
-  extract_links: 'Use scrape formats:["links"]; for a whole site use map_site.',
+  extract_text: 'After a 403/429/444/challenge page call extract_text again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["markdown"]; for a PDF use process_document.',
+  extract_links: 'After a 403/429/444/challenge page call extract_links again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["links"]; for a whole site use map_site.',
   extract_metadata: 'Use scrape formats:["metadata"]; after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_content: 'Use scrape formats:["markdown"] (same clean output, renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_embedded_state: 'Call again without `path` to see the top-level keys, or use scrape formats:["markdown"] if the page has no framework payload.',
@@ -56,8 +56,8 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
-const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2; extract_text and extract_links since Phase E2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state', 'extract_text', 'extract_links']);
 
 /** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {

package/src/server/inlineThreshold.js

@@ -20,8 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `when` gates
- * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * on. An empty list means the pretty-printed JSON is the view. `keep` names
+ * top-level fields carried inline past the scalar filter (keepNamedFields).
+ * `when` gates on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
  * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
@@ -37,7 +38,14 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   // an async job's page came back as 111 KB whole (R20, 2026-09-07).
   get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
-  scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  // `keep` names the top-level reports that stay inline past the scalar
+  // filter (keepNamedFields): a walled Guardian chain over 40,000 chars lost
+  // `blocked`, `navigations` and every action's outcome (2026-10-03).
+  scrape_with_actions: {
+    textPaths: ['content.markdown', 'content.text', 'content.html'],
+    truncate: true,
+    keep: ['error', 'blocked', 'consent', 'navigations', 'actionResults']
+  },
   // `read` hands back the same content shape scrape_with_actions does, and was
   // the one content-returning tool with no cap: a read of the World War II
   // article returned 541,308 characters inline where scrape returned 42,259
@@ -48,6 +56,7 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   browser_session: {
     textPaths: ['content.markdown', 'content.text', 'content.html', 'snapshot.tree'],
     truncate: true,
+    keep: ['error', 'blocked', 'actionResults'],
     when: (params) => BROWSER_SESSION_CONTENT_OPERATIONS.has(params?.operation)
   },
   process_document: { textPaths: ['content.text'], truncate: true },
@@ -111,6 +120,42 @@ export function keepSmallContentFields(content, textPaths = [], maxInline = DEFA
   return Object.keys(kept).length > 0 ? kept : null;
 }
 
+/**
+ * The top-level fields a tool names in `keep`, each carried inline when its
+ * JSON fits a quarter of the inline budget. An actionResults list over that
+ * keeps every entry without its `result` (a snapshot tree, an extracted
+ * value), so which action ran and whether it worked stays inline.
+ * @returns {{ kept: object|null, partial: string[], dropped: string[] }}
+ */
+export function keepNamedFields(resultObject, keys = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  const partial = [];
+  const dropped = [];
+  for (const key of keys) {
+    const value = resultObject[key];
+    if (value === undefined) continue;
+    if (JSON.stringify(value).length <= cap) {
+      kept[key] = value;
+      continue;
+    }
+    if (key === 'actionResults' && Array.isArray(value)) {
+      const slim = value.map((entry) => {
+        if (!entry || typeof entry !== 'object') return entry;
+        const { result: _result, ...rest } = entry;
+        return rest;
+      });
+      if (JSON.stringify(slim).length <= cap) {
+        kept[key] = slim;
+        partial.push(key);
+        continue;
+      }
+    }
+    dropped.push(key);
+  }
+  return { kept: Object.keys(kept).length > 0 ? kept : null, partial, dropped };
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -156,7 +201,16 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
 
   const preview = text.slice(0, maxInline);
   const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
-  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const named = keepNamedFields(resultObject, config.keep, maxInline);
+  const keptNames = [
+    ...(named.kept ? Object.keys(named.kept).filter((key) => !named.partial.includes(key)) : []),
+    ...(keptFields ? Object.keys(keptFields).map((key) => `content.${key}`) : [])
+  ];
+  const keptDesc = [
+    keptNames.length > 0 ? `; ${keptNames.join(', ')} kept inline` : '',
+    named.partial.includes('actionResults') ? '; actionResults kept without each action\'s result (json_path "actionResults[<i>].result" reads one)' : '',
+    named.dropped.length > 0 ? `; ${named.dropped.join(', ')} too large to keep inline (json_path "${named.dropped[0]}" reads it)` : ''
+  ].join('');
   const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
@@ -171,6 +225,7 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  if (named.kept) Object.assign(shaped, named.kept);
   // The query-scoped formats live beside the page text under `content`
   // (highlights, answer, json, metadata, links). They are the small, exact
   // answer the caller paid for, and truncating the markdown must not drop
@@ -191,6 +246,14 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   return { result: shaped, stored: true };
 }
 
+/**
+ * Room for the `_cost` block withAuth adds to the text after shaping:
+ * pretty-printed, projected/actual/remaining_credits and the projection
+ * note, whose longest form for extract_embedded_state (escalate:true) is
+ * about 200 chars, about 340 in all.
+ */
+export const COST_BLOCK_RESERVE = 400;
+
 /** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
 export function embeddedDataKeys(data) {
   if (Array.isArray(data)) return `array(${data.length})`;
@@ -227,10 +290,12 @@ export function wholeLinePreview(text, budget) {
  * everything that says what the page carries — url, path, bytes, the whole
  * `found` list, data_keys, the escalation report and window_state's list —
  * then a preview of the pretty-printed JSON cut at a line boundary. The
- * preview gets whatever max_inline_chars leaves after the rest, measured as
- * compact JSON, so the whole inline result stays within the limit whenever
- * the rest fits. The full result is under the handle for read_result
- * json_path. The REST route shapes it the same way; keep the two in step.
+ * preview gets whatever max_inline_chars leaves after the rest, measured the
+ * way withAuth sends it — pretty-printed, with COST_BLOCK_RESERVE held back
+ * for the `_cost` block it adds after shaping — so the text the client
+ * receives stays within the limit whenever the rest fits. The full result is
+ * under the handle for read_result json_path. The REST route shapes it the
+ * same way, measured as the compact JSON it sends; keep the two in step.
  */
 function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
   const dataKeys = embeddedDataKeys(resultObject.data);
@@ -257,8 +322,9 @@ function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expir
     expires_at,
     warnings: [...warningsOf(resultObject), hint]
   };
-  // `"preview":"",` is what the key itself adds once the value is in.
-  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  // The preview's value is one line of the pretty-printed text and escapes
+  // the same way there as in compact JSON; `""` is its empty value's quotes.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }, null, 2).length + COST_BLOCK_RESERVE;
   const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
   return { ...shaped, preview, ...tail };
 }

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -8,8 +8,8 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 | Tool | Notes |
 |------|-------|
 | `fetch_url` | Raw HTTP fetch. |
-| `extract_text` | Tag-stripped text/markdown. |
-| `extract_links` | All links on a page. |
+| `extract_text` | Tag-stripped text/markdown. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
+| `extract_links` | All links on a page. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
 | `extract_metadata` | Title / meta / OG / schema.org. |
 | `scrape_template` | Pre-built site extractor. |
 | `list_ollama_models` | List local LLMs. |

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -158,6 +158,10 @@ Returns headers + body. Good for JSON/XML APIs or as a first step before
 `extract_text` (cost 1) = faster, strips tags; pass `output_format:"markdown"`
 for RAG-friendly output.
 
+`extract_text` and `extract_links` take `escalate:true` for a site known to
+block: the plain fetch runs first and the stealth browser re-reads the page
+only if it is walled (projected 6, charged 1 when the plain fetch worked).
+
 ## extract_links / extract_metadata (cost: 1 each)
 
 ```json

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -34,14 +34,28 @@ your provider).
 | `remove_scripts` | boolean | `true` | Strip `script` tags. |
 | `remove_styles` | boolean | `true` | Strip `style` tags. |
 | `output_format` | enum | `text` | `text` or `markdown` (use markdown for RAG). |
+| `selector` | string | — | CSS selector: read only the matched elements (nav/header/footer inside them are kept). No match is an error: `No elements found for selector: <sel>`. |
+| `max_length` | integer | — | 1–1,000,000. Text or markdown longer than this is cut and ends with `...`; `word_count`/`char_count` count the cut result. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Text mode returns one line per block element (`<h1>Hi</h1><p>there</p>` reads `Hi\nthere`). The REST route (`/api/v1/tools/extract_text`) takes the same parameters plus `preserve_links` and `preserve_formatting`, which the MCP tool does not.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_links (cost: 1)
 
 | Param | Type | Default | Notes |
 |-------|------|---------|-------|
 | `url` | string (URL) | — | Required. |
-| `filter_external` | boolean | `false` | Only return outbound links. |
-| `base_url` | string (URL) | — | Resolve relative links against this. |
+| `filter_external` | boolean | `false` | Drop internal (same-host) links; external and `other` links stay. |
+| `base_url` | string (URL) | — | Resolve relative links against this. Without it, the page's `<base href>`, else the page URL. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Each link is `{ href, text, type, domain, rel, original_href }` (plus `title` when the link has one). `type` is `internal` (same host as the page), `external` or `other` (`mailto:`, `tel:`, `javascript:` and other non-web schemes; only `mailto:`/`tel:` keep an `href`, the rest are `null`). `#fragment`-only links are skipped, and links are deduplicated on the URL without its fragment or trailing slash. Counts: `total_count`, `internal_count`, `external_count`, `other_count`. The REST route returns the same records.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_metadata (cost: 1)
 

package/src/tools/basic/_targetFailure.js

@@ -0,0 +1,54 @@
+/**
+ * Why a fetched document fails extract_links / extract_text, with the same
+ * verdict and wording as the REST routes (fix plan Phase E1).
+ *
+ * A named vendor's wall fails on any status: a challenge arrives as a 403 or
+ * as a 200 with the wall in the body. Any other non-2xx fails with its
+ * status. An empty shell or an error placeholder on a 2xx is not raised here;
+ * extraction proceeds as before.
+ */
+
+import { load } from 'cheerio';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
+import { pageTitle } from '../../utils/pageTitle.js';
+
+// Response bodies a verdict can be read from; anything else is a binary. A
+// missing Content-Type is read, as _fetchAndParse.js treats it as HTML.
+const TEXTUAL_BODY = /^(?:text\/(?:html|plain)|application\/xhtml\+xml)\b/i;
+
+/**
+ * The plain fetch's verdict on a document, or null for a body a verdict
+ * cannot be read from (a binary, JSON). The fetch ladder reads it to decide
+ * whether escalation could help (src/utils/fetchLadder.js).
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ */
+export function plainVerdict(response, url) {
+  const contentType = response.headers?.get?.('content-type');
+  if (contentType && !TEXTUAL_BODY.test(contentType)) return null;
+  // A parse of its own, scripts and styles removed, so the handler's
+  // document stays intact.
+  const $ = load(response._body ?? '');
+  $('script, style, noscript').remove();
+  return stealthDocumentVerdict(
+    {
+      url: response.url || url,
+      status: response.status,
+      title: pageTitle($),
+      text: $('body').text().replace(/\s+/g, ' ').trim(),
+      html: response._body
+    },
+    { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+  );
+}
+
+/**
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ * @param {object|null} [verdict] - plainVerdict's, when the caller already has it
+ * @returns {string|null} the error message, or null when the document is usable
+ */
+export function targetFailure(response, url, verdict = plainVerdict(response, url)) {
+  if (verdict?.blocked) return `Target answered HTTP ${response.status}: ${verdict.error}`;
+  return response.ok ? null : `Target answered HTTP ${response.status}`;
+}

package/src/tools/basic/extractLinks.js

@@ -1,76 +1,84 @@
 /**
  * extract_links — Extract all links from a webpage with optional filtering.
  * Extracted from server.js inline handler.
+ * E3: Link records come from crawlforge-extractors' extractLinkRecords, the
+ * reader the REST route uses, so both surfaces return the same links.
  */
 
 import { load } from 'cheerio';
-import { fetchWithTimeout } from './_fetch.js';
+import { extractLinkRecords } from 'crawlforge-extractors';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 
 /**
  * @param {{ url: string, filter_external?: boolean, base_url?: string,
- *   user_agent?: string, respect_robots?: boolean }} params
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractLinksHandler({ url, filter_external, base_url, user_agent, respect_robots }) {
+async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_links',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_links'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_links', 'Failed to extract links: ', ladder);
+
+    // A JSON body carries no <a href>.
+    if (isJsonType(ladder.type)) {
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            links: [],
+            total_count: 0,
+            internal_count: 0,
+            external_count: 0,
+            other_count: 0,
+            base_url: base_url || ladder.url,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; it has no HTML links']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
-    const finalUrl = response.url || url;
-    const pageUrl = new URL(finalUrl);
+    const finalUrl = ladder.url;
 
-    // <base href>, if present, overrides the page URL as the resolution base
-    // for relative links (but an explicit base_url override wins over both).
+    // Reported base: an explicit base_url, else <base href> resolved against
+    // the page, else the page URL (extractLinkRecords resolves the same way).
     let docBase = finalUrl;
     const baseHref = $('base[href]').first().attr('href');
     if (baseHref) {
       try { docBase = new URL(baseHref, finalUrl).toString(); } catch { /* ignore invalid <base href> */ }
     }
-
     const baseUrl = base_url || docBase;
-    const links = [];
-
-    $('a[href]').each((_, element) => {
-      const href = $(element).attr('href');
-      const text = $(element).text().trim();
-
-      // A javascript: pseudo-link ("Cookie Settings") is a button, not a
-      // link; it was counted as an external link on boeing.com (R20).
-      if (!href || /^\s*javascript:/i.test(href)) return;
 
-      try {
-        const absoluteUrl = new URL(href, baseUrl).toString();
-        const isExternal = new URL(absoluteUrl).origin !== pageUrl.origin;
-
-        if (filter_external && !isExternal) return;
-
-        links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
-      } catch {
-        // skip invalid URLs
-      }
-    });
-
-    const uniqueLinks = links.filter((link, index, arr) =>
-      arr.findIndex(l => l.href === link.href) === index
-    );
+    // Records are deduplicated on the URL without fragment or trailing slash;
+    // mailto:/tel:/javascript: links are type "other". filter_external drops
+    // only the internal ones.
+    const records = extractLinkRecords($, { pageUrl: finalUrl, baseUrl: base_url });
+    const links = filter_external ? records.filter(l => l.type !== 'internal') : records;
+    const count = (type) => links.filter(l => l.type === type).length;
 
     return {
       content: [{
         type: 'text',
         text: JSON.stringify({
-          links: uniqueLinks,
-          total_count: uniqueLinks.length,
-          internal_count: uniqueLinks.filter(l => !l.is_external).length,
-          external_count: uniqueLinks.filter(l => l.is_external).length,
-          base_url: baseUrl
+          links,
+          total_count: links.length,
+          internal_count: count('internal'),
+          external_count: count('external'),
+          other_count: count('other'),
+          base_url: baseUrl,
+          ...ladder.fields,
+          ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
         }, null, 2)
       }]
     };
@@ -81,3 +89,15 @@ export async function extractLinksHandler({ url, filter_external, base_url, user
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractLinksHandler({ escalateFetch } = {}) {
+  return (params) => extractLinks(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractLinksHandler = createExtractLinksHandler();

package/src/tools/basic/extractText.js

@@ -3,12 +3,15 @@
  * Extracted from server.js inline handler.
  * D3.1: Added output_format:"markdown" option backed by Turndown.
  * B1: Preserve block structure for text mode; use Readability + GFM for markdown mode.
+ * E3: Text mode reads through crawlforge-extractors' flattenText, as the REST
+ * route does; `selector` and `max_length` take the REST route's meaning.
  */
 
 import { load } from 'cheerio';
+import { flattenText } from 'crawlforge-extractors';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
-import { fetchWithTimeout } from './_fetch.js';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
@@ -73,22 +76,61 @@ export function readabilityToMarkdown(html, pageUrl) {
   return htmlToMarkdown(articleHtml);
 }
 
+/**
+ * Cut text to max_length characters and mark the cut with '...' (the REST rule).
+ * @param {string} text
+ * @param {number} [maxLength]
+ * @returns {string}
+ */
+function truncate(text, maxLength) {
+  return maxLength && text.length > maxLength ? text.substring(0, maxLength) + '...' : text;
+}
+
 /**
  * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
- *   output_format?: "text"|"markdown", user_agent?: string, respect_robots?: boolean }} params
+ *   output_format?: "text"|"markdown", selector?: string, max_length?: number,
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractTextHandler({ url, remove_scripts, remove_styles, output_format, user_agent, respect_robots }) {
+async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_text',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_text'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_text', 'Failed to extract text: ', ladder);
+
+    const extras = {
+      ...ladder.fields,
+      ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
+    };
+
+    // A JSON body has no markup to strip: it is returned as it came.
+    if (isJsonType(ladder.type)) {
+      const body = truncate(ladder.html, max_length);
+      const format = output_format === 'markdown' ? 'markdown' : 'text';
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            url: ladder.url,
+            [format]: body,
+            output_format: format,
+            word_count: body.split(/\s+/).filter(w => w.length > 0).length,
+            char_count: body.length,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; its body is returned as text']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
     if (remove_scripts !== false) $('script').remove();
@@ -101,21 +143,32 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     // enabled never render noscript content, so always strip it.
     $('noscript').remove();
 
-    $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    // A selector names the content, so the boilerplate strip is skipped.
+    let $target;
+    if (selector) {
+      $target = $(selector);
+      if ($target.length === 0) throw new Error(`No elements found for selector: ${selector}`);
+    } else {
+      $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    }
 
     const result = {
-      url: response.url
+      url: ladder.url
     };
 
     if (output_format === 'markdown') {
-      // Run Readability first to get main content, then convert to GFM markdown
-      result.markdown = readabilityToMarkdown(html, response.url);
+      // Run Readability first to get main content, then convert to GFM markdown;
+      // a selector's matches are converted as they are.
+      const markdown = $target
+        ? htmlToMarkdown($target.toArray().map(el => $.html(el)).join('\n'))
+        : readabilityToMarkdown(html, ladder.url);
+      result.markdown = truncate(markdown, max_length);
       result.output_format = 'markdown';
       const plainText = result.markdown.replace(/[#*`_\[\]]/g, '').replace(/\s+/g, ' ').trim();
       result.word_count = plainText.split(/\s+/).filter(w => w.length > 0).length;
       result.char_count = plainText.length;
     } else {
-      const text = extractBlockText($);
+      const text = truncate(flattenText($, $target), max_length);
       result.text = text;
       result.output_format = 'text';
       result.word_count = text.split(/\s+/).filter(w => w.length > 0).length;
@@ -125,7 +178,7 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     return {
       content: [{
         type: 'text',
-        text: JSON.stringify(result, null, 2)
+        text: JSON.stringify({ ...result, ...extras }, null, 2)
       }]
     };
   } catch (error) {
@@ -135,3 +188,15 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractTextHandler({ escalateFetch } = {}) {
+  return (params) => extractText(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractTextHandler = createExtractTextHandler();

package/src/tools/extract/_fetchAndParse.js

@@ -11,7 +11,7 @@
  */
 
 import { load } from 'cheerio';
-import { readBody } from 'crawlforge-extractors';
+import { readBody, flattenText } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -83,35 +83,11 @@ function classifyContentType(contentType) {
 }
 
 /**
- * Flatten a parsed document's <body> to text while preserving line structure.
- *
- * Cheerio's .text() joins adjacent elements with no separator, which welds
- * table rows / list items together ("1.Story title329 points") and starves
- * downstream LLM extraction (AgentOrchestrator, extractStructured,
- * extractWithLlm) of any structure to parse. Works on a detached clone so the
- * caller's $ tree is untouched — unifiedScrape reuses $ for other formats.
- *
- * @param {import('cheerio').CheerioAPI} $
- * @returns {string}
+ * Flatten a parsed document's <body> to text, one line per block element.
+ * The flattener moved to crawlforge-extractors (`flattenText`) so the REST
+ * extract_text route shares it; this name stays for extractStructured.
  */
-export function flattenBodyText($) {
-  // Block boundaries are marked with a U+E000 private-use sentinel so the
-  // HTML source's own insignificant newlines can be collapsed to spaces
-  // first, and only the sentinels become line breaks. (NUL won't survive:
-  // .after()/.replaceWith() parse their argument as HTML and the parser
-  // strips NUL; U+E000 passes through and never occurs in real page text.)
-  const $body = $('body').clone();
-  $body.find('br').replaceWith('\uE000');
-  $body.find('td, th').after(' ');
-  $body
-    .find('p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, section, article, header, footer')
-    .after('\uE000');
-  return $body
-    .text()
-    .replace(/\s+/g, ' ')
-    .replace(/ ?(?:\uE000 ?)+/g, '\n')
-    .trim();
-}
+export { flattenText as flattenBodyText };
 
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
@@ -190,7 +166,7 @@ export async function fetchAndParse(url, options = {}) {
     $(stripTags.join(', ')).remove();
   }
 
-  const textContent = flattenBodyText($);
+  const textContent = flattenText($);
 
   return { html, $, textContent, finalUrl: response.url, warnings, status: response.status };
 }

package/src/utils/fetchLadder.js

@@ -0,0 +1,228 @@
+/**
+ * fetchLadder — scrape's fetch ladder for extract_text and extract_links (fix
+ * plan Phase E2). Same rules and wording as the website's
+ * src/lib/tools/fetch-ladder.ts — keep the two in step.
+ *
+ *   1. A plain fetch (15 s), retried once when a 429/503 asks for at most
+ *      10 s with Retry-After.
+ *   2. A 2xx body that is not text (a PDF, an image, an archive) stops here as
+ *      UNSUPPORTED_CONTENT_TYPE; it never escalates.
+ *   3. Only with escalate:true, and only when a browser could change the
+ *      outcome (aBrowserMightPass, or a 444 refusal): the injected stealth
+ *      stage — server.js `stealthEscalation`, the one scrape and
+ *      extract_embedded_state run, behind the same compliance gate (impit
+ *      first under "auto", then the browser).
+ *   4. Otherwise E1's verdict stands unchanged (targetFailure).
+ *
+ * scrape and extract_embedded_state keep their own copies of steps 3–4; this
+ * reuses their exported pieces rather than moving them.
+ */
+
+import { load } from 'cheerio';
+import { z } from 'zod';
+import { fetchWithTimeout } from '../tools/basic/_fetch.js';
+import { plainVerdict, targetFailure } from '../tools/basic/_targetFailure.js';
+import { stealthDocumentVerdict } from './stealthVerdict.js';
+import { parseRetryAfter } from './hostRateLimiter.js';
+import { setActualCost } from '../server/requestContext.js';
+import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from '../tools/scrape/escalation.js';
+
+/** extract_text's and extract_links's table price; escalation adds SCRAPE_ESCALATION_CREDITS. */
+const EXTRACT_BASE_CREDITS = 1;
+
+export const LADDER_TIMEOUT_MS = 15000;
+
+/** A Retry-After longer than this is not waited out inside one call. */
+export const RETRY_AFTER_MAX_MS = 10000;
+
+export const CLIENT_RENDERED_WARNING = 'page renders client-side; use scrape';
+
+/** The two public fields both tools take, in tools/list order. */
+export const EXTRACT_ESCALATION_SHAPE = {
+  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser and extract from the rendered document. Under escalate_engine "auto" a Chrome TLS handshake with the honest CrawlForge User-Agent (impit) is tried first, and the browser runs only when that does not get the page. A 404 or 5xx never escalates. Projected at 1+5; the actual charge stays at 1 when the plain fetch succeeded. Default: false'),
+  escalate_engine: SCRAPE_ESCALATION_SHAPE.escalate_engine
+};
+
+// Bodies these tools read. A missing Content-Type is read as HTML.
+const TEXTUAL_TYPE = /^(?:text\/[\w.+-]+|application\/(?:xhtml\+xml|xml|json)|application\/[\w.+-]+\+(?:xml|json))$/;
+const JSON_TYPE = /^application\/(?:[\w.+-]+\+)?json$/;
+
+/** The media type without parameters, lowercased; '' when absent. */
+function mediaType(response) {
+  return (response.headers?.get?.('content-type') ?? '').split(';')[0].trim().toLowerCase();
+}
+
+/** Whether a media type is JSON (application/json or a +json suffix). */
+export function isJsonType(type) {
+  return JSON_TYPE.test(type);
+}
+
+// A page that says it needs JavaScript (scrapingcourse.com's
+// /javascript-rendering serves "Enable JavaScript to see products" over empty
+// product cards, with no mount point to find).
+// No \b before the verbs: cheerio's text() joins <h1>Challenge</h1><p>Enable…
+// into one word.
+const JS_REQUIRED = /(?:enable|requires?|turn on)\s+javascript\b|\bjavascript\s+(?:is\s+)?(?:required|disabled)\b/i;
+
+/**
+ * A client-rendered shell: under 200 characters of visible text, and either an
+ * empty <div id="root">, "app" or "__next" for the framework to mount into or
+ * text saying the page needs JavaScript.
+ * @param {string} html
+ */
+export function isClientRenderedShell(html) {
+  const $ = load(html);
+  $('script, style, noscript').remove();
+  const text = $('body').text().replace(/\s+/g, ' ').trim();
+  if (text.length >= 200) return false;
+  if (JS_REQUIRED.test(text)) return true;
+  return $('div#root, div#app, div#__next').toArray().some((el) => $(el).text().trim() === '');
+}
+
+/**
+ * Retry-After in ms, or null when the header is absent or unreadable.
+ * @param {Response} response
+ */
+function retryAfterMs(response) {
+  const header = response.headers?.get?.('retry-after');
+  if (header == null) return null;
+  const raw = String(header).trim();
+  if (!/^\d+$/.test(raw) && Number.isNaN(Date.parse(raw))) return null;
+  return parseRetryAfter(raw);
+}
+
+/**
+ * The plain fetch, retried once on a 429/503 whose Retry-After is at most
+ * RETRY_AFTER_MAX_MS. fetchWithTimeout has already recorded the Retry-After
+ * (noteRetryAfter), and the second request's gate waits it out in the
+ * per-host throttle — so the wait happens once, there.
+ */
+async function plainFetch(url, options) {
+  const response = await fetchWithTimeout(url, options);
+  if (response.status !== 429 && response.status !== 503) return response;
+  const waitMs = retryAfterMs(response);
+  if (waitMs === null || waitMs > RETRY_AFTER_MAX_MS) return response;
+  return fetchWithTimeout(url, options);
+}
+
+/**
+ * @param {string} url
+ * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean,
+ *   escalate?: boolean, escalateEngine?: string,
+ *   escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} options
+ * @returns {Promise<
+ *   { unsupported: string } |
+ *   { error: string, url: string, status?: number|null, blocked?: object, warnings: string[], fields: object } |
+ *   { html: string, url: string, type: string, warnings: string[], fields: object }>}
+ *   `fields` carries `escalated` (only when escalate was asked for), `stealth`
+ *   (when the stage ran) and `rendered: false` (a client-rendered shell)
+ */
+export async function fetchLadder(url, { tool, userAgent, respectRobots, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
+  let escalationRan = false;
+  try {
+    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, tool });
+    const type = mediaType(response);
+    if (response.ok && type && !TEXTUAL_TYPE.test(type)) return { unsupported: type };
+
+    const warnings = [...(response._warnings ?? [])];
+    // Reported only to a caller who asked to escalate.
+    const fields = escalate ? { escalated: false } : {};
+    const verdict = plainVerdict(response, url);
+    const failed = verdict ? !verdict.success : !response.ok;
+
+    // A 404 or a 5xx never escalates (aBrowserMightPass); a 444 is a refusal
+    // of our network, which a browser on another identity can pass. A host
+    // whose Retry-After is still open (over 10 s, or repeated on the retry)
+    // asked us to back off, and a stealth request now would override it (G6).
+    const backingOff = (response.status === 429 || response.status === 503) && retryAfterMs(response) !== null;
+    if (escalate && failed && !backingOff && (aBrowserMightPass(verdict, response.status) || response.status === 444)) {
+      const vendorDetected = verdict?.blocked?.vendor ?? null;
+      if (!escalateFetch) {
+        warnings.push('escalate: no stealth stage is wired into this server build');
+      } else {
+        try {
+          const stealth = await escalateFetch({ url: response.url || url, engine: escalateEngine, respectRobots });
+          escalationRan = true;
+          const engine = stealth.engine || escalateEngine;
+          // The stage's gate repeats the plain fetch's robots notes; say each once.
+          for (const w of stealth.warnings ?? []) if (!warnings.includes(w)) warnings.push(w);
+          const html = stealth.html || '';
+          const finalUrl = stealth.url || response.url || url;
+          const status = stealth.status ?? null;
+          const rendered = stealthDocumentVerdict(
+            { url: finalUrl, title: stealth.title || '', text: stealth.text || '', html, status },
+            { waitedMs: stealth.gracedMs || 0, fetcher: 'the stealth browser', rendered: true, contentReturned: false }
+          );
+          warnings.push(
+            `escalate: the plain fetch ${vendorDetected ? `was blocked by ${vendorDetected}` : 'did not return the page'}; ` +
+            `${engine === 'impit' ? 'a Chrome TLS handshake (impit, honest User-Agent)' : `the ${engine} stealth browser`} ${rendered.success ? 'returned it' : 'did not get it either'}`
+          );
+          fields.escalated = true;
+          fields.stealth = { engine, vendor_detected: vendorDetected };
+          if (!rendered.success) {
+            return { error: rendered.error, url: finalUrl, status, ...(rendered.blocked ? { blocked: rendered.blocked } : {}), warnings, fields };
+          }
+          return { html, url: finalUrl, type: 'text/html', warnings, fields };
+        } catch (err) {
+          // A robots refusal inside the gate stamps markPreflightRefusal, and
+          // withAuth then bills the whole call zero: nothing was rendered.
+          warnings.push(`escalate: the stealth retry did not run — ${err.message}`);
+        }
+      }
+    }
+
+    const failure = targetFailure(response, url, verdict);
+    if (failure) {
+      return { error: failure, url: response.url || url, status: response.status, ...(verdict?.blocked ? { blocked: verdict.blocked } : {}), warnings, fields };
+    }
+
+    const html = response._body ?? '';
+    if (!isJsonType(type) && isClientRenderedShell(html)) {
+      warnings.push(CLIENT_RENDERED_WARNING);
+      fields.rendered = false;
+    }
+    return { html, url: response.url || url, type, warnings, fields };
+  } finally {
+    // The projection is 1+5 when escalating; the stage's 5 is owed only if it ran.
+    if (escalate) setActualCost(EXTRACT_BASE_CREDITS + (escalationRan ? SCRAPE_ESCALATION_CREDITS : 0));
+  }
+}
+
+/**
+ * The error result for a ladder that did not return a page. Unescalated, the
+ * E1 text error; with escalate asked for, a JSON body carrying `escalated`,
+ * so withAuth's hint does not send the caller back to the stealth browser.
+ * @param {string} tool
+ * @param {string} prefix e.g. 'Failed to extract text: '
+ * @param {object} ladder fetchLadder's result
+ */
+export function ladderErrorResult(tool, prefix, ladder) {
+  if (ladder.unsupported) {
+    return {
+      content: [{
+        type: 'text',
+        text: `${prefix}UNSUPPORTED_CONTENT_TYPE: Unsupported content type ${ladder.unsupported}: ${tool} reads ${tool === 'extract_links' ? 'HTML' : 'HTML and text'}\n` +
+          'Next step: process_document reads PDF and DOCX documents from a URL (2 credits); an image or an archive has no text to extract.'
+      }],
+      isError: true
+    };
+  }
+  if (!('escalated' in ladder.fields)) {
+    return { content: [{ type: 'text', text: `${prefix}${ladder.error}` }], isError: true };
+  }
+  return {
+    content: [{
+      type: 'text',
+      text: JSON.stringify({
+        success: false,
+        url: ladder.url,
+        ...(typeof ladder.status === 'number' ? { status: ladder.status } : {}),
+        error: ladder.error,
+        ...(ladder.blocked ? { blocked: ladder.blocked } : {}),
+        ...ladder.fields,
+        ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
+      }, null, 2)
+    }],
+    isError: true
+  };
+}