crawlforge-mcp-server 6.16.0 → 6.18.0

package/src/server/inlineThreshold.js

@@ -20,8 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `when` gates
- * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * on. An empty list means the pretty-printed JSON is the view. `keep` names
+ * top-level fields carried inline past the scalar filter (keepNamedFields).
+ * `when` gates on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
  * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
@@ -37,7 +38,14 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   // an async job's page came back as 111 KB whole (R20, 2026-09-07).
   get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
-  scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  // `keep` names the top-level reports that stay inline past the scalar
+  // filter (keepNamedFields): a walled Guardian chain over 40,000 chars lost
+  // `blocked`, `navigations` and every action's outcome (2026-10-03).
+  scrape_with_actions: {
+    textPaths: ['content.markdown', 'content.text', 'content.html'],
+    truncate: true,
+    keep: ['error', 'blocked', 'consent', 'navigations', 'actionResults']
+  },
   // `read` hands back the same content shape scrape_with_actions does, and was
   // the one content-returning tool with no cap: a read of the World War II
   // article returned 541,308 characters inline where scrape returned 42,259
@@ -48,6 +56,7 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   browser_session: {
     textPaths: ['content.markdown', 'content.text', 'content.html', 'snapshot.tree'],
     truncate: true,
+    keep: ['error', 'blocked', 'actionResults'],
     when: (params) => BROWSER_SESSION_CONTENT_OPERATIONS.has(params?.operation)
   },
   process_document: { textPaths: ['content.text'], truncate: true },
@@ -111,6 +120,42 @@ export function keepSmallContentFields(content, textPaths = [], maxInline = DEFA
   return Object.keys(kept).length > 0 ? kept : null;
 }
 
+/**
+ * The top-level fields a tool names in `keep`, each carried inline when its
+ * JSON fits a quarter of the inline budget. An actionResults list over that
+ * keeps every entry without its `result` (a snapshot tree, an extracted
+ * value), so which action ran and whether it worked stays inline.
+ * @returns {{ kept: object|null, partial: string[], dropped: string[] }}
+ */
+export function keepNamedFields(resultObject, keys = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  const partial = [];
+  const dropped = [];
+  for (const key of keys) {
+    const value = resultObject[key];
+    if (value === undefined) continue;
+    if (JSON.stringify(value).length <= cap) {
+      kept[key] = value;
+      continue;
+    }
+    if (key === 'actionResults' && Array.isArray(value)) {
+      const slim = value.map((entry) => {
+        if (!entry || typeof entry !== 'object') return entry;
+        const { result: _result, ...rest } = entry;
+        return rest;
+      });
+      if (JSON.stringify(slim).length <= cap) {
+        kept[key] = slim;
+        partial.push(key);
+        continue;
+      }
+    }
+    dropped.push(key);
+  }
+  return { kept: Object.keys(kept).length > 0 ? kept : null, partial, dropped };
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -156,7 +201,16 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
 
   const preview = text.slice(0, maxInline);
   const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
-  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const named = keepNamedFields(resultObject, config.keep, maxInline);
+  const keptNames = [
+    ...(named.kept ? Object.keys(named.kept).filter((key) => !named.partial.includes(key)) : []),
+    ...(keptFields ? Object.keys(keptFields).map((key) => `content.${key}`) : [])
+  ];
+  const keptDesc = [
+    keptNames.length > 0 ? `; ${keptNames.join(', ')} kept inline` : '',
+    named.partial.includes('actionResults') ? '; actionResults kept without each action\'s result (json_path "actionResults[<i>].result" reads one)' : '',
+    named.dropped.length > 0 ? `; ${named.dropped.join(', ')} too large to keep inline (json_path "${named.dropped[0]}" reads it)` : ''
+  ].join('');
   const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
@@ -171,6 +225,7 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  if (named.kept) Object.assign(shaped, named.kept);
   // The query-scoped formats live beside the page text under `content`
   // (highlights, answer, json, metadata, links). They are the small, exact
   // answer the caller paid for, and truncating the markdown must not drop
@@ -191,6 +246,14 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   return { result: shaped, stored: true };
 }
 
+/**
+ * Room for the `_cost` block withAuth adds to the text after shaping:
+ * pretty-printed, projected/actual/remaining_credits and the projection
+ * note, whose longest form for extract_embedded_state (escalate:true) is
+ * about 200 chars, about 340 in all.
+ */
+export const COST_BLOCK_RESERVE = 400;
+
 /** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
 export function embeddedDataKeys(data) {
   if (Array.isArray(data)) return `array(${data.length})`;
@@ -227,10 +290,12 @@ export function wholeLinePreview(text, budget) {
  * everything that says what the page carries — url, path, bytes, the whole
  * `found` list, data_keys, the escalation report and window_state's list —
  * then a preview of the pretty-printed JSON cut at a line boundary. The
- * preview gets whatever max_inline_chars leaves after the rest, measured as
- * compact JSON, so the whole inline result stays within the limit whenever
- * the rest fits. The full result is under the handle for read_result
- * json_path. The REST route shapes it the same way; keep the two in step.
+ * preview gets whatever max_inline_chars leaves after the rest, measured the
+ * way withAuth sends it — pretty-printed, with COST_BLOCK_RESERVE held back
+ * for the `_cost` block it adds after shaping — so the text the client
+ * receives stays within the limit whenever the rest fits. The full result is
+ * under the handle for read_result json_path. The REST route shapes it the
+ * same way, measured as the compact JSON it sends; keep the two in step.
  */
 function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
   const dataKeys = embeddedDataKeys(resultObject.data);
@@ -257,8 +322,9 @@ function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expir
     expires_at,
     warnings: [...warningsOf(resultObject), hint]
   };
-  // `"preview":"",` is what the key itself adds once the value is in.
-  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  // The preview's value is one line of the pretty-printed text and escapes
+  // the same way there as in compact JSON; `""` is its empty value's quotes.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }, null, 2).length + COST_BLOCK_RESERVE;
   const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
   return { ...shaped, preview, ...tail };
 }

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -8,8 +8,8 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 | Tool | Notes |
 |------|-------|
 | `fetch_url` | Raw HTTP fetch. |
-| `extract_text` | Tag-stripped text/markdown. |
-| `extract_links` | All links on a page. |
+| `extract_text` | Tag-stripped text/markdown. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
+| `extract_links` | All links on a page. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
 | `extract_metadata` | Title / meta / OG / schema.org. |
 | `scrape_template` | Pre-built site extractor. |
 | `list_ollama_models` | List local LLMs. |

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -158,6 +158,10 @@ Returns headers + body. Good for JSON/XML APIs or as a first step before
 `extract_text` (cost 1) = faster, strips tags; pass `output_format:"markdown"`
 for RAG-friendly output.
 
+`extract_text` and `extract_links` take `escalate:true` for a site known to
+block: the plain fetch runs first and the stealth browser re-reads the page
+only if it is walled (projected 6, charged 1 when the plain fetch worked).
+
 ## extract_links / extract_metadata (cost: 1 each)
 
 ```json

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -34,14 +34,28 @@ your provider).
 | `remove_scripts` | boolean | `true` | Strip `script` tags. |
 | `remove_styles` | boolean | `true` | Strip `style` tags. |
 | `output_format` | enum | `text` | `text` or `markdown` (use markdown for RAG). |
+| `selector` | string | — | CSS selector: read only the matched elements (nav/header/footer inside them are kept). No match is an error: `No elements found for selector: <sel>`. |
+| `max_length` | integer | — | 1–1,000,000. Text or markdown longer than this is cut and ends with `...`; `word_count`/`char_count` count the cut result. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Text mode returns one line per block element (`<h1>Hi</h1><p>there</p>` reads `Hi\nthere`). The REST route (`/api/v1/tools/extract_text`) takes the same parameters plus `preserve_links` and `preserve_formatting`, which the MCP tool does not.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_links (cost: 1)
 
 | Param | Type | Default | Notes |
 |-------|------|---------|-------|
 | `url` | string (URL) | — | Required. |
-| `filter_external` | boolean | `false` | Only return outbound links. |
-| `base_url` | string (URL) | — | Resolve relative links against this. |
+| `filter_external` | boolean | `false` | Drop internal (same-host) links; external and `other` links stay. |
+| `base_url` | string (URL) | — | Resolve relative links against this. Without it, the page's `<base href>`, else the page URL. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Each link is `{ href, text, type, domain, rel, original_href }` (plus `title` when the link has one). `type` is `internal` (same host as the page), `external` or `other` (`mailto:`, `tel:`, `javascript:` and other non-web schemes; only `mailto:`/`tel:` keep an `href`, the rest are `null`). `#fragment`-only links are skipped, and links are deduplicated on the URL without its fragment or trailing slash. Counts: `total_count`, `internal_count`, `external_count`, `other_count`. The REST route returns the same records.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_metadata (cost: 1)
 

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -14,7 +14,7 @@ import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
-import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
+import { resolveStealthEngine, redactProxyEntry, assertProxyEngineAllowed } from '../../core/StealthBrowserManager.js';
 import { actionQueue as sharedActionQueue } from '../../core/browser/actionQueue.js';
 
 // Recording / replay helpers
@@ -27,6 +27,20 @@ import {
   recordedEntryToAction
 } from './scrapeWithActions/recorder.js';
 
+// metadata.browserOptions echoes the merged options, and a proxy entry carries
+// its password.
+function redactProxyCredentials(browserOptions) {
+  const proxies = browserOptions.proxyRotation?.proxies;
+  if (!proxies) return browserOptions;
+  return {
+    ...browserOptions,
+    proxyRotation: {
+      ...browserOptions.proxyRotation,
+      proxies: proxies.map(redactProxyEntry)
+    }
+  };
+}
+
 // Action schemas (re-using from ActionExecutor but with tool-specific additions)
 const BaseActionSchema = z.object({
   type: z.string(),
@@ -203,7 +217,15 @@ const ScrapeWithActionsSchema = z.object({
     // Answer a cookie/consent wall (autoconsent) after the initial load and
     // after each navigate action: "reject" opts out, "accept" opts in. Off by
     // default so a chain that clicks the banner itself keeps working.
-    consent: z.enum(['off', 'reject', 'accept']).default('off')
+    consent: z.enum(['off', 'reject', 'accept']).default('off'),
+    // The caller's own proxies for a stealth chain — the shape and parsing of
+    // stealth_mode's stealthConfig.proxyRotation. Refused without stealth:true
+    // (see executeSession); CrawlForge supplies none.
+    proxyRotation: z.object({
+      enabled: z.boolean().default(false),
+      proxies: z.array(z.string()).optional(),
+      rotationInterval: z.number().default(300000)
+    }).optional()
   }).optional(),
 
   // Content extraction options
@@ -425,6 +447,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
     const warnings = [];
     if (browserOptions.stealth) {
       const resolved = await resolveStealthEngine(browserOptions.engine);
+      // Refused on the engine that will run, before the chain starts.
+      assertProxyEngineAllowed(resolved.engine, browserOptions);
       browserOptions.stealthMode = { enabled: true, engine: resolved.engine };
       if (resolved.fallbackWarning) warnings.push(resolved.fallbackWarning);
     } else if (browserOptions.engine === 'camoufox') {
@@ -439,6 +463,15 @@ export class ScrapeWithActionsTool extends EventEmitter {
         'to use the standard Chromium browser.'
       );
     }
+    if (browserOptions.proxyRotation?.enabled && !browserOptions.stealth) {
+      // Only the stealth browser takes a proxy; the standard pool would run
+      // the chain from this server's own address while the caller believes
+      // it is proxied.
+      throw new Error(
+        'browserOptions.proxyRotation requires browserOptions.stealth:true — only the stealth ' +
+        'browser routes through a proxy. Set stealth:true, or drop proxyRotation.'
+      );
+    }
 
     // The robots override rides with the browser options so ActionExecutor's
     // gate sees it on the initial load and on every `navigate` action.
@@ -534,7 +567,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
       text: finalContent?.content?.text || '',
       html: chainResult?.finalHtml || '',
       status: chainResult?.navigationStatus ?? null
-    }, { allowEmpty: true });
+    }, { allowEmpty: true, fetcher: browserOptions.stealthMode?.enabled ? 'the stealth browser' : 'the browser' });
 
     return {
       success: chainResult.success && verdict.success,
@@ -553,6 +586,9 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // The landing page's consent result; each navigate action's own is on
       // its result. Absent when consent handling is off.
       ...(chainResult.consent ? { consent: chainResult.consent } : {}),
+      // Every navigation of the reported attempt — [0] the initial load, then
+      // each navigate action — with its status and any wall it reached.
+      navigations: chainResult.navigations || [],
 
       actionResults,
       attempt: chainResult.attempt ?? attempts.length,
@@ -576,7 +612,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
       replayedFrom: params.replayRecording || undefined,
 
       metadata: {
-        browserOptions,
+        browserOptions: redactProxyCredentials(browserOptions),
         formAutoFillApplied: !!params.formAutoFill,
         intermediateStatesCount: intermediateStates.length,
         screenshotsCount: sessionContext.screenshots.length,

package/src/tools/basic/_targetFailure.js

@@ -0,0 +1,54 @@
+/**
+ * Why a fetched document fails extract_links / extract_text, with the same
+ * verdict and wording as the REST routes (fix plan Phase E1).
+ *
+ * A named vendor's wall fails on any status: a challenge arrives as a 403 or
+ * as a 200 with the wall in the body. Any other non-2xx fails with its
+ * status. An empty shell or an error placeholder on a 2xx is not raised here;
+ * extraction proceeds as before.
+ */
+
+import { load } from 'cheerio';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
+import { pageTitle } from '../../utils/pageTitle.js';
+
+// Response bodies a verdict can be read from; anything else is a binary. A
+// missing Content-Type is read, as _fetchAndParse.js treats it as HTML.
+const TEXTUAL_BODY = /^(?:text\/(?:html|plain)|application\/xhtml\+xml)\b/i;
+
+/**
+ * The plain fetch's verdict on a document, or null for a body a verdict
+ * cannot be read from (a binary, JSON). The fetch ladder reads it to decide
+ * whether escalation could help (src/utils/fetchLadder.js).
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ */
+export function plainVerdict(response, url) {
+  const contentType = response.headers?.get?.('content-type');
+  if (contentType && !TEXTUAL_BODY.test(contentType)) return null;
+  // A parse of its own, scripts and styles removed, so the handler's
+  // document stays intact.
+  const $ = load(response._body ?? '');
+  $('script, style, noscript').remove();
+  return stealthDocumentVerdict(
+    {
+      url: response.url || url,
+      status: response.status,
+      title: pageTitle($),
+      text: $('body').text().replace(/\s+/g, ' ').trim(),
+      html: response._body
+    },
+    { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+  );
+}
+
+/**
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ * @param {object|null} [verdict] - plainVerdict's, when the caller already has it
+ * @returns {string|null} the error message, or null when the document is usable
+ */
+export function targetFailure(response, url, verdict = plainVerdict(response, url)) {
+  if (verdict?.blocked) return `Target answered HTTP ${response.status}: ${verdict.error}`;
+  return response.ok ? null : `Target answered HTTP ${response.status}`;
+}

package/src/tools/basic/extractLinks.js

@@ -1,76 +1,84 @@
 /**
  * extract_links — Extract all links from a webpage with optional filtering.
  * Extracted from server.js inline handler.
+ * E3: Link records come from crawlforge-extractors' extractLinkRecords, the
+ * reader the REST route uses, so both surfaces return the same links.
  */
 
 import { load } from 'cheerio';
-import { fetchWithTimeout } from './_fetch.js';
+import { extractLinkRecords } from 'crawlforge-extractors';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 
 /**
  * @param {{ url: string, filter_external?: boolean, base_url?: string,
- *   user_agent?: string, respect_robots?: boolean }} params
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractLinksHandler({ url, filter_external, base_url, user_agent, respect_robots }) {
+async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_links',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_links'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_links', 'Failed to extract links: ', ladder);
+
+    // A JSON body carries no <a href>.
+    if (isJsonType(ladder.type)) {
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            links: [],
+            total_count: 0,
+            internal_count: 0,
+            external_count: 0,
+            other_count: 0,
+            base_url: base_url || ladder.url,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; it has no HTML links']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
-    const finalUrl = response.url || url;
-    const pageUrl = new URL(finalUrl);
+    const finalUrl = ladder.url;
 
-    // <base href>, if present, overrides the page URL as the resolution base
-    // for relative links (but an explicit base_url override wins over both).
+    // Reported base: an explicit base_url, else <base href> resolved against
+    // the page, else the page URL (extractLinkRecords resolves the same way).
     let docBase = finalUrl;
     const baseHref = $('base[href]').first().attr('href');
     if (baseHref) {
       try { docBase = new URL(baseHref, finalUrl).toString(); } catch { /* ignore invalid <base href> */ }
     }
-
     const baseUrl = base_url || docBase;
-    const links = [];
-
-    $('a[href]').each((_, element) => {
-      const href = $(element).attr('href');
-      const text = $(element).text().trim();
-
-      // A javascript: pseudo-link ("Cookie Settings") is a button, not a
-      // link; it was counted as an external link on boeing.com (R20).
-      if (!href || /^\s*javascript:/i.test(href)) return;
 
-      try {
-        const absoluteUrl = new URL(href, baseUrl).toString();
-        const isExternal = new URL(absoluteUrl).origin !== pageUrl.origin;
-
-        if (filter_external && !isExternal) return;
-
-        links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
-      } catch {
-        // skip invalid URLs
-      }
-    });
-
-    const uniqueLinks = links.filter((link, index, arr) =>
-      arr.findIndex(l => l.href === link.href) === index
-    );
+    // Records are deduplicated on the URL without fragment or trailing slash;
+    // mailto:/tel:/javascript: links are type "other". filter_external drops
+    // only the internal ones.
+    const records = extractLinkRecords($, { pageUrl: finalUrl, baseUrl: base_url });
+    const links = filter_external ? records.filter(l => l.type !== 'internal') : records;
+    const count = (type) => links.filter(l => l.type === type).length;
 
     return {
       content: [{
         type: 'text',
         text: JSON.stringify({
-          links: uniqueLinks,
-          total_count: uniqueLinks.length,
-          internal_count: uniqueLinks.filter(l => !l.is_external).length,
-          external_count: uniqueLinks.filter(l => l.is_external).length,
-          base_url: baseUrl
+          links,
+          total_count: links.length,
+          internal_count: count('internal'),
+          external_count: count('external'),
+          other_count: count('other'),
+          base_url: baseUrl,
+          ...ladder.fields,
+          ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
         }, null, 2)
       }]
     };
@@ -81,3 +89,15 @@ export async function extractLinksHandler({ url, filter_external, base_url, user
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractLinksHandler({ escalateFetch } = {}) {
+  return (params) => extractLinks(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractLinksHandler = createExtractLinksHandler();

package/src/tools/basic/extractText.js

@@ -3,12 +3,15 @@
  * Extracted from server.js inline handler.
  * D3.1: Added output_format:"markdown" option backed by Turndown.
  * B1: Preserve block structure for text mode; use Readability + GFM for markdown mode.
+ * E3: Text mode reads through crawlforge-extractors' flattenText, as the REST
+ * route does; `selector` and `max_length` take the REST route's meaning.
  */
 
 import { load } from 'cheerio';
+import { flattenText } from 'crawlforge-extractors';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
-import { fetchWithTimeout } from './_fetch.js';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
@@ -73,22 +76,61 @@ export function readabilityToMarkdown(html, pageUrl) {
   return htmlToMarkdown(articleHtml);
 }
 
+/**
+ * Cut text to max_length characters and mark the cut with '...' (the REST rule).
+ * @param {string} text
+ * @param {number} [maxLength]
+ * @returns {string}
+ */
+function truncate(text, maxLength) {
+  return maxLength && text.length > maxLength ? text.substring(0, maxLength) + '...' : text;
+}
+
 /**
  * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
- *   output_format?: "text"|"markdown", user_agent?: string, respect_robots?: boolean }} params
+ *   output_format?: "text"|"markdown", selector?: string, max_length?: number,
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractTextHandler({ url, remove_scripts, remove_styles, output_format, user_agent, respect_robots }) {
+async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_text',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_text'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_text', 'Failed to extract text: ', ladder);
+
+    const extras = {
+      ...ladder.fields,
+      ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
+    };
+
+    // A JSON body has no markup to strip: it is returned as it came.
+    if (isJsonType(ladder.type)) {
+      const body = truncate(ladder.html, max_length);
+      const format = output_format === 'markdown' ? 'markdown' : 'text';
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            url: ladder.url,
+            [format]: body,
+            output_format: format,
+            word_count: body.split(/\s+/).filter(w => w.length > 0).length,
+            char_count: body.length,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; its body is returned as text']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
     if (remove_scripts !== false) $('script').remove();
@@ -101,21 +143,32 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     // enabled never render noscript content, so always strip it.
     $('noscript').remove();
 
-    $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    // A selector names the content, so the boilerplate strip is skipped.
+    let $target;
+    if (selector) {
+      $target = $(selector);
+      if ($target.length === 0) throw new Error(`No elements found for selector: ${selector}`);
+    } else {
+      $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    }
 
     const result = {
-      url: response.url
+      url: ladder.url
     };
 
     if (output_format === 'markdown') {
-      // Run Readability first to get main content, then convert to GFM markdown
-      result.markdown = readabilityToMarkdown(html, response.url);
+      // Run Readability first to get main content, then convert to GFM markdown;
+      // a selector's matches are converted as they are.
+      const markdown = $target
+        ? htmlToMarkdown($target.toArray().map(el => $.html(el)).join('\n'))
+        : readabilityToMarkdown(html, ladder.url);
+      result.markdown = truncate(markdown, max_length);
       result.output_format = 'markdown';
       const plainText = result.markdown.replace(/[#*`_\[\]]/g, '').replace(/\s+/g, ' ').trim();
       result.word_count = plainText.split(/\s+/).filter(w => w.length > 0).length;
       result.char_count = plainText.length;
     } else {
-      const text = extractBlockText($);
+      const text = truncate(flattenText($, $target), max_length);
       result.text = text;
       result.output_format = 'text';
       result.word_count = text.split(/\s+/).filter(w => w.length > 0).length;
@@ -125,7 +178,7 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     return {
       content: [{
         type: 'text',
-        text: JSON.stringify(result, null, 2)
+        text: JSON.stringify({ ...result, ...extras }, null, 2)
       }]
     };
   } catch (error) {
@@ -135,3 +188,15 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractTextHandler({ escalateFetch } = {}) {
+  return (params) => extractText(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractTextHandler = createExtractTextHandler();