crawlforge-mcp-server 6.21.1 → 6.22.1

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 
@@ -32,20 +32,13 @@ Still never reach for `stealth_mode` first.
 
 ## stealth_mode (cost: 5)
 
-`stealth_mode` is operation-based. Typical flow: create a context, then create a
-page that navigates to the target URL.
+`stealth_mode` is operation-based. `operation:"scrape"` is the one-shot path:
+it renders one URL and returns the requested formats (examples below).
 
-```json
-{
-  "tool": "stealth_mode",
-  "params": {
-    "operation": "create_context",
-    "stealthConfig": { "level": "advanced", "simulateHumanBehavior": true }
-  }
-}
-```
-
-Then use the returned `contextId`:
+`create_context` then `create_page` is a navigation and challenge check, not a
+multi-step path: `create_page` loads `urlToTest`, reports its status, title and
+any challenge page it detected, and closes the page before the call returns.
+Nothing on that page carries over to a later call.
 
 ```json
 {
@@ -54,6 +47,10 @@ Then use the returned `contextId`:
 }
 ```
 
+For work over several calls on one page (log in, then read; click through a
+flow), use `browser_session` with `stealth:true` (see the
+crawlforge-browser-sessions skill).
+
 Operations: `scrape`, `configure`, `create_context`, `create_page`,
 `get_stats`, `cleanup`. `stealthConfig.level` is `basic` / `medium` (default) /
 `advanced`. Always run `cleanup` when done to release the browser.

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/tools/advanced/BrowserSessionTool.js

@@ -25,6 +25,7 @@
 
 import { z } from 'zod';
 import { load } from 'cheerio';
+import { absoluteUrls } from 'crawlforge-extractors';
 import { createHash } from 'node:crypto';
 
 import ActionExecutor from '../../core/ActionExecutor.js';
@@ -577,8 +578,16 @@ export class BrowserSessionTool {
     if (params.formats.includes('markdown')) {
       // Readability finds no article on most app pages, and then no markdown is
       // produced at all (R20, 2026-09-07). Convert the DOM we hold instead of
-      // handing back a placeholder.
-      content.markdown = (params.onlyMainContent && extracted.content?.markdown) || htmlToMarkdown(html);
+      // handing back a placeholder. The raw DOM carries no base URL, so its
+      // links are resolved against the page first: HN's "item?id=…" hrefs came
+      // back relative and unusable (R25, 2026-10-19).
+      let markdown = params.onlyMainContent && extracted.content?.markdown;
+      if (!markdown) {
+        const $page = load(html);
+        absoluteUrls($page, url);
+        markdown = htmlToMarkdown($page.html());
+      }
+      content.markdown = markdown;
     }
     if (params.formats.includes('json')) {
       content.json = {

package/src/tools/advanced/batchScrape/queue.js

@@ -3,7 +3,7 @@
  * Semaphore-based concurrency runner that dispatches work to worker.js.
  */
 
-import { scrapeUrl } from './worker.js';
+import { scrapeUrl, failedEntryHint } from './worker.js';
 
 /** Semaphore for concurrency limiting. */
 class Semaphore {
@@ -57,10 +57,12 @@ export async function scrapeUrlsBatch(urlConfigs, options, defaultTimeout) {
 
   return settled.map((result, index) => {
     if (result.status === 'fulfilled') return result.value;
+    const error = result.reason?.message || 'Unknown error';
     return {
       success: false,
       url: urlConfigs[index].url,
-      error: result.reason?.message || 'Unknown error',
+      error,
+      next_step: failedEntryHint(error),
       timestamp: Date.now()
     };
   });

package/src/tools/advanced/batchScrape/worker.js

@@ -11,10 +11,14 @@ import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { fetchResigned } from '../../../utils/resignedFetch.js';
+import { fetchErrorMessage } from '../../../utils/fetchError.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
+import { extractMainContent, isThinMainContent, keepPageHeading } from '../../scrape/_mainContent.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
 import { classifyContentType, unsupportedContentTypeError } from '../../extract/_fetchAndParse.js';
+import { requestContext } from '../../../server/requestContext.js';
+import { classifyError, CLASS_HINTS, FALLBACK_HINTS } from '../../../server/fallbackHints.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -57,7 +61,9 @@ export async function fetchUrl(url, options = {}) {
   } catch (error) {
     if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
     if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
-    throw error;
+    const message = fetchErrorMessage(error, url);
+    if (message === error.message) throw error;
+    throw new Error(message, { cause: error });
   } finally {
     // Cleared only after the body read finishes (or the fetch itself fails),
     // so the abort timer keeps protecting against a trickling body, not just
@@ -103,13 +109,14 @@ export async function scrapeUrl(config, options, defaultTimeout) {
       result.extracted = extractStructuredData($, { ...config.selectors, ...options.extractionSchema });
     }
 
-    result.content = generateFormats($, html, options.formats);
+    result.content = generateFormats($, html, options.formats, config.url);
     return result;
   } catch (error) {
     return {
       success: false,
       url: config.url,
       error: error.message,
+      next_step: failedEntryHint(error.message),
       timestamp: Date.now(),
       executionTime: Date.now() - startTime,
       metadata: config.metadata || {}
@@ -117,6 +124,17 @@ export async function scrapeUrl(config, options, defaultTimeout) {
   }
 }
 
+/**
+ * The next step for one failed URL (R25 3.1): its error class's hint, else
+ * scrape's. A batch with failed URLs still succeeds, so withAuth never hints
+ * it. Classified from the message alone: the invocation's preflight-refusal
+ * flag belongs to the whole batch, and would give every URL one URL's refusal.
+ */
+export function failedEntryHint(message) {
+  const c = requestContext.exit(() => classifyError(null, message));
+  return c ? CLASS_HINTS[c.cls]('batch_scrape', c.params) : FALLBACK_HINTS.scrape;
+}
+
 function extractStructuredData($, selectors) {
   const extracted = {};
   for (const [key, selector] of Object.entries(selectors)) {
@@ -144,7 +162,7 @@ function valueText($, el) {
     : flattenText($, $el);
 }
 
-export function generateFormats($, html, formats) {
+export function generateFormats($, html, formats, url) {
   // Script, style, noscript and template bodies are never rendered text.
   // Left in, the text format carried an entire theme-toggle script
   // (erlang.org/downloads) and CSS rules (opennet.ru) — R15, 2026-09-04.
@@ -153,7 +171,7 @@ export function generateFormats($, html, formats) {
   const content = {};
   if (formats.includes('html')) content.html = html;
   if (formats.includes('text')) content.text = flattenText($);
-  if (formats.includes('markdown')) content.markdown = buildMarkdown($);
+  if (formats.includes('markdown')) content.markdown = buildMarkdown($, html, url);
   if (formats.includes('json')) {
     content.json = {
       title: pageTitle($),
@@ -166,8 +184,15 @@ export function generateFormats($, html, formats) {
   return content;
 }
 
-function buildMarkdown($) {
-  const title = pageTitle($);
+/**
+ * The page's main content, by the same pass scrape's markdown uses: converting
+ * the whole <main> handed back Wikipedia's language list and menus (R25,
+ * 2026-10-19). A page Readability finds no article in, or only a thin
+ * fragment of, falls back to the first content container below.
+ */
+function mainContentHtml($, html, url) {
+  const main = extractMainContent(html, url);
+  if (main.html && !isThinMainContent(main.html, html)) return keepPageHeading(main.html, $);
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
   let $body = null;
@@ -176,17 +201,31 @@ function buildMarkdown($) {
     if ($body.length > 0) break;
   }
   if (!$body || $body.length === 0) $body = $('body');
+  return $.html($body);
+}
+
+// "Web scraping - Wikipedia" names the page "Web scraping": the H1 plus a
+// separator and the site's name.
+function titleRepeatsHeading(title, heading) {
+  const t = title.toLowerCase();
+  const h = heading.toLowerCase();
+  return t === h || (t.startsWith(h) && /^\s+[-–—|·]\s+\S/.test(t.slice(h.length)));
+}
+
+function buildMarkdown($, html, url) {
+  const title = pageTitle($);
+  const mainHtml = mainContentHtml($, html, url);
 
   // Full-fidelity conversion via the shared Turndown helper (same converter
   // the unified `scrape` tool uses). The previous hand-rolled h1–h3/p/li walk
   // silently dropped any text living outside those tags (e.g. quotes in
   // <span>/<small> on quotes.toscrape.com).
-  let md = htmlToMarkdown($.html($body));
+  let md = htmlToMarkdown(mainHtml);
 
-  // C3: de-dup title — only emit the <title> heading if the page has no <h1>
-  // or if the first <h1> text differs from the <title> text (case-insensitive).
-  const firstH1 = $body.find('h1').first().text().trim();
-  const titleDuplicated = firstH1 && firstH1.toLowerCase() === title.toLowerCase();
+  // C3: de-dup title — only emit the <title> heading if the content has no
+  // <h1> or its first <h1> is not what the <title> names (case-insensitive).
+  const firstH1 = load(mainHtml)('h1').first().text().replace(/\s+/g, ' ').trim();
+  const titleDuplicated = firstH1 && titleRepeatsHeading(title, firstH1);
   if (title && !titleDuplicated) md = `# ${title}\n\n${md}`;
 
   return md.trim();

package/src/tools/agent/agent.js

@@ -16,7 +16,7 @@ import { AGENT_ESCALATION_CREDITS } from './escalation.js';
 export const AGENT_BASE_CREDITS = 8;
 
 export const AgentInputSchema = z.object({
-  prompt: z.string().min(1).max(2000).describe('Natural-language task or question'),
+  prompt: z.string().trim().min(1).max(2000).describe('Natural-language task or question'),
   urls: z.array(z.string().url()).max(20).optional().describe('Optional seed URLs to include (max 20)'),
   schema: z.record(z.any()).optional().describe('Optional JSON schema for structured output'),
   model: z.enum(['default', 'pro']).optional().default('default').describe('"default" = SamplingClient loop; "pro" = full ResearchOrchestrator'),

package/src/tools/basic/_fetch.js

@@ -9,6 +9,7 @@ import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { fetchResigned } from '../../utils/resignedFetch.js';
+import { withFetchCause } from '../../utils/fetchError.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -68,7 +69,7 @@ export async function fetchWithTimeout(url, options = {}) {
       if (error.name === 'AbortError') {
         throw new Error(`Request timeout after ${timeout}ms`);
       }
-      throw error;
+      throw withFetchCause(error, url);
     }
 
     // A host asking us to back off is honoured on the *next* request to it,

package/src/tools/basic/extractMetadata.js

@@ -7,6 +7,7 @@
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { parseJsonLd, filterJsonLdByType } from '../../utils/jsonLd.js';
 
 /**
@@ -161,7 +162,7 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
     };
   } catch (error) {
     return {
-      content: [{ type: 'text', text: `Failed to extract metadata: ${error.message}` }],
+      content: [{ type: 'text', text: `Failed to extract metadata: ${fetchErrorMessage(error, url)}` }],
       isError: true
     };
   }

package/src/tools/basic/extractText.js

@@ -15,6 +15,7 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { tooLargeForReadability } from '../scrape/_mainContent.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
 const BLOCK_ELEMENTS = new Set([
@@ -66,14 +67,17 @@ export function extractBlockText($) {
  * @returns {string}
  */
 export function readabilityToMarkdown(html, pageUrl) {
-  let articleHtml;
-  try {
-    const dom = new JSDOM(html, { url: pageUrl });
-    const reader = new Readability(dom.window.document);
-    const article = reader.parse();
-    articleHtml = article ? article.content : html;
-  } catch {
-    articleHtml = html;
+  let articleHtml = html;
+  // A page too large for Readability is converted whole (R25).
+  if (!tooLargeForReadability(html)) {
+    try {
+      const dom = new JSDOM(html, { url: pageUrl });
+      const reader = new Readability(dom.window.document);
+      const article = reader.parse();
+      if (article) articleHtml = article.content;
+    } catch {
+      // the whole page
+    }
   }
   const $ = load(articleHtml);
   absoluteUrls($, pageUrl);

package/src/tools/crawl/mapSite.js

@@ -167,6 +167,11 @@ export class MapSiteTool {
       // unresolvable host used to return a success with total_urls: 0 (R24).
       try {
         const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
+        // The start page is a page of the site too: a one-page site with no
+        // sitemap used to map to 0 URLs (R25 1.2).
+        if (!domainFilter || domainFilter.isAllowed(validated.url).allowed) {
+          urls.add(normalizeUrl(validated.url));
+        }
         pageUrls.forEach(url => {
           if (urls.size < poolLimit) {
             urls.add(normalizeUrl(url));
@@ -200,6 +205,9 @@ export class MapSiteTool {
 
       // Convert to array and limit
       const urlArray = pool.slice(0, validated.max_urls);
+      if (urlArray.length === 0) {
+        warnings.push('No URLs were found: no sitemap listed any, and the start page and its links were all excluded by the domain filter.');
+      }
 
       // Fetch metadata if requested
       if (validated.include_metadata) {

package/src/tools/extract/extractContent.js

@@ -401,7 +401,8 @@ export class ExtractContentTool {
    */
   extractTitleFromHTML(html) {
     const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    return titleMatch ? titleMatch[1].trim() : null;
+    // Decode entities: MDN's <table> page is titled "&lt;table&gt; ...".
+    return titleMatch ? load(titleMatch[1], null, false).root().text().trim() : null;
   }
 
   /**

package/src/tools/extract/extractStructured.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
 import { verifyNumericProvenance } from '../../utils/provenance.js';
 import { validateAgainstSchema, validateFieldsAgainstSchema } from '../../utils/schemaValidate.js';
@@ -532,6 +533,7 @@ export class ExtractStructuredTool {
       };
 
     } catch (error) {
+      const message = fetchErrorMessage(error, params.url);
       return {
         success: false,
         url: params.url || 'unknown',
@@ -540,8 +542,8 @@ export class ExtractStructuredTool {
         confidence: 0,
         schema_used: params.schema || {},
         processingTime: Date.now() - startTime,
-        error: `Structured extraction failed: ${error.message}`,
-        validation: { valid: false, errors: [error.message] }
+        error: `Structured extraction failed: ${message}`,
+        validation: { valid: false, errors: [message] }
       };
     }
   }

package/src/tools/extract/extractWithLlm.js

@@ -529,8 +529,10 @@ export class ExtractWithLlm {
       }));
     } catch (llmErr) {
       if (signal?.aborted) return { success: false, error: `LLM call aborted: ${llmErr.message}` };
-      // D1.3: If provider is 'auto'/'ollama' and it failed, try MCP sampling as final fallback
-      if (providerParam === 'auto' || providerParam === 'ollama') {
+      // D1.3: If provider is 'auto'/'ollama' and it failed, try MCP sampling as final fallback.
+      // Not when the caller named a model: sampling would answer with a
+      // different one, silently (R25 3.3).
+      if ((providerParam === 'auto' || providerParam === 'ollama') && !modelParam) {
         try {
           const SamplingClient = await getSamplingClient();
           const samplingClient = new SamplingClient({ mcpServer: this._mcpServer });
@@ -639,7 +641,9 @@ export class ExtractWithLlm {
       if (checked.skipped) provenance.skipped = checked.skipped;
     }
 
-    // C3: surface truncation metadata so callers know the input was clipped
+    // C3: surface truncation metadata so callers know the input was clipped.
+    // `input_truncated`, not `truncated`: a top-level `truncated: true` means
+    // "this result is a preview, read the rest with read_result" (R25).
     const result = {
       success: true,
       data: parsed,
@@ -649,7 +653,7 @@ export class ExtractWithLlm {
       usage
     };
     if (inputTruncated) {
-      result.truncated = true;
+      result.input_truncated = true;
       result.original_length = original_length;
     }
     if (fetchWarnings.length > 0) result.warnings = fetchWarnings;

package/src/tools/extract/processDocument.js

@@ -300,6 +300,33 @@ export class ProcessDocumentTool {
         tool: 'process_document'
       });
       if (gate.warnings.length > 0) result.warnings = gate.warnings;
+
+      // What came back is checked before the PDF parser sees it, as the url
+      // branch does: a PNG fetched as pdf_url failed as "Invalid PDF
+      // structure" (R25 3.3).
+      const response = await safeFetch(source, {
+        headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
+        signal: AbortSignal.timeout(30000)
+      });
+      if (!response.ok) {
+        if (response.status === 429 || response.status === 503) {
+          noteRetryAfter(source, response.headers.get('retry-after'));
+        }
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const contentType = response.headers.get('content-type') || '';
+      const buffer = await this.pdfProcessor.readBodyWithSizeCap(response);
+      const kind = sniffDocumentKind(contentType, buffer, source);
+      if (kind !== 'pdf') {
+        throw new Error(
+          `${contentType.split(';')[0] || kind} content is not a PDF: sourceType "pdf_url" needs a URL that serves one. ` +
+            'process_document reads PDF, DOCX, HTML and plain text; sourceType "url" detects which.'
+        );
+      }
+      source = buffer;
+      sourceType = 'pdf_buffer';
+      gate = undefined;
     }
 
     const pdfResult = await this.pdfProcessor.processPDF({
@@ -619,7 +646,8 @@ export class ProcessDocumentTool {
    */
   extractTitleFromHTML(html) {
     const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    return titleMatch ? titleMatch[1].trim() : null;
+    // Decode entities: MDN's <table> page is titled "&lt;table&gt; ...".
+    return titleMatch ? load(titleMatch[1], null, false).root().text().trim() : null;
   }
 
   /**

package/src/tools/extract/summarizeContent.js

@@ -211,11 +211,15 @@ export class SummarizeContentTool {
       return result;
 
     } catch (error) {
+      // A ZodError's message is its issue array as JSON; one line per issue reads better.
+      const reason = error instanceof z.ZodError
+        ? `Invalid input - ${error.issues.map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`).join('; ')}`
+        : error.message;
       return {
         originalText: params.text?.substring(0, 100) || 'unknown',
         summarizedAt: new Date().toISOString(),
         success: false,
-        error: `Content summarization failed: ${error.message}`,
+        error: `Content summarization failed: ${reason}`,
         processingTime: Date.now() - startTime,
         summary: {
           text: '',

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -33,7 +33,7 @@ const GenerateLLMsTxtSchema = z.object({
     robotsStyle: z.boolean().optional().default(false).describe('Emit legacy robots.txt-style directives instead of spec-compliant llmstxt.org markdown')
   }).optional().prefault({}),
 
-  complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Compliance level for generated guidelines'),
+  complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Applies only with outputOptions.robotsStyle:true, where it sets how many paths the llms.txt Disallow list covers (basic < standard < strict). The default llmstxt.org markdown is the same at every level; llms-full.txt only records the level as a label.'),
   
   format: z.enum(['both', 'llms-txt', 'llms-full-txt']).optional().default('both').describe('Which files to generate')
 });
@@ -740,7 +740,11 @@ export class GenerateLLMsTxtTool {
     lines.push('If you encounter issues or need clarification on these guidelines:');
     lines.push('1. Check this document for guidance');
     lines.push('2. Review the basic llms.txt file');
-    lines.push('3. Contact us using the information above');
+    // Only when section 1 printed something to contact: without either field
+    // this pointed at contact information the file did not contain.
+    if (outputOptions.contactEmail || outputOptions.organizationName) {
+      lines.push('3. Contact us using the information above');
+    }
     lines.push('');
     lines.push('### Updates');
     lines.push('These guidelines may be updated periodically. Check the generation date above');

package/src/tools/scrape/_mainContent.js

@@ -49,6 +49,12 @@ import { load } from 'cheerio';
 // Readability already kept it. Long enough to be unique to that table.
 const SIGNATURE_LENGTH = 120;
 
+// Navigation tables are link lists laid out as tables, not data. Wikipedia's
+// navbox and sidebar templates pass the data-table test (a `<th>` and 2+
+// cells a row), so List_of_HTTP_status_codes came back with its "HTTP" sidebar
+// and footer navbox re-attached as pipe tables (R25, 2026-10-19).
+const NAVIGATION_TABLE = 'table.navbox, .navbox-inner, .vertical-navbox, [role="navigation"], nav';
+
 // Copied verbatim from @mozilla/readability's REGEXPS. Two of them delete
 // content, and both are unanchored: `unlikelyCandidates` removes an element
 // outright, `negative` docks 25 from its class weight, which is enough for
@@ -151,6 +157,26 @@ export function sanitizeAccidentalClasses(document) {
   return sanitized;
 }
 
+/**
+ * Undo MediaWiki's heading wrapper before Readability sees the page.
+ *
+ * Since 2024 MediaWiki emits every section heading as
+ * `<div class="mw-heading"><h2>History</h2><span class="mw-editsection">[edit]</span></div>`.
+ * Readability scores that div as a link-heavy fragment: the History and
+ * Techniques h2s of Wikipedia's *Web scraping* article were dropped, and the
+ * headings that survived each came back followed by a "[edit]" link line
+ * (R25, 2026-10-19). Deleting the edit links and unwrapping the div leaves the
+ * bare heading Readability keeps.
+ *
+ * @param {Document} document - mutated in place
+ */
+export function unwrapMediaWikiHeadings(document) {
+  for (const el of Array.from(document.querySelectorAll('.mw-editsection'))) el.remove();
+  for (const wrapper of Array.from(document.querySelectorAll('div.mw-heading'))) {
+    wrapper.replaceWith(...Array.from(wrapper.childNodes));
+  }
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -232,6 +258,7 @@ export function recoverDroppedTables(html, url, keptText = '') {
       // A nested table travels with its parent; re-attaching it separately
       // would duplicate it.
       .filter((table) => !table.parentElement?.closest('table'))
+      .filter((table) => !table.closest(NAVIGATION_TABLE))
       .filter((table) => isDataTable(table) || isHeadedTable(table))
       .filter((table) => {
         const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
@@ -243,6 +270,28 @@ export function recoverDroppedTables(html, url, keptText = '') {
   }
 }
 
+// Readability's time grows faster than the page, and all of it blocks the
+// event loop: about 1 s at 25k elements (the largest Wikipedia articles), 9 s
+// at 114k (docs.python.org's whatsnew/changelog, 7.3 MB), which starved other
+// crawls past their 30 s queue budget (R25 Phase 2 follow-up). Above this many
+// elements a page skips the main-content pass and callers use the whole page.
+export const MAX_READABILITY_ELEMENTS = 50000;
+
+/**
+ * True when `html` has more elements than Readability should be given. Counts
+ * opening tags with a regex (about 5 ms on 7 MB) rather than building a DOM,
+ * which would itself cost seconds.
+ * @param {string} html
+ */
+export function tooLargeForReadability(html) {
+  const tag = /<[a-zA-Z]/g;
+  let count = 0;
+  while (tag.exec(html)) {
+    if (++count > MAX_READABILITY_ELEMENTS) return true;
+  }
+  return false;
+}
+
 /**
  * Extract a page's main content, re-attaching data tables Readability dropped.
  *
@@ -252,14 +301,17 @@ export function recoverDroppedTables(html, url, keptText = '') {
  *
  * @param {string} html - the page's HTML
  * @param {string} [url] - document URL, used as the base for relative links
- * @returns {{ html: string|null, title: string, tablesRecovered: number }} `html`
- *   is null when Readability found no article; callers decide what to fall back to.
+ * @returns {{ html: string|null, title: string, tablesRecovered: number, tooLarge?: boolean }}
+ *   `html` is null when Readability found no article, or was not run because the
+ *   page is too large (`tooLarge`); callers decide what to fall back to.
  */
 export function extractMainContent(html, url) {
+  if (tooLargeForReadability(html)) return { html: null, title: '', tablesRecovered: 0, tooLarge: true };
   let article;
   try {
     const dom = new JSDOM(html, { url });
     sanitizeAccidentalClasses(dom.window.document);
+    unwrapMediaWikiHeadings(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };