crawlforge-mcp-server 5.6.2 → 5.6.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.2",
+  "version": "5.6.4",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.2",
+    "crawlforge-extractors": "^1.6.3",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.2",
+  version: "5.6.4",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1392,6 +1392,15 @@ registerToolIfEnabled("stealth_mode", {
         if (challenge) {
           result.blocked = challenge;
           result.error = `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`;
+        } else if (!scraped.title && !String(scraped.text || '').trim()) {
+          // A document with no title and no text is not a scraped page; it
+          // is a render that has not happened yet (or an empty response).
+          // Reported as success it reads like a page with nothing on it.
+          result.success = false;
+          result.error =
+            `The stealth browser reached ${scraped.url} but the document rendered no title and no text` +
+            ` after ${wait_for || 0}ms of extra wait (${(scraped.html || '').length} bytes of HTML).` +
+            ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.';
         }
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as

package/src/core/LocalizationManager.js

@@ -308,7 +308,7 @@ export class LocalizationManager extends EventEmitter {
     
     // Generate comprehensive browser locale with RTL support
     const browserLocale = {
-      languages: [language, langCode, 'en'],
+      languages: [...new Set([language, langCode, 'en'])],
       timezone: timezone,
       locale: language,
       currency: currency,
@@ -777,7 +777,7 @@ export class LocalizationManager extends EventEmitter {
   async generateNavigatorProperties(localizationConfig) {
     return {
       language: localizationConfig.language,
-      languages: [localizationConfig.language, localizationConfig.language.split('-')[0], 'en'],
+      languages: [...new Set([localizationConfig.language, localizationConfig.language.split('-')[0], 'en'])],
       platform: 'Win32',
       userAgent: localizationConfig.userAgent || this.generateUserAgent(localizationConfig.countryCode),
       cookieEnabled: true,
@@ -923,7 +923,7 @@ export class LocalizationManager extends EventEmitter {
     if (!config) return null;
     
     return {
-      languages: [config.language, config.language.split("-")[0], "en"],
+      languages: [...new Set([config.language, config.language.split("-")[0], "en"])],
       timezone: config.timezone,
       currency: config.currency,
       dateFormat: this.getDateFormat(countryCode),

package/src/core/StealthBrowserManager.js

@@ -198,17 +198,27 @@ export class StealthBrowserManager {
       safari: 0.15
     };
 
-    // Realistic viewport sizes with market distribution
+    // Screen sizes (CSS pixels) with market distribution. The screen is the
+    // root of the display — the window is derived from it in generateDisplay.
+    // 1536×864 and 1280×720 are 1920×1080 panels at 125% / 150% scaling, so
+    // each size names the device scale factors it is really seen at.
     this.viewportSizes = [
-      { width: 1920, height: 1080, weight: 0.25 }, // Most common
-      { width: 1366, height: 768, weight: 0.20 },  // Second most common
-      { width: 1536, height: 864, weight: 0.15 },
-      { width: 1440, height: 900, weight: 0.12 },
-      { width: 1280, height: 720, weight: 0.10 },
-      { width: 1600, height: 900, weight: 0.08 },
-      { width: 1024, height: 768, weight: 0.05 },  // Legacy but still used
-      { width: 2560, height: 1440, weight: 0.03 }, // High-res displays
-      { width: 3840, height: 2160, weight: 0.02 }  // 4K displays
+      { width: 1920, height: 1080, weight: 0.27, dpr: [1] },     // Most common
+      { width: 1366, height: 768, weight: 0.20, dpr: [1] },      // Second most common
+      { width: 1536, height: 864, weight: 0.15, dpr: [1.25] },
+      { width: 1440, height: 900, weight: 0.12, dpr: [1] },
+      { width: 1280, height: 720, weight: 0.10, dpr: [1.5, 1] },
+      { width: 1600, height: 900, weight: 0.08, dpr: [1] },
+      { width: 1024, height: 768, weight: 0.05, dpr: [1] },      // Legacy but still used
+      { width: 2560, height: 1440, weight: 0.03, dpr: [1] }      // High-res displays
+    ];
+    // Retina Macs report their "looks like" size at a scale factor of 2.
+    this.macScreenSizes = [
+      { width: 1440, height: 900, weight: 0.35, dpr: [2] },
+      { width: 1512, height: 982, weight: 0.25, dpr: [2] },
+      { width: 1728, height: 1117, weight: 0.15, dpr: [2] },
+      { width: 1680, height: 1050, weight: 0.10, dpr: [2] },
+      { width: 1920, height: 1080, weight: 0.15, dpr: [1, 2] }   // External display
     ];
     
     // Mobile viewport sizes for mobile emulation
@@ -516,14 +526,17 @@ export class StealthBrowserManager {
     const selectedOS = this.selectOS(config);
     const persona = this.selectLocalePersona(config);
     const userAgent = this.selectRealisticUserAgent(config, selectedOS);
+    const display = this.generateDisplay(selectedOS, config);
     const fingerprint = {
       userAgent,
       locale: persona.locale,
-      viewport: config.customViewport || this.selectWeightedViewport(),
+      viewport: display.viewport,
       timezone: config.timezone || persona.timezone,
-      deviceScaleFactor: this.randomFloat(1, 2, 1),
-      isMobile: Math.random() < 0.1, // 10% mobile
-      hasTouch: Math.random() < 0.15, // 15% touch
+      deviceScaleFactor: display.deviceScaleFactor,
+      // Every user agent in the pool is a desktop one; mobile emulation under
+      // it (10% of fingerprints until R16) contradicted the UA outright.
+      isMobile: false,
+      hasTouch: selectedOS === 'windows' && Math.random() < 0.15, // touch laptops
       colorScheme: Math.random() < 0.3 ? 'dark' : 'light',
       reducedMotion: Math.random() < 0.1 ? 'reduce' : 'no-preference',
       forcedColors: Math.random() < 0.05 ? 'active' : 'none',
@@ -537,7 +550,8 @@ export class StealthBrowserManager {
       fonts: this.generateAdvancedFontList(selectedOS),
       plugins: this.generateAdvancedPluginList(),
       geolocation: this.generateRealisticGeolocation(persona),
-      screen: this.generateAdvancedScreenProperties(),
+      screen: display.screen,
+      window: display.window,
       battery: this.generateBatteryFingerprint()
     };
 
@@ -575,7 +589,17 @@ export class StealthBrowserManager {
     // An unmodelled locale still gets a coherent timezone/geolocation pair.
     const pool = exact.length ? exact : (sameLanguage.length ? sameLanguage : this.localePersonas);
 
-    const persona = pool[Math.floor(Math.random() * pool.length)];
+    // The host's own zone is where the egress address is. A persona drawn at
+    // random put America/Chicago in JavaScript beside a Florida IP, and both
+    // pixelscan and iphey called the timezone spoofed (R16, 2026-09-04). So a
+    // persona in the host zone is preferred — from the requested locale's
+    // pool, or from any locale when nothing beyond the default was asked for.
+    // An explicit config.timezone still overrides in the caller.
+    const hostZone = this.hostTimezone();
+    const inHostZone = (personas) => (hostZone ? personas.find((p) => p.timezone === hostZone) : undefined);
+    const persona = inHostZone(pool)
+      || (requested.toLowerCase() === 'en-us' ? inHostZone(this.localePersonas) : undefined)
+      || pool[Math.floor(Math.random() * pool.length)];
     return { ...persona, locale: requested };
   }
 
@@ -648,10 +672,58 @@ export class StealthBrowserManager {
   }
 
   /**
-   * Select viewport size based on weights
+   * One display for the whole fingerprint: the screen is drawn from the pool
+   * and the window derived from it as a maximised browser — viewport width is
+   * the screen width, viewport height the available height minus the browser
+   * chrome, outer size the available area. The screen and the viewport used
+   * to be drawn from the pool independently and the scale factor was a random
+   * float, so a 1536×864 window sat on a 1366×768 screen at a scale of 1.7
+   * (which Chromium reports as 1.7000000476837158 and a 1408.0000305175781px
+   * screen) — pixelscan's "Your browser fingerprint is inconsistent" (R16,
+   * 2026-09-04). A custom viewport gets the smallest pooled screen it fits on.
    */
-  selectWeightedViewport() {
-    return this.weightedRandomFromArray(this.viewportSizes);
+  generateDisplay(selectedOS, config = {}) {
+    const chrome = selectedOS === 'macos' ? 87 : 85;
+    const taskbar = selectedOS === 'macos' ? 25 : (selectedOS === 'linux' ? 27 : 40);
+    const pool = selectedOS === 'macos' ? this.macScreenSizes : this.viewportSizes;
+
+    let screen;
+    let viewport;
+    if (config.customViewport) {
+      viewport = { width: config.customViewport.width, height: config.customViewport.height };
+      const needed = { width: viewport.width, height: viewport.height + chrome + taskbar };
+      screen = pool
+        .filter((s) => s.width >= needed.width && s.height >= needed.height)
+        .sort((a, b) => a.width * a.height - b.width * b.height)[0]
+        || { ...needed, dpr: [1] };
+    } else {
+      screen = this.weightedRandomFromArray(pool);
+      viewport = { width: screen.width, height: screen.height - taskbar - chrome };
+    }
+
+    return {
+      viewport,
+      deviceScaleFactor: screen.dpr[Math.floor(Math.random() * screen.dpr.length)],
+      screen: {
+        width: screen.width,
+        height: screen.height,
+        availWidth: screen.width,
+        availHeight: screen.height - taskbar,
+        colorDepth: 24,
+        pixelDepth: 24,
+        orientation: { angle: 0, type: 'landscape-primary' }
+      },
+      window: { outerWidth: viewport.width, outerHeight: viewport.height + chrome }
+    };
+  }
+
+  /**
+   * The machine's own timezone, or null when it is the UTC of a container —
+   * set TZ on such a host to the zone its egress address geolocates to.
+   */
+  hostTimezone() {
+    const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+    return zone && zone !== 'UTC' && !/^(?:Etc|GMT)/.test(zone) ? zone : null;
   }
 
   /**
@@ -1139,25 +1211,6 @@ export class StealthBrowserManager {
     };
   }
 
-  /**
-   * Generate advanced screen properties
-   */
-  generateAdvancedScreenProperties() {
-    const viewport = this.selectWeightedViewport();
-    
-    return {
-      width: viewport.width,
-      height: viewport.height,
-      availWidth: viewport.width,
-      availHeight: viewport.height - (30 + Math.floor(Math.random() * 20)), // Account for taskbar
-      colorDepth: Math.random() < 0.95 ? 24 : 32,
-      pixelDepth: 24,
-      orientation: {
-        angle: 0,
-        type: 'landscape-primary'
-      }
-    };
-  }
 
   /**
    * Generate battery API fingerprint
@@ -1474,6 +1527,21 @@ export class StealthBrowserManager {
           
           return ctx;
         };
+
+        // A webgl2 context (and any OffscreenCanvas context) never passed
+        // through the wrapper above: iphey read the GPU through webgl2 and
+        // printed "SwiftShader" under a Windows user agent (R16, 2026-09-04).
+        // The two unmasked parameters are answered on the prototypes, where
+        // every context of either kind reads them.
+        for (const Context of [window.WebGLRenderingContext, window.WebGL2RenderingContext]) {
+          if (!Context || !Context.prototype) continue;
+          const getParameter = Context.prototype.getParameter;
+          Context.prototype.getParameter = function(parameter) {
+            if (parameter === 37446) return webglConfig.renderer;
+            if (parameter === 37445) return webglConfig.vendor;
+            return getParameter.call(this, parameter);
+          };
+        }
       }, fingerprint.webGL);
     }
 
@@ -1568,7 +1636,7 @@ export class StealthBrowserManager {
     }
 
     // Screen resolution spoofing
-    await context.addInitScript((screenConfig) => {
+    await context.addInitScript(({ screen: screenConfig, window: windowConfig }) => {
       Object.defineProperties(screen, {
         width: { value: screenConfig.width, configurable: true },
         height: { value: screenConfig.height, configurable: true },
@@ -1577,7 +1645,18 @@ export class StealthBrowserManager {
         colorDepth: { value: screenConfig.colorDepth, configurable: true },
         pixelDepth: { value: screenConfig.pixelDepth, configurable: true }
       });
-    }, fingerprint.screen);
+      // Headless Chromium reports outerWidth/outerHeight equal to the inner
+      // size — a window with no tabs and no toolbar.
+      if (windowConfig) {
+        Object.defineProperty(window, 'outerWidth', { get: () => windowConfig.outerWidth, configurable: true });
+        Object.defineProperty(window, 'outerHeight', { get: () => windowConfig.outerHeight, configurable: true });
+      }
+    }, {
+      screen: fingerprint.screen,
+      // camoufox runs without a fixed viewport (see createStealthContext), so
+      // an outer size derived from one would contradict the real window.
+      window: this._launchedEngine === 'camoufox' ? null : fingerprint.window
+    });
 
     // Timezone spoofing
     if (config.spoofTimezone) {
@@ -1853,16 +1932,34 @@ export class StealthBrowserManager {
     const { contextId } = await this.createStealthContext({ ...stealthConfig, engine });
     try {
       const page = await this.createStealthPage(contextId);
+      let crashed = false;
+      page.on('crash', () => { crashed = true; });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
       await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
       if (wait_for > 0) await page.waitForTimeout(wait_for);
 
-      const [title, html, text] = await Promise.all([
-        page.title().catch(() => ''),
-        page.content().catch(() => ''),
-        page.innerText('body').catch(() => '')
-      ]);
+      // A failure to read the document is a failure. With every read wrapped
+      // in .catch(() => ''), a renderer that crashed or a page closed during
+      // the wait (iphey.com under software GL, R16 2026-09-04) came back as
+      // success:true with an empty title and body — indistinguishable from a
+      // page that rendered nothing. The body text is read through evaluate,
+      // not innerText('body'): a frameset document has no <body>, and
+      // innerText waited its full 30s for one before throwing.
+      let title; let html; let text;
+      try {
+        [title, html, text] = await Promise.all([
+          page.title(),
+          page.content(),
+          page.evaluate(() => (document.body ? document.body.innerText : ''))
+        ]);
+      } catch (error) {
+        const what = crashed ? 'crashed' : (page.isClosed() ? 'was closed' : 'could not be read');
+        throw new Error(
+          `The stealth page ${what} before its content could be read (${String(error.message).split('\n')[0]}). ` +
+          'The page did not render; retry, or use scrape_with_actions for a page this heavy.'
+        );
+      }
       const shot = screenshot
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;

package/src/core/analysis/ContentAnalyzer.js

@@ -398,7 +398,7 @@ export class ContentAnalyzer {
     const freq = {};
     const sentenceWords = sentences.map(sentence => {
       const words = nlp(sentence).terms().out('array')
-        .map(w => w.toLowerCase().replace(/[^a-z0-9]/g, ''))
+        .map(w => w.toLowerCase().replace(/[^\p{L}\p{N}]/gu, ''))
         .filter(w => w.length > 2 && !this.isStopWord(w));
       words.forEach(w => { freq[w] = (freq[w] || 0) + 1; });
       return words;
@@ -471,7 +471,7 @@ export class ContentAnalyzer {
       
       allPhrases.forEach(phrase => {
         // Strip edge punctuation and skip phrases made only of stop words
-        const cleaned = phrase.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '');
+        const cleaned = phrase.toLowerCase().trim().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
         const words = cleaned.split(/\s+/);
         if (cleaned.length > 2 && words.some(w => !this.isStopWord(w))) {
           phraseCount[cleaned] = (phraseCount[cleaned] || 0) + 1;
@@ -550,7 +550,7 @@ export class ContentAnalyzer {
       // (corporate suffix or organization-headed alias definition).
       const demotedAcronyms = [];
       organizations = organizations.filter(e => {
-        if (/^[A-Z]{2,}$/.test(e) && !this.hasOrganizationEvidence(e, text)) {
+        if (/^\p{Lu}{2,}$/u.test(e) && !this.hasOrganizationEvidence(e, text)) {
           demotedAcronyms.push(e);
           return false;
         }
@@ -565,11 +565,14 @@ export class ContentAnalyzer {
         ...people, ...places, ...organizations, ...other
       ].map(e => e.toLowerCase()));
 
-      const properNouns = (text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [])
+      // Unicode classes, not [A-Z]: the ASCII version cut "Universität" to
+      // "Universit" and "Wisłą" to "Wis" and dropped "Środkowej" outright (R16).
+      // \b stays ASCII-only even under the u flag, hence the lookbehind.
+      const properNouns = (text.match(/(?<![\p{L}\p{N}])\p{Lu}[\p{L}.]+(?:\s+\p{Lu}[\p{L}.]+)*/gu) || [])
         // Split matches that crossed a sentence boundary ("XPath. In") — a
         // period after a lowercase letter followed by a capital is a sentence
         // end, while abbreviations ("U.S. Holdings") stay intact.
-        .flatMap(n => n.split(/(?<=[a-z]\.)\s+(?=[A-Z])/));
+        .flatMap(n => n.split(/(?<=\p{Ll}\.)\s+(?=\p{Lu})/u));
       const supplemental = this.dedupeEntities(properNouns.map(n => this.cleanEntityText(n)))
         .filter(n => n.length > 1
           && !existingEntities.has(n.toLowerCase())
@@ -656,7 +659,7 @@ export class ContentAnalyzer {
       
       [...nouns, ...verbs, ...adjectives].forEach(term => {
         // Strip leading/trailing punctuation but preserve internal periods (e.g. Node.js)
-        const cleaned = term.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9.]+$/gi, '').replace(/\.+$/, '');
+        const cleaned = term.toLowerCase().trim().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}.]+$/gu, '').replace(/\.+$/, '');
         if (cleaned.length > 2 && !this.isStopWord(cleaned)) {
           termFreq[cleaned] = (termFreq[cleaned] || 0) + 1;
 
@@ -893,7 +896,7 @@ export class ContentAnalyzer {
       entity = entity.replace(/[\s"'‘’“”`)\]}>,;:!?«»–—-]+$/, '');
       if (entity.endsWith('.')) {
         const lastToken = entity.split(/\s+/).pop();
-        const isAbbreviation = /^(?:[A-Za-z]\.)+$/.test(lastToken)
+        const isAbbreviation = /^(?:\p{L}\.)+$/u.test(lastToken)
           || /^(?:inc|corp|ltd|co|llc|jr|sr|st|mr|mrs|ms|dr|no|vs?)\.$/i.test(lastToken);
         if (!isAbbreviation) {
           entity = entity.replace(/\.+$/, '');
@@ -944,13 +947,13 @@ export class ContentAnalyzer {
     if (entity.includes(' ')) return false;
     const lower = entity.toLowerCase();
     if (this.isStopWord(lower)) return true;
-    if (!/^[A-Z][a-z]+$/.test(entity)) return false;
+    if (!/^\p{Lu}\p{Ll}+$/u.test(entity)) return false;
 
     // Word also appears in lowercase → ordinary word, capitalized only by position
-    if (new RegExp(`(?:^|[^A-Za-z])${lower}(?:[^A-Za-z]|$)`).test(text)) return true;
+    if (new RegExp(`(?:^|\\P{L})${lower}(?:\\P{L}|$)`, 'u').test(text)) return true;
 
     // Keep only if at least one occurrence is NOT at a sentence start
-    const occurrence = new RegExp(`\\b${entity}\\b`, 'g');
+    const occurrence = new RegExp(`(?<![\\p{L}\\p{N}])${entity}(?![\\p{L}\\p{N}])`, 'gu');
     let match;
     while ((match = occurrence.exec(text)) !== null) {
       const before = text.slice(0, match.index).replace(/[\s"'‘’“”()[\]]+$/, '');
@@ -975,13 +978,13 @@ export class ContentAnalyzer {
    * a corporate suffix follows it ("IBM Corp"), or it is introduced as the
    * alias of an organization-headed name ("American Airlines (AA)",
    * "French Data Protection Authority (CNIL)").
-   * @param {string} acronym - ALL-CAPS candidate (already /^[A-Z]{2,}$/)
+   * @param {string} acronym - ALL-CAPS candidate (already /^\p{Lu}{2,}$/u)
    * @param {string} text - Full source text
    * @returns {boolean} - True if the text corroborates the org classification
    */
   hasOrganizationEvidence(acronym, text) {
     const corporateSuffix = new RegExp(
-      `\\b${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:[^a-zA-Z]|$)`
+      `(?<!\\p{L})${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:\\P{L}|$)`, 'u'
     );
     if (corporateSuffix.test(text)) return true;
 

package/src/core/processing/ContentProcessor.js

@@ -8,6 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
+import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -30,7 +31,8 @@ const ReadabilityResult = z.object({
   byline: z.string().nullable(),
   dir: z.string().nullable(),
   siteName: z.string().nullable(),
-  lang: z.string().nullable()
+  lang: z.string().nullable(),
+  tablesRecovered: z.number()
 });
 
 export class ContentProcessor {
@@ -78,16 +80,28 @@ export class ContentProcessor {
 
         const article = reader.parse();
         if (article) {
+          // Readability keeps one article candidate and drops every data
+          // table outside it — scrape_with_actions returned CoinMarketCap's
+          // historical-data page as its FAQ copy with no table (2026-09-04).
+          // Re-attach what it dropped, as scrape's main-content path does.
+          const recovered = recoverDroppedTables(html, url, article.textContent);
+          const content = recovered.length > 0
+            ? `${article.content}\n${recovered.map((table) => table.html).join('\n')}`
+            : article.content;
+          const textContent = recovered.length > 0
+            ? `${article.textContent}\n${recovered.map((table) => table.text).join('\n')}`
+            : article.textContent;
           result.readability = {
             title: article.title,
-            content: article.content,
-            textContent: article.textContent,
-            length: article.length,
+            content,
+            textContent,
+            length: textContent.length,
             excerpt: article.excerpt,
             byline: article.byline,
             dir: article.dir,
             siteName: article.siteName,
-            lang: article.lang
+            lang: article.lang,
+            tablesRecovered: recovered.length
           };
 
           // Calculate readability score

package/src/tools/extract/extractContent.js

@@ -8,6 +8,7 @@ import { ContentProcessor } from '../../core/processing/ContentProcessor.js';
 import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
+import { isThinMainContent } from '../scrape/_mainContent.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -215,8 +216,18 @@ export class ExtractContentTool {
         }
       });
 
-      // Step 3: Extract and format content
-      if (processingResult.readability) {
+      // Step 3: Extract and format content.
+      // Readability keeps one dense block; on a landing page that is a
+      // fraction of the visible text (libreoffice.org came back as its
+      // 509-character "Welcome" box at confidence 0.9, R16). A thin article
+      // is handed to the boilerplate-removal fallback and named as such.
+      const thin = processingResult.readability
+        ? isThinMainContent(processingResult.readability.content, html)
+        : null;
+      if (thin) {
+        processingResult.fallback_content = this.contentProcessor.extractFallbackContent(html);
+      }
+      if (processingResult.readability && !thin) {
         result.readability = processingResult.readability;
         result.content = {
           text: processingResult.readability.textContent || processingResult.readability.content,
@@ -224,6 +235,11 @@ export class ExtractContentTool {
         result.extractionMethod = 'readability';
         result.confidence = 0.9;
         result.finalUrl = url;
+        if (processingResult.readability.tablesRecovered > 0) {
+          (result.warnings ||= []).push(
+            `mainContent: re-attached ${processingResult.readability.tablesRecovered} data table(s) that main-content extraction had dropped`
+          );
+        }
 
         // Convert to markdown if requested
         if (options.outputFormat === 'markdown') {
@@ -234,7 +250,9 @@ export class ExtractContentTool {
           text: processingResult.fallback_content.content
         };
         result.extractionMethod = 'fallback_boilerplate_removal';
-        result.fallback_reason = 'Readability did not detect an article; used boilerplate-removal fallback';
+        result.fallback_reason = thin
+          ? `Readability kept only ${thin.kept} of ${thin.visible} visible characters (a landing page, not an article); used boilerplate-removal fallback`
+          : 'Readability did not detect an article; used boilerplate-removal fallback';
         result.confidence = 0.5;
         result.finalUrl = url;
       } else {

package/src/tools/scrape/_mainContent.js

@@ -35,11 +35,15 @@
  *    12th sits inside a collapsed `<details>`, which Readability drops whole
  *    and which this deliberately does not try to rescue.
  *
- * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
+ * Shared by unifiedScrape (wants HTML) and extractStructured (wants text);
+ * the table recovery alone is also used by ContentProcessor, which runs its
+ * own Readability pass for extract_content, scrape_with_actions and
+ * process_document.
  */
 
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
+import { load } from 'cheerio';
 
 // How much of a table's text is compared against the article to decide whether
 // Readability already kept it. Long enough to be unique to that table.
@@ -168,6 +172,61 @@ function isDataTable(table) {
   return rows >= 10 || columns > 4;
 }
 
+/**
+ * A data table's text with its structure kept: one line per row, cells joined
+ * by " | ". A bare `textContent` runs every cell together
+ * ("DateOpen*HighLowClose**…"), which is what scrape_with_actions' selector
+ * extraction handed back for CoinMarketCap's history table.
+ * @param {HTMLTableElement} table
+ * @returns {string}
+ */
+function tableText(table) {
+  return Array.from(table.rows)
+    .map((row) => Array.from(row.cells).map((cell) => normalizeWhitespace(cell.textContent || '')).join(' | '))
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * The data tables in `html` that a Readability pass left out of its article.
+ *
+ * Readability mutates the document it is handed — the dropped tables are
+ * already gone by the time parse() returns — so this parses the original HTML
+ * again. Shared by every caller that runs Readability itself: extractMainContent
+ * below, and ContentProcessor, whose consumers (extract_content,
+ * scrape_with_actions, process_document) lost tables the same way until
+ * 2026-09-04, when scrape_with_actions returned CoinMarketCap's historical-data
+ * page as its FAQ copy — the wait for the table rows had succeeded, and the
+ * markdown then had no table at all.
+ *
+ * @param {string} html - the page's HTML
+ * @param {string} [url] - document URL, used as the base for relative links
+ * @param {string} [keptText] - the article's textContent, so a table Readability
+ *   did keep is not attached a second time
+ * @returns {{ html: string, text: string }[]} each dropped table's outerHTML and
+ *   its row-per-line text; empty when there is nothing to recover
+ */
+export function recoverDroppedTables(html, url, keptText = '') {
+  // Nothing to recover, and no reason to pay for a second parse.
+  if (!/<table[\s>]/i.test(html)) return [];
+  try {
+    const fresh = new JSDOM(html, { url });
+    const kept = normalizeWhitespace(keptText || '');
+    return Array.from(fresh.window.document.querySelectorAll('table'))
+      // A nested table travels with its parent; re-attaching it separately
+      // would duplicate it.
+      .filter((table) => !table.parentElement?.closest('table'))
+      .filter(isDataTable)
+      .filter((table) => {
+        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
+        return signature.length > 0 && !kept.includes(signature);
+      })
+      .map((table) => ({ html: table.outerHTML, text: tableText(table) }));
+  } catch {
+    return [];
+  }
+}
+
 /**
  * Extract a page's main content, re-attaching data tables Readability dropped.
  *
@@ -192,32 +251,7 @@ export function extractMainContent(html, url) {
   if (!article || !article.content) return { html: null, title: '', tablesRecovered: 0 };
 
   const title = article.title || '';
-
-  // Nothing to recover, and no reason to pay for a second parse.
-  if (!/<table[\s>]/i.test(html)) {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
-
-  // Readability mutates the document it is handed — the dropped tables are
-  // already gone from `dom` by the time parse() returns — so recovery has to
-  // parse the original HTML again.
-  let recovered;
-  try {
-    const fresh = new JSDOM(html, { url });
-    const kept = normalizeWhitespace(article.textContent || '');
-    recovered = Array.from(fresh.window.document.querySelectorAll('table'))
-      // A nested table travels with its parent; re-attaching it separately
-      // would duplicate it.
-      .filter((table) => !table.parentElement?.closest('table'))
-      .filter(isDataTable)
-      .filter((table) => {
-        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
-        return signature.length > 0 && !kept.includes(signature);
-      })
-      .map((table) => table.outerHTML);
-  } catch {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
+  const recovered = recoverDroppedTables(html, url, article.textContent).map((table) => table.html);
 
   return {
     html: recovered.length > 0 ? `${article.content}\n${recovered.join('\n')}` : article.content,
@@ -226,4 +260,34 @@ export function extractMainContent(html, url) {
   };
 }
 
+// Below both of these, Readability's article is a fragment of the page, not
+// its main content.
+export const THIN_MAIN_CONTENT = { maxChars: 1500, maxShare: 0.35 };
+
+function visibleTextLength(html) {
+  const $ = load(html);
+  $('script, style, noscript, template').remove();
+  return $('body').text().replace(/\s+/g, ' ').trim().length;
+}
+
+/**
+ * Readability keeps ONE dense block and drops the rest. On a landing page
+ * that block is a fraction of what the visitor reads: gnome.org came back as
+ * ~150 of 1,666 visible characters and libreoffice.org as its 509-character
+ * "Welcome" box, both reported at full confidence (R16, 2026-09-04). A short
+ * article on a page with much more visible text is not the main content.
+ *
+ * @param {string} mainHtml - Readability's article HTML
+ * @param {string} pageHtml - the whole page
+ * @returns {{ kept: number, visible: number }|null} the two lengths when the
+ *   article is thin, null when it is the page's main content
+ */
+export function isThinMainContent(mainHtml, pageHtml) {
+  const kept = visibleTextLength(mainHtml || '');
+  if (kept >= THIN_MAIN_CONTENT.maxChars) return null;
+  const visible = visibleTextLength(pageHtml || '');
+  if (visible === 0 || kept / visible >= THIN_MAIN_CONTENT.maxShare) return null;
+  return { kept, visible };
+}
+
 export default extractMainContent;

package/src/tools/scrape/unifiedScrape.js

@@ -12,7 +12,7 @@
 
 import { z } from 'zod';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
-import { extractMainContent } from './_mainContent.js';
+import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
@@ -236,6 +236,15 @@ export class UnifiedScrapeTool {
           `mainContent: re-attached ${main.tablesRecovered} data table(s) that main-content extraction had dropped`
         );
       }
+      // A thin article on a landing page is not the main content (gnome.org
+      // came back as ~150 of 1,666 visible characters, R16): use the page.
+      const thin = main.html ? isThinMainContent(main.html, html) : null;
+      if (thin) {
+        mainHtml = html;
+        warnings.push(
+          `mainContent: main-content extraction kept ${thin.kept} of ${thin.visible} visible characters; the whole page is used instead`
+        );
+      }
       return mainHtml;
     }