crawlforge-mcp-server 5.6.1 → 5.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.1",
+  "version": "5.6.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.1",
+    "crawlforge-extractors": "^1.6.3",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -33,6 +33,7 @@ import { LocalizationManager } from "./src/core/LocalizationManager.js";
 // Stealth scrape: format conversion + the pre-fetch compliance gate (G5/G6/G7)
 import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
+import { detectChallengePage } from "./src/utils/challengeDetection.js";
 import { browserPreflight } from "./src/utils/robotsGate.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
@@ -105,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.1",
+  version: "5.6.3",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1377,12 +1378,30 @@ registerToolIfEnabled("stealth_mode", {
           stealthConfig
         });
 
+        // A bot-wall interstitial arrives as HTTP 200 with a title and prose
+        // of its own; reported as a successful scrape it hid the block for
+        // three rounds (R10 Q1 → R15). Name it — the content is still returned
+        // so the caller can see what the site served.
+        const challenge = detectChallengePage(scraped);
         result = {
-          success: true,
+          success: !challenge,
           url: scraped.url,
           title: scraped.title,
           content: stealthScrapeFormats(formats, scraped)
         };
+        if (challenge) {
+          result.blocked = challenge;
+          result.error = `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`;
+        } else if (!scraped.title && !String(scraped.text || '').trim()) {
+          // A document with no title and no text is not a scraped page; it
+          // is a render that has not happened yet (or an empty response).
+          // Reported as success it reads like a page with nothing on it.
+          result.success = false;
+          result.error =
+            `The stealth browser reached ${scraped.url} but the document rendered no title and no text` +
+            ` after ${wait_for || 0}ms of extra wait (${(scraped.html || '').length} bytes of HTML).` +
+            ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.';
+        }
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as
         // scrape and scrape_with_actions, so the base64 never bloats the result.
@@ -1452,6 +1471,12 @@ registerToolIfEnabled("stealth_mode", {
               ok: response ? response.ok() : null,
               title: await page.title().catch(() => null)
             };
+            const challenge = detectChallengePage({
+              title: navigation.title || '',
+              html: await page.content().catch(() => ''),
+              text: await page.innerText('body').catch(() => '')
+            });
+            if (challenge) navigation.blocked = challenge;
           }
         } finally {
           // No operation can ever reference this page again — keeping it open

package/src/core/LocalizationManager.js

@@ -308,7 +308,7 @@ export class LocalizationManager extends EventEmitter {
     
     // Generate comprehensive browser locale with RTL support
     const browserLocale = {
-      languages: [language, langCode, 'en'],
+      languages: [...new Set([language, langCode, 'en'])],
       timezone: timezone,
       locale: language,
       currency: currency,
@@ -777,7 +777,7 @@ export class LocalizationManager extends EventEmitter {
   async generateNavigatorProperties(localizationConfig) {
     return {
       language: localizationConfig.language,
-      languages: [localizationConfig.language, localizationConfig.language.split('-')[0], 'en'],
+      languages: [...new Set([localizationConfig.language, localizationConfig.language.split('-')[0], 'en'])],
       platform: 'Win32',
       userAgent: localizationConfig.userAgent || this.generateUserAgent(localizationConfig.countryCode),
       cookieEnabled: true,
@@ -923,7 +923,7 @@ export class LocalizationManager extends EventEmitter {
     if (!config) return null;
     
     return {
-      languages: [config.language, config.language.split("-")[0], "en"],
+      languages: [...new Set([config.language, config.language.split("-")[0], "en"])],
       timezone: config.timezone,
       currency: config.currency,
       dateFormat: this.getDateFormat(countryCode),

package/src/core/StealthBrowserManager.js

@@ -140,47 +140,50 @@ export class StealthBrowserManager {
       }
     };
 
-    // Enhanced User agent pools with realistic patterns
+    // User agent pools. Chrome majors track the bundled Chromium (151 at
+    // playwright-core 1.62) and Firefox tracks the Camoufox build (135): a
+    // UA thirty majors behind the engine — the pool sat at 119–121 while
+    // Chrome 152 shipped — is a staleness tell for anything that reads
+    // sec-ch-ua (R15, 2026-09-04).
     this.userAgentPools = {
       chrome: {
         windows: [
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
         ],
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
         ],
         linux: [
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
         ]
       },
       firefox: {
         windows: [
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0'
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0'
         ],
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0'
         ],
         linux: [
-          'Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0'
+          'Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0',
+          'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0'
         ]
       },
       safari: {
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15'
         ]
       }
     };
-    
+
     // Operating system distributions for realistic user agent selection
     this.osDistribution = {
       windows: 0.75,
@@ -195,17 +198,27 @@ export class StealthBrowserManager {
       safari: 0.15
     };
 
-    // Realistic viewport sizes with market distribution
+    // Screen sizes (CSS pixels) with market distribution. The screen is the
+    // root of the display — the window is derived from it in generateDisplay.
+    // 1536×864 and 1280×720 are 1920×1080 panels at 125% / 150% scaling, so
+    // each size names the device scale factors it is really seen at.
     this.viewportSizes = [
-      { width: 1920, height: 1080, weight: 0.25 }, // Most common
-      { width: 1366, height: 768, weight: 0.20 },  // Second most common
-      { width: 1536, height: 864, weight: 0.15 },
-      { width: 1440, height: 900, weight: 0.12 },
-      { width: 1280, height: 720, weight: 0.10 },
-      { width: 1600, height: 900, weight: 0.08 },
-      { width: 1024, height: 768, weight: 0.05 },  // Legacy but still used
-      { width: 2560, height: 1440, weight: 0.03 }, // High-res displays
-      { width: 3840, height: 2160, weight: 0.02 }  // 4K displays
+      { width: 1920, height: 1080, weight: 0.27, dpr: [1] },     // Most common
+      { width: 1366, height: 768, weight: 0.20, dpr: [1] },      // Second most common
+      { width: 1536, height: 864, weight: 0.15, dpr: [1.25] },
+      { width: 1440, height: 900, weight: 0.12, dpr: [1] },
+      { width: 1280, height: 720, weight: 0.10, dpr: [1.5, 1] },
+      { width: 1600, height: 900, weight: 0.08, dpr: [1] },
+      { width: 1024, height: 768, weight: 0.05, dpr: [1] },      // Legacy but still used
+      { width: 2560, height: 1440, weight: 0.03, dpr: [1] }      // High-res displays
+    ];
+    // Retina Macs report their "looks like" size at a scale factor of 2.
+    this.macScreenSizes = [
+      { width: 1440, height: 900, weight: 0.35, dpr: [2] },
+      { width: 1512, height: 982, weight: 0.25, dpr: [2] },
+      { width: 1728, height: 1117, weight: 0.15, dpr: [2] },
+      { width: 1680, height: 1050, weight: 0.10, dpr: [2] },
+      { width: 1920, height: 1080, weight: 0.15, dpr: [1, 2] }   // External display
     ];
     
     // Mobile viewport sizes for mobile emulation
@@ -467,8 +480,10 @@ export class StealthBrowserManager {
         height: fingerprint.screen.height
       },
       
-      // Bypass various detections
-      bypassCSP: true,
+      // bypassCSP is deliberately NOT set: ignoring a page's Content Security
+      // Policy is "invalid behavior for a normal browser" and rebrowser's
+      // bot detector flags it red (R15, 2026-09-04). Every init script here
+      // goes through addInitScript/evaluate, which need no CSP bypass.
       javaScriptEnabled: true
     };
 
@@ -511,14 +526,17 @@ export class StealthBrowserManager {
     const selectedOS = this.selectOS(config);
     const persona = this.selectLocalePersona(config);
     const userAgent = this.selectRealisticUserAgent(config, selectedOS);
+    const display = this.generateDisplay(selectedOS, config);
     const fingerprint = {
       userAgent,
       locale: persona.locale,
-      viewport: config.customViewport || this.selectWeightedViewport(),
+      viewport: display.viewport,
       timezone: config.timezone || persona.timezone,
-      deviceScaleFactor: this.randomFloat(1, 2, 1),
-      isMobile: Math.random() < 0.1, // 10% mobile
-      hasTouch: Math.random() < 0.15, // 15% touch
+      deviceScaleFactor: display.deviceScaleFactor,
+      // Every user agent in the pool is a desktop one; mobile emulation under
+      // it (10% of fingerprints until R16) contradicted the UA outright.
+      isMobile: false,
+      hasTouch: selectedOS === 'windows' && Math.random() < 0.15, // touch laptops
       colorScheme: Math.random() < 0.3 ? 'dark' : 'light',
       reducedMotion: Math.random() < 0.1 ? 'reduce' : 'no-preference',
       forcedColors: Math.random() < 0.05 ? 'active' : 'none',
@@ -532,7 +550,8 @@ export class StealthBrowserManager {
       fonts: this.generateAdvancedFontList(selectedOS),
       plugins: this.generateAdvancedPluginList(),
       geolocation: this.generateRealisticGeolocation(persona),
-      screen: this.generateAdvancedScreenProperties(),
+      screen: display.screen,
+      window: display.window,
       battery: this.generateBatteryFingerprint()
     };
 
@@ -570,7 +589,17 @@ export class StealthBrowserManager {
     // An unmodelled locale still gets a coherent timezone/geolocation pair.
     const pool = exact.length ? exact : (sameLanguage.length ? sameLanguage : this.localePersonas);
 
-    const persona = pool[Math.floor(Math.random() * pool.length)];
+    // The host's own zone is where the egress address is. A persona drawn at
+    // random put America/Chicago in JavaScript beside a Florida IP, and both
+    // pixelscan and iphey called the timezone spoofed (R16, 2026-09-04). So a
+    // persona in the host zone is preferred — from the requested locale's
+    // pool, or from any locale when nothing beyond the default was asked for.
+    // An explicit config.timezone still overrides in the caller.
+    const hostZone = this.hostTimezone();
+    const inHostZone = (personas) => (hostZone ? personas.find((p) => p.timezone === hostZone) : undefined);
+    const persona = inHostZone(pool)
+      || (requested.toLowerCase() === 'en-us' ? inHostZone(this.localePersonas) : undefined)
+      || pool[Math.floor(Math.random() * pool.length)];
     return { ...persona, locale: requested };
   }
 
@@ -643,10 +672,58 @@ export class StealthBrowserManager {
   }
 
   /**
-   * Select viewport size based on weights
+   * One display for the whole fingerprint: the screen is drawn from the pool
+   * and the window derived from it as a maximised browser — viewport width is
+   * the screen width, viewport height the available height minus the browser
+   * chrome, outer size the available area. The screen and the viewport used
+   * to be drawn from the pool independently and the scale factor was a random
+   * float, so a 1536×864 window sat on a 1366×768 screen at a scale of 1.7
+   * (which Chromium reports as 1.7000000476837158 and a 1408.0000305175781px
+   * screen) — pixelscan's "Your browser fingerprint is inconsistent" (R16,
+   * 2026-09-04). A custom viewport gets the smallest pooled screen it fits on.
+   */
+  generateDisplay(selectedOS, config = {}) {
+    const chrome = selectedOS === 'macos' ? 87 : 85;
+    const taskbar = selectedOS === 'macos' ? 25 : (selectedOS === 'linux' ? 27 : 40);
+    const pool = selectedOS === 'macos' ? this.macScreenSizes : this.viewportSizes;
+
+    let screen;
+    let viewport;
+    if (config.customViewport) {
+      viewport = { width: config.customViewport.width, height: config.customViewport.height };
+      const needed = { width: viewport.width, height: viewport.height + chrome + taskbar };
+      screen = pool
+        .filter((s) => s.width >= needed.width && s.height >= needed.height)
+        .sort((a, b) => a.width * a.height - b.width * b.height)[0]
+        || { ...needed, dpr: [1] };
+    } else {
+      screen = this.weightedRandomFromArray(pool);
+      viewport = { width: screen.width, height: screen.height - taskbar - chrome };
+    }
+
+    return {
+      viewport,
+      deviceScaleFactor: screen.dpr[Math.floor(Math.random() * screen.dpr.length)],
+      screen: {
+        width: screen.width,
+        height: screen.height,
+        availWidth: screen.width,
+        availHeight: screen.height - taskbar,
+        colorDepth: 24,
+        pixelDepth: 24,
+        orientation: { angle: 0, type: 'landscape-primary' }
+      },
+      window: { outerWidth: viewport.width, outerHeight: viewport.height + chrome }
+    };
+  }
+
+  /**
+   * The machine's own timezone, or null when it is the UTC of a container —
+   * set TZ on such a host to the zone its egress address geolocates to.
    */
-  selectWeightedViewport() {
-    return this.weightedRandomFromArray(this.viewportSizes);
+  hostTimezone() {
+    const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+    return zone && zone !== 'UTC' && !/^(?:Etc|GMT)/.test(zone) ? zone : null;
   }
 
   /**
@@ -704,10 +781,10 @@ export class StealthBrowserManager {
    * @param {string} [userAgent] — the selected user agent string
    */
   generateSecChUaHeader(userAgent = '') {
-    // Extract Chrome major version from the UA (e.g. "Chrome/121.0.0.0" → "121").
-    // Fall back to 121 if the UA is not a Chrome UA.
+    // Extract Chrome major version from the UA (e.g. "Chrome/151.0.0.0" → "151").
+    // Fall back to the bundled Chromium major if the UA is not a Chrome UA.
     const match = userAgent.match(/Chrome\/(\d+)/i);
-    const version = match ? match[1] : '121';
+    const version = match ? match[1] : '151';
 
     const brands = [
       { brand: 'Not_A Brand', version: '8' },
@@ -1134,25 +1211,6 @@ export class StealthBrowserManager {
     };
   }
 
-  /**
-   * Generate advanced screen properties
-   */
-  generateAdvancedScreenProperties() {
-    const viewport = this.selectWeightedViewport();
-    
-    return {
-      width: viewport.width,
-      height: viewport.height,
-      availWidth: viewport.width,
-      availHeight: viewport.height - (30 + Math.floor(Math.random() * 20)), // Account for taskbar
-      colorDepth: Math.random() < 0.95 ? 24 : 32,
-      pixelDepth: 24,
-      orientation: {
-        angle: 0,
-        type: 'landscape-primary'
-      }
-    };
-  }
 
   /**
    * Generate battery API fingerprint
@@ -1282,6 +1340,26 @@ export class StealthBrowserManager {
           Object.defineProperty(pluginArray, plugin.name, { value: plugin });
         });
         Object.defineProperty(navigator, 'plugins', { get: () => pluginArray });
+
+        // Real Chrome pairs those plugins with two navigator.mimeTypes entries
+        // (application/pdf, text/pdf). Five plugins beside an empty mimeTypes
+        // is its own tell — infosimples' detect-headless read "5 plugins /
+        // 0 mime types" (R15, 2026-09-04).
+        if (typeof MimeTypeArray !== 'undefined') {
+          const mimeTypeArray = Object.create(MimeTypeArray.prototype);
+          const mimeTypes = Array.from(plugins[0]);
+          Object.defineProperties(mimeTypeArray, {
+            length: { value: mimeTypes.length },
+            item: { value: (index) => mimeTypes[index] || null },
+            namedItem: { value: (type) => mimeTypes.find((m) => m.type === type) || null },
+            [Symbol.iterator]: { value: function* () { yield* mimeTypes; } }
+          });
+          mimeTypes.forEach((mimeType, index) => {
+            Object.defineProperty(mimeTypeArray, index, { value: mimeType, enumerable: true });
+            Object.defineProperty(mimeTypeArray, mimeType.type, { value: mimeType });
+          });
+          Object.defineProperty(navigator, 'mimeTypes', { get: () => mimeTypeArray });
+        }
       }
 
       // Override languages with the fingerprint's own locale — a hardcoded
@@ -1449,6 +1527,21 @@ export class StealthBrowserManager {
           
           return ctx;
         };
+
+        // A webgl2 context (and any OffscreenCanvas context) never passed
+        // through the wrapper above: iphey read the GPU through webgl2 and
+        // printed "SwiftShader" under a Windows user agent (R16, 2026-09-04).
+        // The two unmasked parameters are answered on the prototypes, where
+        // every context of either kind reads them.
+        for (const Context of [window.WebGLRenderingContext, window.WebGL2RenderingContext]) {
+          if (!Context || !Context.prototype) continue;
+          const getParameter = Context.prototype.getParameter;
+          Context.prototype.getParameter = function(parameter) {
+            if (parameter === 37446) return webglConfig.renderer;
+            if (parameter === 37445) return webglConfig.vendor;
+            return getParameter.call(this, parameter);
+          };
+        }
       }, fingerprint.webGL);
     }
 
@@ -1543,7 +1636,7 @@ export class StealthBrowserManager {
     }
 
     // Screen resolution spoofing
-    await context.addInitScript((screenConfig) => {
+    await context.addInitScript(({ screen: screenConfig, window: windowConfig }) => {
       Object.defineProperties(screen, {
         width: { value: screenConfig.width, configurable: true },
         height: { value: screenConfig.height, configurable: true },
@@ -1552,7 +1645,18 @@ export class StealthBrowserManager {
         colorDepth: { value: screenConfig.colorDepth, configurable: true },
         pixelDepth: { value: screenConfig.pixelDepth, configurable: true }
       });
-    }, fingerprint.screen);
+      // Headless Chromium reports outerWidth/outerHeight equal to the inner
+      // size — a window with no tabs and no toolbar.
+      if (windowConfig) {
+        Object.defineProperty(window, 'outerWidth', { get: () => windowConfig.outerWidth, configurable: true });
+        Object.defineProperty(window, 'outerHeight', { get: () => windowConfig.outerHeight, configurable: true });
+      }
+    }, {
+      screen: fingerprint.screen,
+      // camoufox runs without a fixed viewport (see createStealthContext), so
+      // an outer size derived from one would contradict the real window.
+      window: this._launchedEngine === 'camoufox' ? null : fingerprint.window
+    });
 
     // Timezone spoofing
     if (config.spoofTimezone) {
@@ -1828,16 +1932,34 @@ export class StealthBrowserManager {
     const { contextId } = await this.createStealthContext({ ...stealthConfig, engine });
     try {
       const page = await this.createStealthPage(contextId);
+      let crashed = false;
+      page.on('crash', () => { crashed = true; });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
       await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
       if (wait_for > 0) await page.waitForTimeout(wait_for);
 
-      const [title, html, text] = await Promise.all([
-        page.title().catch(() => ''),
-        page.content().catch(() => ''),
-        page.innerText('body').catch(() => '')
-      ]);
+      // A failure to read the document is a failure. With every read wrapped
+      // in .catch(() => ''), a renderer that crashed or a page closed during
+      // the wait (iphey.com under software GL, R16 2026-09-04) came back as
+      // success:true with an empty title and body — indistinguishable from a
+      // page that rendered nothing. The body text is read through evaluate,
+      // not innerText('body'): a frameset document has no <body>, and
+      // innerText waited its full 30s for one before throwing.
+      let title; let html; let text;
+      try {
+        [title, html, text] = await Promise.all([
+          page.title(),
+          page.content(),
+          page.evaluate(() => (document.body ? document.body.innerText : ''))
+        ]);
+      } catch (error) {
+        const what = crashed ? 'crashed' : (page.isClosed() ? 'was closed' : 'could not be read');
+        throw new Error(
+          `The stealth page ${what} before its content could be read (${String(error.message).split('\n')[0]}). ` +
+          'The page did not render; retry, or use scrape_with_actions for a page this heavy.'
+        );
+      }
       const shot = screenshot
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
@@ -1848,47 +1970,34 @@ export class StealthBrowserManager {
     }
   }
 
+  /**
+   * Whether a request is dropped before it leaves the browser. Only the
+   * advanced level sheds assets, never by URL and never the document itself.
+   * @param {string} resourceType - Playwright's request.resourceType()
+   * @param {string} level - stealth level
+   * @returns {boolean}
+   */
+  static shouldAbortRequest(resourceType, level) {
+    if (level !== 'advanced') return false;
+    if (!['image', 'font', 'stylesheet'].includes(resourceType)) return false;
+    // Allow some images/fonts to maintain realism
+    return Math.random() >= 0.3;
+  }
+
   /**
    * Apply page-level stealth measures
    */
   async applyPageStealthMeasures(page, config, fingerprint) {
-    // Enhanced resource blocking with stealth considerations
+    // Resource routing. Nothing is aborted by URL any more: the old list
+    // blocked challenges.cloudflare.com and every URL containing "selenium",
+    // "webdriver" or "puppeteer" — the first meant a Cloudflare challenge
+    // could never complete (the interstitial named the blocked host), the
+    // second killed the top-level navigation to www.selenium.dev on every
+    // stealth path (R15, 2026-09-04). A vendor script that never loads is a
+    // louder tell than one that runs.
     await page.route('**/*', route => {
-      const resourceType = route.request().resourceType();
-      const url = route.request().url();
-      
-      // Block known bot detection resources
-      const blockedDomains = [
-        'botd.fpjs.io',
-        'challenges.cloudflare.com',
-        'datadome.co',
-        'perimeterx.net',
-        'distilnetworks.com'
-      ];
-      
-      if (blockedDomains.some(domain => url.includes(domain))) {
+      if (StealthBrowserManager.shouldAbortRequest(route.request().resourceType(), config.level)) {
         route.abort();
-        return;
-      }
-
-      // Don't block detection-related resources that might be expected
-      if (url.includes('webdriver') || url.includes('selenium') || url.includes('puppeteer')) {
-        route.abort();
-        return;
-      }
-
-      // Selective resource blocking based on level
-      if (config.level === 'advanced') {
-        if (['image', 'font', 'stylesheet'].includes(resourceType)) {
-          // Allow some images/fonts to maintain realism
-          if (Math.random() < 0.3) {
-            route.continue();
-          } else {
-            route.abort();
-          }
-        } else {
-          route.continue();
-        }
       } else {
         route.continue();
       }

package/src/core/analysis/ContentAnalyzer.js

@@ -6,7 +6,8 @@
 import {
   LANGUAGE_NAMES,
   cjkScriptCounts as sharedCjkScriptCounts,
-  detectLanguage as sharedDetectLanguage
+  detectLanguage as sharedDetectLanguage,
+  needsWordSegmentation
 } from '../../utils/languageDetection.js';
 import nlp from 'compromise';
 import { z } from 'zod';
@@ -168,7 +169,9 @@ export class ContentAnalyzer {
    * @returns {string[]} - Word tokens
    */
   tokenizeWords(text) {
-    return this.isCjkText(text)
+    // CJK and the unspaced South-East Asian scripts (Thai, Lao, Khmer,
+    // Burmese) have no word spaces to split on.
+    return needsWordSegmentation(text)
       ? this.segmentWords(text)
       : text.split(/\s+/).filter(w => w.length > 0);
   }
@@ -395,7 +398,7 @@ export class ContentAnalyzer {
     const freq = {};
     const sentenceWords = sentences.map(sentence => {
       const words = nlp(sentence).terms().out('array')
-        .map(w => w.toLowerCase().replace(/[^a-z0-9]/g, ''))
+        .map(w => w.toLowerCase().replace(/[^\p{L}\p{N}]/gu, ''))
         .filter(w => w.length > 2 && !this.isStopWord(w));
       words.forEach(w => { freq[w] = (freq[w] || 0) + 1; });
       return words;
@@ -468,7 +471,7 @@ export class ContentAnalyzer {
       
       allPhrases.forEach(phrase => {
         // Strip edge punctuation and skip phrases made only of stop words
-        const cleaned = phrase.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '');
+        const cleaned = phrase.toLowerCase().trim().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
         const words = cleaned.split(/\s+/);
         if (cleaned.length > 2 && words.some(w => !this.isStopWord(w))) {
           phraseCount[cleaned] = (phraseCount[cleaned] || 0) + 1;
@@ -547,7 +550,7 @@ export class ContentAnalyzer {
       // (corporate suffix or organization-headed alias definition).
       const demotedAcronyms = [];
       organizations = organizations.filter(e => {
-        if (/^[A-Z]{2,}$/.test(e) && !this.hasOrganizationEvidence(e, text)) {
+        if (/^\p{Lu}{2,}$/u.test(e) && !this.hasOrganizationEvidence(e, text)) {
           demotedAcronyms.push(e);
           return false;
         }
@@ -562,11 +565,14 @@ export class ContentAnalyzer {
         ...people, ...places, ...organizations, ...other
       ].map(e => e.toLowerCase()));
 
-      const properNouns = (text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [])
+      // Unicode classes, not [A-Z]: the ASCII version cut "Universität" to
+      // "Universit" and "Wisłą" to "Wis" and dropped "Środkowej" outright (R16).
+      // \b stays ASCII-only even under the u flag, hence the lookbehind.
+      const properNouns = (text.match(/(?<![\p{L}\p{N}])\p{Lu}[\p{L}.]+(?:\s+\p{Lu}[\p{L}.]+)*/gu) || [])
         // Split matches that crossed a sentence boundary ("XPath. In") — a
         // period after a lowercase letter followed by a capital is a sentence
         // end, while abbreviations ("U.S. Holdings") stay intact.
-        .flatMap(n => n.split(/(?<=[a-z]\.)\s+(?=[A-Z])/));
+        .flatMap(n => n.split(/(?<=\p{Ll}\.)\s+(?=\p{Lu})/u));
       const supplemental = this.dedupeEntities(properNouns.map(n => this.cleanEntityText(n)))
         .filter(n => n.length > 1
           && !existingEntities.has(n.toLowerCase())
@@ -653,7 +659,7 @@ export class ContentAnalyzer {
       
       [...nouns, ...verbs, ...adjectives].forEach(term => {
         // Strip leading/trailing punctuation but preserve internal periods (e.g. Node.js)
-        const cleaned = term.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9.]+$/gi, '').replace(/\.+$/, '');
+        const cleaned = term.toLowerCase().trim().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}.]+$/gu, '').replace(/\.+$/, '');
         if (cleaned.length > 2 && !this.isStopWord(cleaned)) {
           termFreq[cleaned] = (termFreq[cleaned] || 0) + 1;
 
@@ -890,7 +896,7 @@ export class ContentAnalyzer {
       entity = entity.replace(/[\s"'‘’“”`)\]}>,;:!?«»–—-]+$/, '');
       if (entity.endsWith('.')) {
         const lastToken = entity.split(/\s+/).pop();
-        const isAbbreviation = /^(?:[A-Za-z]\.)+$/.test(lastToken)
+        const isAbbreviation = /^(?:\p{L}\.)+$/u.test(lastToken)
           || /^(?:inc|corp|ltd|co|llc|jr|sr|st|mr|mrs|ms|dr|no|vs?)\.$/i.test(lastToken);
         if (!isAbbreviation) {
           entity = entity.replace(/\.+$/, '');
@@ -941,13 +947,13 @@ export class ContentAnalyzer {
     if (entity.includes(' ')) return false;
     const lower = entity.toLowerCase();
     if (this.isStopWord(lower)) return true;
-    if (!/^[A-Z][a-z]+$/.test(entity)) return false;
+    if (!/^\p{Lu}\p{Ll}+$/u.test(entity)) return false;
 
     // Word also appears in lowercase → ordinary word, capitalized only by position
-    if (new RegExp(`(?:^|[^A-Za-z])${lower}(?:[^A-Za-z]|$)`).test(text)) return true;
+    if (new RegExp(`(?:^|\\P{L})${lower}(?:\\P{L}|$)`, 'u').test(text)) return true;
 
     // Keep only if at least one occurrence is NOT at a sentence start
-    const occurrence = new RegExp(`\\b${entity}\\b`, 'g');
+    const occurrence = new RegExp(`(?<![\\p{L}\\p{N}])${entity}(?![\\p{L}\\p{N}])`, 'gu');
     let match;
     while ((match = occurrence.exec(text)) !== null) {
       const before = text.slice(0, match.index).replace(/[\s"'‘’“”()[\]]+$/, '');
@@ -972,13 +978,13 @@ export class ContentAnalyzer {
    * a corporate suffix follows it ("IBM Corp"), or it is introduced as the
    * alias of an organization-headed name ("American Airlines (AA)",
    * "French Data Protection Authority (CNIL)").
-   * @param {string} acronym - ALL-CAPS candidate (already /^[A-Z]{2,}$/)
+   * @param {string} acronym - ALL-CAPS candidate (already /^\p{Lu}{2,}$/u)
    * @param {string} text - Full source text
    * @returns {boolean} - True if the text corroborates the org classification
    */
   hasOrganizationEvidence(acronym, text) {
     const corporateSuffix = new RegExp(
-      `\\b${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:[^a-zA-Z]|$)`
+      `(?<!\\p{L})${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:\\P{L}|$)`, 'u'
     );
     if (corporateSuffix.test(text)) return true;
 

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -618,6 +618,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
     for (const stateData of capturedStates) {
       try {
         const $ = load(stateData.html);
+        // Never rendered as text — the same strip the batch worker does (R15).
+        $('script, style, noscript, template').remove();
 
         const state = {
           capturePoint: stateData.afterActionIndex + 1,
@@ -688,6 +690,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // over the post-action DOM here, the one place every caller reads.
       if (chainResult?.finalHtml && (params.extractionOptions?.selectors || params.formats?.includes('text'))) {
         const $ = load(chainResult.finalHtml);
+        // Never rendered as text — the same strip the batch worker does (R15).
+        $('script, style, noscript, template').remove();
         extractResult.content = extractResult.content || {};
         if (params.extractionOptions?.selectors) {
           extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);

package/src/tools/advanced/batchScrape/worker.js

@@ -120,7 +120,12 @@ function extractStructuredData($, selectors) {
   return extracted;
 }
 
-function generateFormats($, html, formats) {
+export function generateFormats($, html, formats) {
+  // Script, style, noscript and template bodies are never rendered text.
+  // Left in, the text format carried an entire theme-toggle script
+  // (erlang.org/downloads) and CSS rules (opennet.ru) — R15, 2026-09-04.
+  // `html` below is the raw document and stays untouched.
+  $('script, style, noscript, template').remove();
   const content = {};
   if (formats.includes('html')) content.html = html;
   if (formats.includes('text')) content.text = $('body').text().replace(/\s+/g, ' ').trim();

package/src/tools/extract/extractContent.js

@@ -8,6 +8,7 @@ import { ContentProcessor } from '../../core/processing/ContentProcessor.js';
 import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
+import { isThinMainContent } from '../scrape/_mainContent.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -215,8 +216,18 @@ export class ExtractContentTool {
         }
       });
 
-      // Step 3: Extract and format content
-      if (processingResult.readability) {
+      // Step 3: Extract and format content.
+      // Readability keeps one dense block; on a landing page that is a
+      // fraction of the visible text (libreoffice.org came back as its
+      // 509-character "Welcome" box at confidence 0.9, R16). A thin article
+      // is handed to the boilerplate-removal fallback and named as such.
+      const thin = processingResult.readability
+        ? isThinMainContent(processingResult.readability.content, html)
+        : null;
+      if (thin) {
+        processingResult.fallback_content = this.contentProcessor.extractFallbackContent(html);
+      }
+      if (processingResult.readability && !thin) {
         result.readability = processingResult.readability;
         result.content = {
           text: processingResult.readability.textContent || processingResult.readability.content,
@@ -234,7 +245,9 @@ export class ExtractContentTool {
           text: processingResult.fallback_content.content
         };
         result.extractionMethod = 'fallback_boilerplate_removal';
-        result.fallback_reason = 'Readability did not detect an article; used boilerplate-removal fallback';
+        result.fallback_reason = thin
+          ? `Readability kept only ${thin.kept} of ${thin.visible} visible characters (a landing page, not an article); used boilerplate-removal fallback`
+          : 'Readability did not detect an article; used boilerplate-removal fallback';
         result.confidence = 0.5;
         result.finalUrl = url;
       } else {

package/src/tools/scrape/_mainContent.js

@@ -40,6 +40,7 @@
 
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
+import { load } from 'cheerio';
 
 // How much of a table's text is compared against the article to decide whether
 // Readability already kept it. Long enough to be unique to that table.
@@ -226,4 +227,34 @@ export function extractMainContent(html, url) {
   };
 }
 
+// Below both of these, Readability's article is a fragment of the page, not
+// its main content.
+export const THIN_MAIN_CONTENT = { maxChars: 1500, maxShare: 0.35 };
+
+function visibleTextLength(html) {
+  const $ = load(html);
+  $('script, style, noscript, template').remove();
+  return $('body').text().replace(/\s+/g, ' ').trim().length;
+}
+
+/**
+ * Readability keeps ONE dense block and drops the rest. On a landing page
+ * that block is a fraction of what the visitor reads: gnome.org came back as
+ * ~150 of 1,666 visible characters and libreoffice.org as its 509-character
+ * "Welcome" box, both reported at full confidence (R16, 2026-09-04). A short
+ * article on a page with much more visible text is not the main content.
+ *
+ * @param {string} mainHtml - Readability's article HTML
+ * @param {string} pageHtml - the whole page
+ * @returns {{ kept: number, visible: number }|null} the two lengths when the
+ *   article is thin, null when it is the page's main content
+ */
+export function isThinMainContent(mainHtml, pageHtml) {
+  const kept = visibleTextLength(mainHtml || '');
+  if (kept >= THIN_MAIN_CONTENT.maxChars) return null;
+  const visible = visibleTextLength(pageHtml || '');
+  if (visible === 0 || kept / visible >= THIN_MAIN_CONTENT.maxShare) return null;
+  return { kept, visible };
+}
+
 export default extractMainContent;

package/src/tools/scrape/unifiedScrape.js

@@ -12,7 +12,7 @@
 
 import { z } from 'zod';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
-import { extractMainContent } from './_mainContent.js';
+import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
@@ -236,6 +236,15 @@ export class UnifiedScrapeTool {
           `mainContent: re-attached ${main.tablesRecovered} data table(s) that main-content extraction had dropped`
         );
       }
+      // A thin article on a landing page is not the main content (gnome.org
+      // came back as ~150 of 1,666 visible characters, R16): use the page.
+      const thin = main.html ? isThinMainContent(main.html, html) : null;
+      if (thin) {
+        mainHtml = html;
+        warnings.push(
+          `mainContent: main-content extraction kept ${thin.kept} of ${thin.visible} visible characters; the whole page is used instead`
+        );
+      }
       return mainHtml;
     }
 

package/src/utils/challengeDetection.js

@@ -0,0 +1,68 @@
+/**
+ * challengeDetection.js — recognise a bot-wall interstitial served as a page.
+ *
+ * Cloudflare, Amazon, DataDome, PerimeterX and Akamai all answer a blocked
+ * request with HTTP 200 and a page of their own: a title, some prose and a
+ * challenge script. Reported as a successful scrape, that page hides the
+ * block — producthunt.com came back "success:true, title: Just a moment..."
+ * for three regression rounds (R10 Q1 → R15, 2026-09-04). A title match is
+ * definitive; a script or form marker is definitive only on a short page,
+ * because a real page can legitimately embed a Turnstile widget.
+ */
+
+const SHORT_PAGE_CHARS = 4000;
+
+const CHALLENGES = [
+  {
+    vendor: 'cloudflare',
+    title: /^just a moment/i,
+    markers: /challenges\.cloudflare\.com|cf-chl-|_cf_chl_opt|cf_chl_rc_|window\._cf_chl/i,
+    evidence: 'a Cloudflare challenge script'
+  },
+  {
+    vendor: 'amazon',
+    // The robot check is the only Amazon page whose form posts to validateCaptcha.
+    definitive: /action="[^"]*validateCaptcha/i,
+    evidence: 'the validateCaptcha form'
+  },
+  {
+    vendor: 'datadome',
+    markers: /captcha-delivery\.com\/captcha|geo\.captcha-delivery\.com|dd\.captcha/i,
+    evidence: 'a DataDome captcha frame'
+  },
+  {
+    vendor: 'perimeterx',
+    markers: /px-captcha|_pxCaptcha|human-challenge/i,
+    evidence: 'a PerimeterX / HUMAN challenge element'
+  },
+  {
+    vendor: 'akamai',
+    title: /^access denied/i,
+    markers: /errors\.edgesuite\.net/i,
+    evidence: 'an Akamai access-denied page'
+  }
+];
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function detectChallengePage({ title = '', html = '', text = '' } = {}) {
+  const cleanTitle = (title || '').trim();
+  const visible = (text || '').replace(/\s+/g, ' ').trim();
+  const shortPage = visible.length < SHORT_PAGE_CHARS;
+  for (const challenge of CHALLENGES) {
+    if (challenge.title && challenge.title.test(cleanTitle)) {
+      return { vendor: challenge.vendor, evidence: `title "${cleanTitle}"` };
+    }
+    if (challenge.definitive && challenge.definitive.test(html)) {
+      return { vendor: challenge.vendor, evidence: challenge.evidence };
+    }
+    if (shortPage && challenge.markers && challenge.markers.test(html)) {
+      return { vendor: challenge.vendor, evidence: `${challenge.evidence} on a ${visible.length}-character page` };
+    }
+  }
+  return null;
+}
+
+export default detectChallengePage;

package/src/utils/languageDetection.js

@@ -158,6 +158,26 @@ export function isCjkText(text) {
   return cjkScriptCounts(text).share >= CJK_SCRIPT_THRESHOLD;
 }
 
+// Scripts written without spaces between words. Thai, Lao, Khmer and Burmese
+// need the same dictionary segmentation CJK gets; on the whitespace split a
+// 290-character Thai paragraph counted as 4 words (R15, 2026-09-04).
+const UNSPACED_SCRIPT_RE = /[\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/gu;
+
+/**
+ * True when the text needs Intl.Segmenter to find word boundaries — a CJK
+ * script, or one of the South-East Asian scripts that write no word spaces.
+ *
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function needsWordSegmentation(text) {
+  if (isCjkText(text)) return true;
+  const letters = (text.match(/\p{L}/gu) || []).length;
+  if (letters === 0) return false;
+  const unspaced = (text.match(UNSPACED_SCRIPT_RE) || []).length;
+  return unspaced / letters >= CJK_SCRIPT_THRESHOLD;
+}
+
 /**
  * Detect the language of a piece of text.
  *