crawlforge-mcp-server 5.6.1 → 5.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.1",
+  "version": "5.6.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.1",
+    "crawlforge-extractors": "^1.6.2",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -33,6 +33,7 @@ import { LocalizationManager } from "./src/core/LocalizationManager.js";
 // Stealth scrape: format conversion + the pre-fetch compliance gate (G5/G6/G7)
 import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
+import { detectChallengePage } from "./src/utils/challengeDetection.js";
 import { browserPreflight } from "./src/utils/robotsGate.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
@@ -105,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.1",
+  version: "5.6.2",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1377,12 +1378,21 @@ registerToolIfEnabled("stealth_mode", {
           stealthConfig
         });
 
+        // A bot-wall interstitial arrives as HTTP 200 with a title and prose
+        // of its own; reported as a successful scrape it hid the block for
+        // three rounds (R10 Q1 → R15). Name it — the content is still returned
+        // so the caller can see what the site served.
+        const challenge = detectChallengePage(scraped);
         result = {
-          success: true,
+          success: !challenge,
           url: scraped.url,
           title: scraped.title,
           content: stealthScrapeFormats(formats, scraped)
         };
+        if (challenge) {
+          result.blocked = challenge;
+          result.error = `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`;
+        }
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as
         // scrape and scrape_with_actions, so the base64 never bloats the result.
@@ -1452,6 +1462,12 @@ registerToolIfEnabled("stealth_mode", {
               ok: response ? response.ok() : null,
               title: await page.title().catch(() => null)
             };
+            const challenge = detectChallengePage({
+              title: navigation.title || '',
+              html: await page.content().catch(() => ''),
+              text: await page.innerText('body').catch(() => '')
+            });
+            if (challenge) navigation.blocked = challenge;
           }
         } finally {
           // No operation can ever reference this page again — keeping it open

package/src/core/StealthBrowserManager.js

@@ -140,47 +140,50 @@ export class StealthBrowserManager {
       }
     };
 
-    // Enhanced User agent pools with realistic patterns
+    // User agent pools. Chrome majors track the bundled Chromium (151 at
+    // playwright-core 1.62) and Firefox tracks the Camoufox build (135): a
+    // UA thirty majors behind the engine — the pool sat at 119–121 while
+    // Chrome 152 shipped — is a staleness tell for anything that reads
+    // sec-ch-ua (R15, 2026-09-04).
     this.userAgentPools = {
       chrome: {
         windows: [
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
         ],
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
         ],
         linux: [
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
         ]
       },
       firefox: {
         windows: [
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0'
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0'
         ],
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0'
         ],
         linux: [
-          'Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
-          'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0'
+          'Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0',
+          'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0'
         ]
       },
       safari: {
         macos: [
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15',
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15'
         ]
       }
     };
-    
+
     // Operating system distributions for realistic user agent selection
     this.osDistribution = {
       windows: 0.75,
@@ -467,8 +470,10 @@ export class StealthBrowserManager {
         height: fingerprint.screen.height
       },
       
-      // Bypass various detections
-      bypassCSP: true,
+      // bypassCSP is deliberately NOT set: ignoring a page's Content Security
+      // Policy is "invalid behavior for a normal browser" and rebrowser's
+      // bot detector flags it red (R15, 2026-09-04). Every init script here
+      // goes through addInitScript/evaluate, which need no CSP bypass.
       javaScriptEnabled: true
     };
 
@@ -704,10 +709,10 @@ export class StealthBrowserManager {
    * @param {string} [userAgent] — the selected user agent string
    */
   generateSecChUaHeader(userAgent = '') {
-    // Extract Chrome major version from the UA (e.g. "Chrome/121.0.0.0" → "121").
-    // Fall back to 121 if the UA is not a Chrome UA.
+    // Extract Chrome major version from the UA (e.g. "Chrome/151.0.0.0" → "151").
+    // Fall back to the bundled Chromium major if the UA is not a Chrome UA.
     const match = userAgent.match(/Chrome\/(\d+)/i);
-    const version = match ? match[1] : '121';
+    const version = match ? match[1] : '151';
 
     const brands = [
       { brand: 'Not_A Brand', version: '8' },
@@ -1282,6 +1287,26 @@ export class StealthBrowserManager {
           Object.defineProperty(pluginArray, plugin.name, { value: plugin });
         });
         Object.defineProperty(navigator, 'plugins', { get: () => pluginArray });
+
+        // Real Chrome pairs those plugins with two navigator.mimeTypes entries
+        // (application/pdf, text/pdf). Five plugins beside an empty mimeTypes
+        // is its own tell — infosimples' detect-headless read "5 plugins /
+        // 0 mime types" (R15, 2026-09-04).
+        if (typeof MimeTypeArray !== 'undefined') {
+          const mimeTypeArray = Object.create(MimeTypeArray.prototype);
+          const mimeTypes = Array.from(plugins[0]);
+          Object.defineProperties(mimeTypeArray, {
+            length: { value: mimeTypes.length },
+            item: { value: (index) => mimeTypes[index] || null },
+            namedItem: { value: (type) => mimeTypes.find((m) => m.type === type) || null },
+            [Symbol.iterator]: { value: function* () { yield* mimeTypes; } }
+          });
+          mimeTypes.forEach((mimeType, index) => {
+            Object.defineProperty(mimeTypeArray, index, { value: mimeType, enumerable: true });
+            Object.defineProperty(mimeTypeArray, mimeType.type, { value: mimeType });
+          });
+          Object.defineProperty(navigator, 'mimeTypes', { get: () => mimeTypeArray });
+        }
       }
 
       // Override languages with the fingerprint's own locale — a hardcoded
@@ -1848,47 +1873,34 @@ export class StealthBrowserManager {
     }
   }
 
+  /**
+   * Whether a request is dropped before it leaves the browser. Only the
+   * advanced level sheds assets, never by URL and never the document itself.
+   * @param {string} resourceType - Playwright's request.resourceType()
+   * @param {string} level - stealth level
+   * @returns {boolean}
+   */
+  static shouldAbortRequest(resourceType, level) {
+    if (level !== 'advanced') return false;
+    if (!['image', 'font', 'stylesheet'].includes(resourceType)) return false;
+    // Allow some images/fonts to maintain realism
+    return Math.random() >= 0.3;
+  }
+
   /**
    * Apply page-level stealth measures
    */
   async applyPageStealthMeasures(page, config, fingerprint) {
-    // Enhanced resource blocking with stealth considerations
+    // Resource routing. Nothing is aborted by URL any more: the old list
+    // blocked challenges.cloudflare.com and every URL containing "selenium",
+    // "webdriver" or "puppeteer" — the first meant a Cloudflare challenge
+    // could never complete (the interstitial named the blocked host), the
+    // second killed the top-level navigation to www.selenium.dev on every
+    // stealth path (R15, 2026-09-04). A vendor script that never loads is a
+    // louder tell than one that runs.
     await page.route('**/*', route => {
-      const resourceType = route.request().resourceType();
-      const url = route.request().url();
-      
-      // Block known bot detection resources
-      const blockedDomains = [
-        'botd.fpjs.io',
-        'challenges.cloudflare.com',
-        'datadome.co',
-        'perimeterx.net',
-        'distilnetworks.com'
-      ];
-      
-      if (blockedDomains.some(domain => url.includes(domain))) {
-        route.abort();
-        return;
-      }
-
-      // Don't block detection-related resources that might be expected
-      if (url.includes('webdriver') || url.includes('selenium') || url.includes('puppeteer')) {
+      if (StealthBrowserManager.shouldAbortRequest(route.request().resourceType(), config.level)) {
         route.abort();
-        return;
-      }
-
-      // Selective resource blocking based on level
-      if (config.level === 'advanced') {
-        if (['image', 'font', 'stylesheet'].includes(resourceType)) {
-          // Allow some images/fonts to maintain realism
-          if (Math.random() < 0.3) {
-            route.continue();
-          } else {
-            route.abort();
-          }
-        } else {
-          route.continue();
-        }
       } else {
         route.continue();
       }

package/src/core/analysis/ContentAnalyzer.js

@@ -6,7 +6,8 @@
 import {
   LANGUAGE_NAMES,
   cjkScriptCounts as sharedCjkScriptCounts,
-  detectLanguage as sharedDetectLanguage
+  detectLanguage as sharedDetectLanguage,
+  needsWordSegmentation
 } from '../../utils/languageDetection.js';
 import nlp from 'compromise';
 import { z } from 'zod';
@@ -168,7 +169,9 @@ export class ContentAnalyzer {
    * @returns {string[]} - Word tokens
    */
   tokenizeWords(text) {
-    return this.isCjkText(text)
+    // CJK and the unspaced South-East Asian scripts (Thai, Lao, Khmer,
+    // Burmese) have no word spaces to split on.
+    return needsWordSegmentation(text)
       ? this.segmentWords(text)
       : text.split(/\s+/).filter(w => w.length > 0);
   }

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -618,6 +618,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
     for (const stateData of capturedStates) {
       try {
         const $ = load(stateData.html);
+        // Never rendered as text — the same strip the batch worker does (R15).
+        $('script, style, noscript, template').remove();
 
         const state = {
           capturePoint: stateData.afterActionIndex + 1,
@@ -688,6 +690,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // over the post-action DOM here, the one place every caller reads.
       if (chainResult?.finalHtml && (params.extractionOptions?.selectors || params.formats?.includes('text'))) {
         const $ = load(chainResult.finalHtml);
+        // Never rendered as text — the same strip the batch worker does (R15).
+        $('script, style, noscript, template').remove();
         extractResult.content = extractResult.content || {};
         if (params.extractionOptions?.selectors) {
           extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);

package/src/tools/advanced/batchScrape/worker.js

@@ -120,7 +120,12 @@ function extractStructuredData($, selectors) {
   return extracted;
 }
 
-function generateFormats($, html, formats) {
+export function generateFormats($, html, formats) {
+  // Script, style, noscript and template bodies are never rendered text.
+  // Left in, the text format carried an entire theme-toggle script
+  // (erlang.org/downloads) and CSS rules (opennet.ru) — R15, 2026-09-04.
+  // `html` below is the raw document and stays untouched.
+  $('script, style, noscript, template').remove();
   const content = {};
   if (formats.includes('html')) content.html = html;
   if (formats.includes('text')) content.text = $('body').text().replace(/\s+/g, ' ').trim();

package/src/utils/challengeDetection.js

@@ -0,0 +1,68 @@
+/**
+ * challengeDetection.js — recognise a bot-wall interstitial served as a page.
+ *
+ * Cloudflare, Amazon, DataDome, PerimeterX and Akamai all answer a blocked
+ * request with HTTP 200 and a page of their own: a title, some prose and a
+ * challenge script. Reported as a successful scrape, that page hides the
+ * block — producthunt.com came back "success:true, title: Just a moment..."
+ * for three regression rounds (R10 Q1 → R15, 2026-09-04). A title match is
+ * definitive; a script or form marker is definitive only on a short page,
+ * because a real page can legitimately embed a Turnstile widget.
+ */
+
+const SHORT_PAGE_CHARS = 4000;
+
+const CHALLENGES = [
+  {
+    vendor: 'cloudflare',
+    title: /^just a moment/i,
+    markers: /challenges\.cloudflare\.com|cf-chl-|_cf_chl_opt|cf_chl_rc_|window\._cf_chl/i,
+    evidence: 'a Cloudflare challenge script'
+  },
+  {
+    vendor: 'amazon',
+    // The robot check is the only Amazon page whose form posts to validateCaptcha.
+    definitive: /action="[^"]*validateCaptcha/i,
+    evidence: 'the validateCaptcha form'
+  },
+  {
+    vendor: 'datadome',
+    markers: /captcha-delivery\.com\/captcha|geo\.captcha-delivery\.com|dd\.captcha/i,
+    evidence: 'a DataDome captcha frame'
+  },
+  {
+    vendor: 'perimeterx',
+    markers: /px-captcha|_pxCaptcha|human-challenge/i,
+    evidence: 'a PerimeterX / HUMAN challenge element'
+  },
+  {
+    vendor: 'akamai',
+    title: /^access denied/i,
+    markers: /errors\.edgesuite\.net/i,
+    evidence: 'an Akamai access-denied page'
+  }
+];
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function detectChallengePage({ title = '', html = '', text = '' } = {}) {
+  const cleanTitle = (title || '').trim();
+  const visible = (text || '').replace(/\s+/g, ' ').trim();
+  const shortPage = visible.length < SHORT_PAGE_CHARS;
+  for (const challenge of CHALLENGES) {
+    if (challenge.title && challenge.title.test(cleanTitle)) {
+      return { vendor: challenge.vendor, evidence: `title "${cleanTitle}"` };
+    }
+    if (challenge.definitive && challenge.definitive.test(html)) {
+      return { vendor: challenge.vendor, evidence: challenge.evidence };
+    }
+    if (shortPage && challenge.markers && challenge.markers.test(html)) {
+      return { vendor: challenge.vendor, evidence: `${challenge.evidence} on a ${visible.length}-character page` };
+    }
+  }
+  return null;
+}
+
+export default detectChallengePage;

package/src/utils/languageDetection.js

@@ -158,6 +158,26 @@ export function isCjkText(text) {
   return cjkScriptCounts(text).share >= CJK_SCRIPT_THRESHOLD;
 }
 
+// Scripts written without spaces between words. Thai, Lao, Khmer and Burmese
+// need the same dictionary segmentation CJK gets; on the whitespace split a
+// 290-character Thai paragraph counted as 4 words (R15, 2026-09-04).
+const UNSPACED_SCRIPT_RE = /[\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/gu;
+
+/**
+ * True when the text needs Intl.Segmenter to find word boundaries — a CJK
+ * script, or one of the South-East Asian scripts that write no word spaces.
+ *
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function needsWordSegmentation(text) {
+  if (isCjkText(text)) return true;
+  const letters = (text.match(/\p{L}/gu) || []).length;
+  if (letters === 0) return false;
+  const unspaced = (text.match(UNSPACED_SCRIPT_RE) || []).length;
+  return unspaced / letters >= CJK_SCRIPT_THRESHOLD;
+}
+
 /**
  * Detect the language of a piece of text.
  *