crawlforge-mcp-server 5.5.1 → 5.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.5.1",
+  "version": "5.5.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.5.1",
+  version: "5.5.2",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/tools/scrape/_mainContent.js

@@ -1,14 +1,39 @@
 /**
  * _mainContent.js — Readability main-content extraction, plus recovery of the
- * data tables Readability drops.
+ * content Readability drops.
  *
- * Readability keeps one article candidate and discards everything outside it.
- * On a table-led page that silently loses the payload: Wikipedia's *List of
- * S&P 500 companies* came back with zero table rows at scrape's default
- * onlyMainContent:true, and 505 pipe-table lines with it off. No Readability
- * option recovers them (charThreshold:100 and nbTopCandidates:20 were both
- * probed) — the tables are simply not in the candidate — so they are
- * re-attached afterwards.
+ * Two independent losses are repaired here.
+ *
+ * 1. Readability keeps one article candidate and discards everything outside
+ *    it. On a table-led page that silently loses the payload: Wikipedia's
+ *    *List of S&P 500 companies* came back with zero table rows at scrape's
+ *    default onlyMainContent:true, and 505 pipe-table lines with it off. No
+ *    Readability option recovers them (charThreshold:100 and nbTopCandidates:20
+ *    were both probed) — the tables are simply not in the candidate — so they
+ *    are re-attached afterwards.
+ *
+ * 2. Readability's class regexes are unanchored, so they fire on words their
+ *    author never meant. Two collisions between them deleted every code
+ *    example on next-intl.dev/docs/routing/setup:
+ *
+ *      - `unlikelyCandidates` contains `extra`, which matches inside `nextra-*`
+ *        — the class prefix on every element Nextra emits, and Nextra builds a
+ *        large share of the JS ecosystem's documentation. None of those classes
+ *        match `okMaybeItsACandidate`, so the elements are deleted before
+ *        scoring runs: all 96 inline `<code>` spans and the "setRequestLocale
+ *        is legacy" callout went with them.
+ *      - `negative` contains `hidden`, which matches Tailwind's
+ *        `overflow-hidden`. That is only -25 class weight, but it is enough for
+ *        `_cleanConditionally` to drop the wrapper around all 12 `<pre>` blocks.
+ *
+ *    Both losses were silent, and the second is the dangerous one: the prose
+ *    survived, so the page read as complete while every code example and the
+ *    warning callout were gone, and sentences came back with holes where their
+ *    inline code had been ("In order to use unique pathnames …, can be used to
+ *    handle"). `sanitizeAccidentalClasses` strips just those class tokens
+ *    before Readability sees the document, restoring 11 of the 12 blocks — the
+ *    12th sits inside a collapsed `<details>`, which Readability drops whole
+ *    and which this deliberately does not try to rescue.
  *
  * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
  */
@@ -20,10 +45,108 @@ import { Readability } from '@mozilla/readability';
 // Readability already kept it. Long enough to be unique to that table.
 const SIGNATURE_LENGTH = 120;
 
+// Copied verbatim from @mozilla/readability's REGEXPS. Two of them delete
+// content, and both are unanchored: `unlikelyCandidates` removes an element
+// outright, `negative` docks 25 from its class weight, which is enough for
+// `_cleanConditionally` to drop it. The unit test asserts the real library
+// still behaves this way, so a Readability upgrade that fixes it upstream
+// fails loudly here rather than leaving a no-op pass behind.
+const UNLIKELY_SOURCE =
+  '-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote';
+const NEGATIVE_SOURCE =
+  '-ad-|hidden|^hid$| hid$| hid |^hidden|banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget';
+const REMOVAL_SOURCES = [UNLIKELY_SOURCE, NEGATIVE_SOURCE];
+// Readability's `positive` regex. A token matching it carries a signal worth
+// keeping, so it is never stripped even if it also matches mid-word.
+const POSITIVE_SIGNAL =
+  /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i;
+
+// Utility classes that collide with the `negative` regex on a whole word while
+// meaning something else entirely. Tailwind's `overflow-hidden` clips overflow;
+// it does not mark the element hidden. The mid-word rule below cannot catch
+// these — `hidden` really is a whole segment of `overflow-hidden` — and the
+// cost is real: -25 class weight is enough for `_cleanConditionally` to delete
+// the element, which is what removed every code block on next-intl.dev after
+// the `nextra-` collision had already been handled. Listed explicitly rather
+// than inferred, because `page-footer` and `main-sidebar` are the same shape
+// and *are* genuine.
+const UTILITY_FALSE_POSITIVES = new Set([
+  'overflow-hidden',
+  'overflow-x-hidden',
+  'overflow-y-hidden',
+  'overflow-scroll',
+  'overflow-x-scroll',
+  'overflow-y-scroll',
+  'scroll-smooth',
+  'scroll-auto'
+]);
+
 function normalizeWhitespace(text) {
   return text.replace(/\s+/g, ' ').trim();
 }
 
+/**
+ * True when `token` only ever trips Readability's removal regexes inside a
+ * longer word — `extra` within `nextra`, `hidden` within Tailwind's
+ * `overflow-hidden`.
+ *
+ * Only a *preceding* letter or digit counts as mid-word. A trailing one does
+ * not, because that is how English suffixes attach: rescuing `comments`,
+ * `banners` or `footers` would defeat the regexes on exactly the clutter they
+ * were written to catch, whereas a letter run in front of the match means a
+ * different word entirely.
+ *
+ * @param {string} token - one class name
+ * @returns {boolean}
+ */
+function isAccidentalToken(token) {
+  if (UTILITY_FALSE_POSITIVES.has(token.toLowerCase())) return true;
+  if (POSITIVE_SIGNAL.test(token)) return false;
+  let sawMatch = false;
+  for (const source of REMOVAL_SOURCES) {
+    const scan = new RegExp(source, 'gi');
+    let match;
+    while ((match = scan.exec(token)) !== null) {
+      if (match[0].length === 0) {
+        scan.lastIndex += 1;
+        continue;
+      }
+      sawMatch = true;
+      const before = token[match.index - 1];
+      if (before === undefined || !/[a-z0-9]/i.test(before)) return false;
+    }
+  }
+  return sawMatch;
+}
+
+/**
+ * Strip the class tokens that would make Readability discard an element over a
+ * word the class does not actually contain.
+ *
+ * Only whole tokens that are accidental on every match are removed, so a class
+ * carrying a real signal survives: an element still genuinely marked `sidebar`
+ * or `hidden` is still dropped, as it should be. Ids are left alone — they
+ * anchor in-page links, and the class attribute is what these frameworks put
+ * their names on.
+ *
+ * @param {Document} document - mutated in place
+ * @returns {number} how many elements were sanitized
+ */
+export function sanitizeAccidentalClasses(document) {
+  let sanitized = 0;
+  for (const el of Array.from(document.querySelectorAll('[class]'))) {
+    const className = el.getAttribute('class') || '';
+    const tokens = className.split(/\s+/).filter(Boolean);
+    const kept = tokens.filter((token) => !isAccidentalToken(token));
+    if (kept.length === tokens.length) continue;
+
+    if (kept.length > 0) el.setAttribute('class', kept.join(' '));
+    else el.removeAttribute('class');
+    sanitized += 1;
+  }
+  return sanitized;
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -61,6 +184,7 @@ export function extractMainContent(html, url) {
   let article;
   try {
     const dom = new JSDOM(html, { url });
+    sanitizeAccidentalClasses(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };