crawlforge-mcp-server 5.6.3 → 5.6.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.3",
+  "version": "5.6.4",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.3",
+  version: "5.6.4",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/processing/ContentProcessor.js

@@ -8,6 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
+import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -30,7 +31,8 @@ const ReadabilityResult = z.object({
   byline: z.string().nullable(),
   dir: z.string().nullable(),
   siteName: z.string().nullable(),
-  lang: z.string().nullable()
+  lang: z.string().nullable(),
+  tablesRecovered: z.number()
 });
 
 export class ContentProcessor {
@@ -78,16 +80,28 @@ export class ContentProcessor {
 
         const article = reader.parse();
         if (article) {
+          // Readability keeps one article candidate and drops every data
+          // table outside it — scrape_with_actions returned CoinMarketCap's
+          // historical-data page as its FAQ copy with no table (2026-09-04).
+          // Re-attach what it dropped, as scrape's main-content path does.
+          const recovered = recoverDroppedTables(html, url, article.textContent);
+          const content = recovered.length > 0
+            ? `${article.content}\n${recovered.map((table) => table.html).join('\n')}`
+            : article.content;
+          const textContent = recovered.length > 0
+            ? `${article.textContent}\n${recovered.map((table) => table.text).join('\n')}`
+            : article.textContent;
           result.readability = {
             title: article.title,
-            content: article.content,
-            textContent: article.textContent,
-            length: article.length,
+            content,
+            textContent,
+            length: textContent.length,
             excerpt: article.excerpt,
             byline: article.byline,
             dir: article.dir,
             siteName: article.siteName,
-            lang: article.lang
+            lang: article.lang,
+            tablesRecovered: recovered.length
           };
 
           // Calculate readability score

package/src/tools/extract/extractContent.js

@@ -235,6 +235,11 @@ export class ExtractContentTool {
         result.extractionMethod = 'readability';
         result.confidence = 0.9;
         result.finalUrl = url;
+        if (processingResult.readability.tablesRecovered > 0) {
+          (result.warnings ||= []).push(
+            `mainContent: re-attached ${processingResult.readability.tablesRecovered} data table(s) that main-content extraction had dropped`
+          );
+        }
 
         // Convert to markdown if requested
         if (options.outputFormat === 'markdown') {

package/src/tools/scrape/_mainContent.js

@@ -35,7 +35,10 @@
  *    12th sits inside a collapsed `<details>`, which Readability drops whole
  *    and which this deliberately does not try to rescue.
  *
- * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
+ * Shared by unifiedScrape (wants HTML) and extractStructured (wants text);
+ * the table recovery alone is also used by ContentProcessor, which runs its
+ * own Readability pass for extract_content, scrape_with_actions and
+ * process_document.
  */
 
 import { JSDOM } from 'jsdom';
@@ -169,6 +172,61 @@ function isDataTable(table) {
   return rows >= 10 || columns > 4;
 }
 
+/**
+ * A data table's text with its structure kept: one line per row, cells joined
+ * by " | ". A bare `textContent` runs every cell together
+ * ("DateOpen*HighLowClose**…"), which is what scrape_with_actions' selector
+ * extraction handed back for CoinMarketCap's history table.
+ * @param {HTMLTableElement} table
+ * @returns {string}
+ */
+function tableText(table) {
+  return Array.from(table.rows)
+    .map((row) => Array.from(row.cells).map((cell) => normalizeWhitespace(cell.textContent || '')).join(' | '))
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * The data tables in `html` that a Readability pass left out of its article.
+ *
+ * Readability mutates the document it is handed — the dropped tables are
+ * already gone by the time parse() returns — so this parses the original HTML
+ * again. Shared by every caller that runs Readability itself: extractMainContent
+ * below, and ContentProcessor, whose consumers (extract_content,
+ * scrape_with_actions, process_document) lost tables the same way until
+ * 2026-09-04, when scrape_with_actions returned CoinMarketCap's historical-data
+ * page as its FAQ copy — the wait for the table rows had succeeded, and the
+ * markdown then had no table at all.
+ *
+ * @param {string} html - the page's HTML
+ * @param {string} [url] - document URL, used as the base for relative links
+ * @param {string} [keptText] - the article's textContent, so a table Readability
+ *   did keep is not attached a second time
+ * @returns {{ html: string, text: string }[]} each dropped table's outerHTML and
+ *   its row-per-line text; empty when there is nothing to recover
+ */
+export function recoverDroppedTables(html, url, keptText = '') {
+  // Nothing to recover, and no reason to pay for a second parse.
+  if (!/<table[\s>]/i.test(html)) return [];
+  try {
+    const fresh = new JSDOM(html, { url });
+    const kept = normalizeWhitespace(keptText || '');
+    return Array.from(fresh.window.document.querySelectorAll('table'))
+      // A nested table travels with its parent; re-attaching it separately
+      // would duplicate it.
+      .filter((table) => !table.parentElement?.closest('table'))
+      .filter(isDataTable)
+      .filter((table) => {
+        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
+        return signature.length > 0 && !kept.includes(signature);
+      })
+      .map((table) => ({ html: table.outerHTML, text: tableText(table) }));
+  } catch {
+    return [];
+  }
+}
+
 /**
  * Extract a page's main content, re-attaching data tables Readability dropped.
  *
@@ -193,32 +251,7 @@ export function extractMainContent(html, url) {
   if (!article || !article.content) return { html: null, title: '', tablesRecovered: 0 };
 
   const title = article.title || '';
-
-  // Nothing to recover, and no reason to pay for a second parse.
-  if (!/<table[\s>]/i.test(html)) {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
-
-  // Readability mutates the document it is handed — the dropped tables are
-  // already gone from `dom` by the time parse() returns — so recovery has to
-  // parse the original HTML again.
-  let recovered;
-  try {
-    const fresh = new JSDOM(html, { url });
-    const kept = normalizeWhitespace(article.textContent || '');
-    recovered = Array.from(fresh.window.document.querySelectorAll('table'))
-      // A nested table travels with its parent; re-attaching it separately
-      // would duplicate it.
-      .filter((table) => !table.parentElement?.closest('table'))
-      .filter(isDataTable)
-      .filter((table) => {
-        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
-        return signature.length > 0 && !kept.includes(signature);
-      })
-      .map((table) => table.outerHTML);
-  } catch {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
+  const recovered = recoverDroppedTables(html, url, article.textContent).map((table) => table.html);
 
   return {
     html: recovered.length > 0 ? `${article.content}\n${recovered.join('\n')}` : article.content,