crawlforge-mcp-server 5.6.3 → 5.6.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.3",
+  "version": "5.6.5",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.3",
+  version: "5.6.5",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/processing/ContentProcessor.js

@@ -8,6 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
+import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -30,7 +31,8 @@ const ReadabilityResult = z.object({
   byline: z.string().nullable(),
   dir: z.string().nullable(),
   siteName: z.string().nullable(),
-  lang: z.string().nullable()
+  lang: z.string().nullable(),
+  tablesRecovered: z.number()
 });
 
 export class ContentProcessor {
@@ -78,16 +80,28 @@ export class ContentProcessor {
 
         const article = reader.parse();
         if (article) {
+          // Readability keeps one article candidate and drops every data
+          // table outside it — scrape_with_actions returned CoinMarketCap's
+          // historical-data page as its FAQ copy with no table (2026-09-04).
+          // Re-attach what it dropped, as scrape's main-content path does.
+          const recovered = recoverDroppedTables(html, url, article.textContent);
+          const content = recovered.length > 0
+            ? `${article.content}\n${recovered.map((table) => table.html).join('\n')}`
+            : article.content;
+          const textContent = recovered.length > 0
+            ? `${article.textContent}\n${recovered.map((table) => table.text).join('\n')}`
+            : article.textContent;
           result.readability = {
             title: article.title,
-            content: article.content,
-            textContent: article.textContent,
-            length: article.length,
+            content,
+            textContent,
+            length: textContent.length,
             excerpt: article.excerpt,
             byline: article.byline,
             dir: article.dir,
             siteName: article.siteName,
-            lang: article.lang
+            lang: article.lang,
+            tablesRecovered: recovered.length
           };
 
           // Calculate readability score

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Recording / replay helpers
 import {
@@ -798,12 +799,15 @@ export class ScrapeWithActionsTool extends EventEmitter {
       try {
         const elements = $(selector);
         
+        // elementText keeps table structure (one line per row, cells joined
+        // by " | "); cheerio's .text() ran CoinMarketCap's history table into
+        // "DateOpen*HighLowClose**…" (2026-09-04).
         if (elements.length === 0) {
           extracted[key] = null;
         } else if (elements.length === 1) {
-          extracted[key] = elements.text().trim();
+          extracted[key] = elementText($, elements.get(0));
         } else {
-          extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+          extracted[key] = elements.map((_, el) => elementText($, el)).get();
         }
       } catch (error) {
         extracted[key] = { error: `Invalid selector: ${selector}` };

package/src/tools/extract/extractContent.js

@@ -235,6 +235,11 @@ export class ExtractContentTool {
         result.extractionMethod = 'readability';
         result.confidence = 0.9;
         result.finalUrl = url;
+        if (processingResult.readability.tablesRecovered > 0) {
+          (result.warnings ||= []).push(
+            `mainContent: re-attached ${processingResult.readability.tablesRecovered} data table(s) that main-content extraction had dropped`
+          );
+        }
 
         // Convert to markdown if requested
         if (options.outputFormat === 'markdown') {

package/src/tools/scrape/_mainContent.js

@@ -35,7 +35,10 @@
  *    12th sits inside a collapsed `<details>`, which Readability drops whole
  *    and which this deliberately does not try to rescue.
  *
- * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
+ * Shared by unifiedScrape (wants HTML) and extractStructured (wants text);
+ * the table recovery alone is also used by ContentProcessor, which runs its
+ * own Readability pass for extract_content, scrape_with_actions and
+ * process_document.
  */
 
 import { JSDOM } from 'jsdom';
@@ -169,6 +172,61 @@ function isDataTable(table) {
   return rows >= 10 || columns > 4;
 }
 
+/**
+ * A data table's text with its structure kept: one line per row, cells joined
+ * by " | ". A bare `textContent` runs every cell together
+ * ("DateOpen*HighLowClose**…"), which is what scrape_with_actions' selector
+ * extraction handed back for CoinMarketCap's history table.
+ * @param {HTMLTableElement} table
+ * @returns {string}
+ */
+function tableText(table) {
+  return Array.from(table.rows)
+    .map((row) => Array.from(row.cells).map((cell) => normalizeWhitespace(cell.textContent || '')).join(' | '))
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * The data tables in `html` that a Readability pass left out of its article.
+ *
+ * Readability mutates the document it is handed — the dropped tables are
+ * already gone by the time parse() returns — so this parses the original HTML
+ * again. Shared by every caller that runs Readability itself: extractMainContent
+ * below, and ContentProcessor, whose consumers (extract_content,
+ * scrape_with_actions, process_document) lost tables the same way until
+ * 2026-09-04, when scrape_with_actions returned CoinMarketCap's historical-data
+ * page as its FAQ copy — the wait for the table rows had succeeded, and the
+ * markdown then had no table at all.
+ *
+ * @param {string} html - the page's HTML
+ * @param {string} [url] - document URL, used as the base for relative links
+ * @param {string} [keptText] - the article's textContent, so a table Readability
+ *   did keep is not attached a second time
+ * @returns {{ html: string, text: string }[]} each dropped table's outerHTML and
+ *   its row-per-line text; empty when there is nothing to recover
+ */
+export function recoverDroppedTables(html, url, keptText = '') {
+  // Nothing to recover, and no reason to pay for a second parse.
+  if (!/<table[\s>]/i.test(html)) return [];
+  try {
+    const fresh = new JSDOM(html, { url });
+    const kept = normalizeWhitespace(keptText || '');
+    return Array.from(fresh.window.document.querySelectorAll('table'))
+      // A nested table travels with its parent; re-attaching it separately
+      // would duplicate it.
+      .filter((table) => !table.parentElement?.closest('table'))
+      .filter(isDataTable)
+      .filter((table) => {
+        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
+        return signature.length > 0 && !kept.includes(signature);
+      })
+      .map((table) => ({ html: table.outerHTML, text: tableText(table) }));
+  } catch {
+    return [];
+  }
+}
+
 /**
  * Extract a page's main content, re-attaching data tables Readability dropped.
  *
@@ -193,32 +251,7 @@ export function extractMainContent(html, url) {
   if (!article || !article.content) return { html: null, title: '', tablesRecovered: 0 };
 
   const title = article.title || '';
-
-  // Nothing to recover, and no reason to pay for a second parse.
-  if (!/<table[\s>]/i.test(html)) {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
-
-  // Readability mutates the document it is handed — the dropped tables are
-  // already gone from `dom` by the time parse() returns — so recovery has to
-  // parse the original HTML again.
-  let recovered;
-  try {
-    const fresh = new JSDOM(html, { url });
-    const kept = normalizeWhitespace(article.textContent || '');
-    recovered = Array.from(fresh.window.document.querySelectorAll('table'))
-      // A nested table travels with its parent; re-attaching it separately
-      // would duplicate it.
-      .filter((table) => !table.parentElement?.closest('table'))
-      .filter(isDataTable)
-      .filter((table) => {
-        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
-        return signature.length > 0 && !kept.includes(signature);
-      })
-      .map((table) => table.outerHTML);
-  } catch {
-    return { html: article.content, title, tablesRecovered: 0 };
-  }
+  const recovered = recoverDroppedTables(html, url, article.textContent).map((table) => table.html);
 
   return {
     html: recovered.length > 0 ? `${article.content}\n${recovered.join('\n')}` : article.content,

package/src/utils/elementText.js

@@ -0,0 +1,71 @@
+/**
+ * elementText — the text of a cheerio-matched element with table structure kept.
+ *
+ * cheerio's .text() runs every descendant's text together. For a table that
+ * is one string per row with no cell boundaries: scrape_with_actions'
+ * extractionOptions.selectors returned CoinMarketCap's historical-data table
+ * as "DateOpen*HighLowClose**VolumeMarket CapSep 03, 2026$77,300.17…"
+ * (2026-09-04). Here a table, a row group or a row renders one line per row
+ * with cells joined by " | " (the convention _mainContent.js uses for the text
+ * of a recovered table), an element that wraps a table renders each table that
+ * way in place, and anything else is .text().trim(), unchanged.
+ *
+ * batchScrape/worker.js extractStructuredData and scrapeStructured read
+ * matched elements with the same .text().trim() and have the same gap; they
+ * do not use this yet.
+ */
+
+const ROW_GROUPS = new Set(['table', 'thead', 'tbody', 'tfoot']);
+
+function tagOf(el) {
+  return String(el?.name || el?.tagName || '').toLowerCase();
+}
+
+function cellText($, cell) {
+  return $(cell).text().replace(/\s+/g, ' ').trim();
+}
+
+function rowText($, row) {
+  return $(row).children('td, th').map((_, cell) => cellText($, cell)).get().join(' | ');
+}
+
+/**
+ * One line per row, cells joined by " | ". `el` is a table or one of its row
+ * groups; rows of a table nested inside a cell belong to that table, not to
+ * this one, so they are skipped here (their cell already carries their text).
+ */
+function tableText($, el) {
+  const table = tagOf(el) === 'table' ? el : $(el).closest('table').get(0);
+  return $(el).find('tr')
+    .filter((_, row) => $(row).closest('table').get(0) === table)
+    .map((_, row) => rowText($, row)).get()
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {object} el - a cheerio/domhandler element
+ * @returns {string}
+ */
+export function elementText($, el) {
+  const tag = tagOf(el);
+  if (tag === 'tr') return rowText($, el);
+  if (ROW_GROUPS.has(tag)) return tableText($, el);
+
+  const $el = $(el);
+  if ($el.find('table').length === 0) return $el.text().trim();
+
+  // A wrapper around one or more tables: swap each outermost table for its
+  // line-per-row text (as a text node, so cell content is never re-parsed as
+  // markup), then read the wrapper as lines.
+  const $clone = $el.clone();
+  $clone.find('table')
+    .filter((_, table) => $(table).parents('table').length === 0)
+    .each((_, table) => {
+      $(table).replaceWith($('<div></div>').text(`\n${tableText($, table)}\n`));
+    });
+  return $clone.text().split('\n').map((line) => line.trim()).filter(Boolean).join('\n');
+}
+
+export default elementText;