crawlforge-mcp-server 5.6.4 → 5.6.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.4",
+  "version": "5.6.5",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.4",
+  version: "5.6.5",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Recording / replay helpers
 import {
@@ -798,12 +799,15 @@ export class ScrapeWithActionsTool extends EventEmitter {
       try {
         const elements = $(selector);
         
+        // elementText keeps table structure (one line per row, cells joined
+        // by " | "); cheerio's .text() ran CoinMarketCap's history table into
+        // "DateOpen*HighLowClose**…" (2026-09-04).
         if (elements.length === 0) {
           extracted[key] = null;
         } else if (elements.length === 1) {
-          extracted[key] = elements.text().trim();
+          extracted[key] = elementText($, elements.get(0));
         } else {
-          extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+          extracted[key] = elements.map((_, el) => elementText($, el)).get();
         }
       } catch (error) {
         extracted[key] = { error: `Invalid selector: ${selector}` };

package/src/utils/elementText.js

@@ -0,0 +1,71 @@
+/**
+ * elementText — the text of a cheerio-matched element with table structure kept.
+ *
+ * cheerio's .text() runs every descendant's text together. For a table that
+ * is one string per row with no cell boundaries: scrape_with_actions'
+ * extractionOptions.selectors returned CoinMarketCap's historical-data table
+ * as "DateOpen*HighLowClose**VolumeMarket CapSep 03, 2026$77,300.17…"
+ * (2026-09-04). Here a table, a row group or a row renders one line per row
+ * with cells joined by " | " (the convention _mainContent.js uses for the text
+ * of a recovered table), an element that wraps a table renders each table that
+ * way in place, and anything else is .text().trim(), unchanged.
+ *
+ * batchScrape/worker.js extractStructuredData and scrapeStructured read
+ * matched elements with the same .text().trim() and have the same gap; they
+ * do not use this yet.
+ */
+
+const ROW_GROUPS = new Set(['table', 'thead', 'tbody', 'tfoot']);
+
+function tagOf(el) {
+  return String(el?.name || el?.tagName || '').toLowerCase();
+}
+
+function cellText($, cell) {
+  return $(cell).text().replace(/\s+/g, ' ').trim();
+}
+
+function rowText($, row) {
+  return $(row).children('td, th').map((_, cell) => cellText($, cell)).get().join(' | ');
+}
+
+/**
+ * One line per row, cells joined by " | ". `el` is a table or one of its row
+ * groups; rows of a table nested inside a cell belong to that table, not to
+ * this one, so they are skipped here (their cell already carries their text).
+ */
+function tableText($, el) {
+  const table = tagOf(el) === 'table' ? el : $(el).closest('table').get(0);
+  return $(el).find('tr')
+    .filter((_, row) => $(row).closest('table').get(0) === table)
+    .map((_, row) => rowText($, row)).get()
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {object} el - a cheerio/domhandler element
+ * @returns {string}
+ */
+export function elementText($, el) {
+  const tag = tagOf(el);
+  if (tag === 'tr') return rowText($, el);
+  if (ROW_GROUPS.has(tag)) return tableText($, el);
+
+  const $el = $(el);
+  if ($el.find('table').length === 0) return $el.text().trim();
+
+  // A wrapper around one or more tables: swap each outermost table for its
+  // line-per-row text (as a text node, so cell content is never re-parsed as
+  // markup), then read the wrapper as lines.
+  const $clone = $el.clone();
+  $clone.find('table')
+    .filter((_, table) => $(table).parents('table').length === 0)
+    .each((_, table) => {
+      $(table).replaceWith($('<div></div>').text(`\n${tableText($, table)}\n`));
+    });
+  return $clone.text().split('\n').map((line) => line.trim()).filter(Boolean).join('\n');
+}
+
+export default elementText;