crawlforge-mcp-server 5.6.4 → 5.6.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.4",
+  "version": "5.6.6",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.4",
+  version: "5.6.6",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Recording / replay helpers
 import {
@@ -798,12 +799,15 @@ export class ScrapeWithActionsTool extends EventEmitter {
       try {
         const elements = $(selector);
         
+        // elementText keeps table structure (one line per row, cells joined
+        // by " | "); cheerio's .text() ran CoinMarketCap's history table into
+        // "DateOpen*HighLowClose**…" (2026-09-04).
         if (elements.length === 0) {
           extracted[key] = null;
         } else if (elements.length === 1) {
-          extracted[key] = elements.text().trim();
+          extracted[key] = elementText($, elements.get(0));
         } else {
-          extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+          extracted[key] = elements.map((_, el) => elementText($, el)).get();
         }
       } catch (error) {
         extracted[key] = { error: `Invalid selector: ${selector}` };

package/src/tools/advanced/batchScrape/worker.js

@@ -11,6 +11,7 @@ import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
+import { elementText } from '../../../utils/elementText.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -111,8 +112,8 @@ function extractStructuredData($, selectors) {
     try {
       const elements = $(selector);
       if (elements.length === 0) extracted[key] = null;
-      else if (elements.length === 1) extracted[key] = elements.text().trim();
-      else extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+      else if (elements.length === 1) extracted[key] = elementText($, elements.get(0));
+      else extracted[key] = elements.map((_, el) => elementText($, el)).get();
     } catch {
       extracted[key] = { error: `Invalid selector: ${selector}` };
     }

package/src/tools/basic/scrapeStructured.js

@@ -9,6 +9,7 @@
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Matches a trailing "@attr" suffix (e.g. "@href", "@data-id") — the attribute
 // name must look like a real attribute, not the "@" of a CSS attribute-value
@@ -73,7 +74,7 @@ function extractRows($, { row_selector, selectors, max_results }) {
           record[field] = null;
           continue;
         }
-        record[field] = attribute ? (el.attr(attribute) ?? null) : el.text().trim();
+        record[field] = attribute ? (el.attr(attribute) ?? null) : elementText($, el.get(0));
         matchCounts[field] += 1;
       } catch (selectorError) {
         record[field] = {
@@ -145,7 +146,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
             if (attribute) {
               return $(el).attr(attribute) ?? null;
             }
-            return $(el).text().trim();
+            return elementText($, el);
           };
 
           if (elements.length === 1) {

package/src/utils/elementText.js

@@ -0,0 +1,70 @@
+/**
+ * elementText — the text of a cheerio-matched element with table structure kept.
+ *
+ * cheerio's .text() runs every descendant's text together. For a table that
+ * is one string per row with no cell boundaries: scrape_with_actions'
+ * extractionOptions.selectors returned CoinMarketCap's historical-data table
+ * as "DateOpen*HighLowClose**VolumeMarket CapSep 03, 2026$77,300.17…"
+ * (2026-09-04). Here a table, a row group or a row renders one line per row
+ * with cells joined by " | " (the convention _mainContent.js uses for the text
+ * of a recovered table), an element that wraps a table renders each table that
+ * way in place, and anything else is .text().trim(), unchanged.
+ *
+ * batch_scrape's selector extraction (batchScrape/worker.js) and
+ * scrape_structured read matched elements through this helper too (5.6.6).
+ */
+
+const ROW_GROUPS = new Set(['table', 'thead', 'tbody', 'tfoot']);
+
+function tagOf(el) {
+  return String(el?.name || el?.tagName || '').toLowerCase();
+}
+
+function cellText($, cell) {
+  return $(cell).text().replace(/\s+/g, ' ').trim();
+}
+
+function rowText($, row) {
+  return $(row).children('td, th').map((_, cell) => cellText($, cell)).get().join(' | ');
+}
+
+/**
+ * One line per row, cells joined by " | ". `el` is a table or one of its row
+ * groups; rows of a table nested inside a cell belong to that table, not to
+ * this one, so they are skipped here (their cell already carries their text).
+ */
+function tableText($, el) {
+  const table = tagOf(el) === 'table' ? el : $(el).closest('table').get(0);
+  return $(el).find('tr')
+    .filter((_, row) => $(row).closest('table').get(0) === table)
+    .map((_, row) => rowText($, row)).get()
+    .filter((line) => line.length > 0)
+    .join('\n');
+}
+
+/**
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {object} el - a cheerio/domhandler element
+ * @returns {string}
+ */
+export function elementText($, el) {
+  const tag = tagOf(el);
+  if (tag === 'tr') return rowText($, el);
+  if (ROW_GROUPS.has(tag)) return tableText($, el);
+
+  const $el = $(el);
+  if ($el.find('table').length === 0) return $el.text().trim();
+
+  // A wrapper around one or more tables: swap each outermost table for its
+  // line-per-row text (as a text node, so cell content is never re-parsed as
+  // markup), then read the wrapper as lines.
+  const $clone = $el.clone();
+  $clone.find('table')
+    .filter((_, table) => $(table).parents('table').length === 0)
+    .each((_, table) => {
+      $(table).replaceWith($('<div></div>').text(`\n${tableText($, table)}\n`));
+    });
+  return $clone.text().split('\n').map((line) => line.trim()).filter(Boolean).join('\n');
+}
+
+export default elementText;