crawlforge-mcp-server 5.6.5 → 5.6.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.5",
+  "version": "5.6.6",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.5",
+  version: "5.6.6",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/tools/advanced/batchScrape/worker.js

@@ -11,6 +11,7 @@ import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
+import { elementText } from '../../../utils/elementText.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -111,8 +112,8 @@ function extractStructuredData($, selectors) {
     try {
       const elements = $(selector);
       if (elements.length === 0) extracted[key] = null;
-      else if (elements.length === 1) extracted[key] = elements.text().trim();
-      else extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+      else if (elements.length === 1) extracted[key] = elementText($, elements.get(0));
+      else extracted[key] = elements.map((_, el) => elementText($, el)).get();
     } catch {
       extracted[key] = { error: `Invalid selector: ${selector}` };
     }

package/src/tools/basic/scrapeStructured.js

@@ -9,6 +9,7 @@
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Matches a trailing "@attr" suffix (e.g. "@href", "@data-id") — the attribute
 // name must look like a real attribute, not the "@" of a CSS attribute-value
@@ -73,7 +74,7 @@ function extractRows($, { row_selector, selectors, max_results }) {
           record[field] = null;
           continue;
         }
-        record[field] = attribute ? (el.attr(attribute) ?? null) : el.text().trim();
+        record[field] = attribute ? (el.attr(attribute) ?? null) : elementText($, el.get(0));
         matchCounts[field] += 1;
       } catch (selectorError) {
         record[field] = {
@@ -145,7 +146,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
             if (attribute) {
               return $(el).attr(attribute) ?? null;
             }
-            return $(el).text().trim();
+            return elementText($, el);
           };
 
           if (elements.length === 1) {

package/src/utils/elementText.js

@@ -10,9 +10,8 @@
  * of a recovered table), an element that wraps a table renders each table that
  * way in place, and anything else is .text().trim(), unchanged.
  *
- * batchScrape/worker.js extractStructuredData and scrapeStructured read
- * matched elements with the same .text().trim() and have the same gap; they
- * do not use this yet.
+ * batch_scrape's selector extraction (batchScrape/worker.js) and
+ * scrape_structured read matched elements through this helper too (5.6.6).
  */
 
 const ROW_GROUPS = new Set(['table', 'thead', 'tbody', 'tfoot']);