crawlforge-mcp-server 5.2.1 → 5.2.2

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.1
+**Current Version:** 5.2.2
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.1",
+  "version": "5.2.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.1",
+  version: "5.2.2",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/analysis/ContentAnalyzer.js

@@ -106,24 +106,25 @@ const LANGUAGE_NAMES = {
   'rus': 'Russian',
   'jpn': 'Japanese',
   'kor': 'Korean',
-  'chi': 'Chinese',
-  'ara': 'Arabic',
+  'cmn': 'Chinese',
+  'arb': 'Arabic',
   'hin': 'Hindi',
   'nld': 'Dutch',
   'swe': 'Swedish',
-  'nor': 'Norwegian',
+  'nob': 'Norwegian',
   'dan': 'Danish',
   'fin': 'Finnish',
   'pol': 'Polish',
   'ces': 'Czech',
   'hun': 'Hungarian',
   'tur': 'Turkish',
-  'gre': 'Greek',
+  'ell': 'Greek',
   'heb': 'Hebrew',
   'tha': 'Thai',
   'vie': 'Vietnamese',
   'ind': 'Indonesian',
-  'msa': 'Malay',
+  'zlm': 'Malay',
+  'zsm': 'Malay',
   'tgl': 'Tagalog',
   'ukr': 'Ukrainian',
   'bul': 'Bulgarian',
@@ -282,6 +283,28 @@ export class ContentAnalyzer {
    */
   async detectLanguage(text, options = {}) {
     try {
+      // franc scores the single most common script, so a Chinese, Japanese or
+      // Korean page carrying the usual run of English product names and code
+      // samples is detected as English. Those scripts never appear in
+      // Latin-script prose, so a meaningful share of them settles the question
+      // before trigram scoring gets a say.
+      const letters = (text.match(/\p{L}/gu) || []).length;
+      if (letters > 0) {
+        const han = (text.match(/\p{Script=Han}/gu) || []).length;
+        const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
+        const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
+        if ((han + kana + hangul) / letters >= 0.1) {
+          const code = kana > 0 ? 'jpn' : hangul > han ? 'kor' : 'cmn';
+          return {
+            code,
+            name: LANGUAGE_NAMES[code],
+            confidence: 0.9,
+            alternative: [],
+            detectionMethod: 'script'
+          };
+        }
+      }
+
       // Use franc for language detection
       const detected = franc(text, {
         minLength: 10,

package/src/utils/htmlToMarkdown.js

@@ -20,6 +20,38 @@ import { stripHiddenHtml } from './hiddenContent.js';
 
 let _td = null;
 
+// Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
+// whether it converts a table or keeps it as raw HTML.
+function isHeadingRow(tr) {
+  const parent = tr.parentNode;
+  if (!parent) return false;
+  if (parent.nodeName === 'THEAD') return true;
+  const firstTbody =
+    parent.nodeName === 'TBODY' &&
+    (!parent.previousSibling ||
+      (parent.previousSibling.nodeName === 'THEAD' &&
+        /^\s*$/.test(parent.previousSibling.textContent)));
+  return (
+    parent.firstChild === tr &&
+    (parent.nodeName === 'TABLE' || firstTbody) &&
+    Array.prototype.every.call(tr.childNodes, n => n.nodeName === 'TH')
+  );
+}
+
+function isLayoutTable(node) {
+  return (
+    node.nodeName === 'TABLE' &&
+    !(node.rows && node.rows[0] && isHeadingRow(node.rows[0]))
+  );
+}
+
+function isInLayoutTable(node) {
+  for (let p = node.parentNode; p; p = p.parentNode) {
+    if (p.nodeName === 'TABLE') return isLayoutTable(p);
+  }
+  return false;
+}
+
 function getTurndown() {
   if (_td === null) {
     _td = new TurndownService({
@@ -35,6 +67,26 @@ function getTurndown() {
     // Enable GFM extensions (tables, strikethrough, task lists)
     _td.use(gfm);
 
+    // turndown-plugin-gfm only converts a table whose first row is all <th>;
+    // every other table is passed through its `keep` filter as raw HTML. Pages
+    // we scrape are full of layout tables (Hacker News, older sites), so that
+    // leaks <table> markup into a field the caller asked for as markdown.
+    // Rules added here are matched before keep filters, so these reclaim the
+    // tables the plugin skipped and flatten them to their cell content, while
+    // real data tables still reach the plugin and render as pipe tables.
+    _td.addRule('layoutTable', {
+      filter: isLayoutTable,
+      replacement: content => '\n\n' + content.replace(/\n{3,}/g, '\n\n').trim() + '\n\n'
+    });
+    _td.addRule('layoutTableCell', {
+      filter: node => (node.nodeName === 'TH' || node.nodeName === 'TD') && isInLayoutTable(node),
+      replacement: content => (content.trim() ? content.trim() + '  ' : '')
+    });
+    _td.addRule('layoutTableRow', {
+      filter: node => node.nodeName === 'TR' && isInLayoutTable(node),
+      replacement: content => (content.trim() ? content.trim() + '\n' : '')
+    });
+
     // Remove boilerplate elements before converting
     _td.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
   }