crawlforge-mcp-server 5.2.0 → 5.2.2

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.0
+**Current Version:** 5.2.2
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.0",
+  "version": "5.2.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,7 +113,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.0.0",
+    "crawlforge-extractors": "^1.1.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.0",
+  version: "5.2.2",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1402,7 +1402,7 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package. Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
+  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: shopify-product (any Shopify storefront, read from the store's own /products/<handle>.json rather than the rendered page), amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package. Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     template: z.string().describe("Template ID (e.g. github-repo) or list to enumerate available templates"),

package/src/core/ChangeTracker.js

@@ -11,6 +11,7 @@ import { z } from 'zod';
 import { EventEmitter } from 'events';
 import { load } from 'cheerio';
 import { diffWords, diffLines, diffChars } from 'diff';
+import { structureSignature, structuralSimilarity } from 'crawlforge-extractors';
 import { calculateSimilarity as calculateContentSimilarity } from '../tools/tracking/trackChanges/differ.js';
 
 const ChangeTrackingSchema = z.object({
@@ -720,7 +721,10 @@ export class ChangeTracker extends EventEmitter {
   extractStructure($, options) {
     const structure = {
       elements: [],
-      hierarchy: {},
+      // Element count per nesting depth. This used to be an empty object that
+      // nothing ever wrote to, which made the hierarchy half of the structural
+      // score a constant.
+      hierarchy: structureSignature($).depths,
       semanticStructure: {}
     };
     
@@ -1094,38 +1098,13 @@ export class ChangeTracker extends EventEmitter {
   
   calculateStructuralSimilarity(baseline, current) {
     if (!baseline || !current) return 0;
-    
-    const baselineElements = baseline.elements || [];
-    const currentElements = current.elements || [];
-    
-    if (baselineElements.length === 0 && currentElements.length === 0) return 1;
-    if (baselineElements.length === 0 || currentElements.length === 0) return 0;
-    
-    const tagSimilarity = this.calculateTagSimilarity(baselineElements, currentElements);
-    const hierarchySimilarity = this.calculateHierarchySimilarity(baseline.hierarchy, current.hierarchy);
-
-    // Clamp defensively — this is a 0-1 metric and must never leave that range.
-    return Math.max(0, Math.min(1, (tagSimilarity + hierarchySimilarity) / 2));
-  }
-
-  calculateTagSimilarity(baselineElements, currentElements) {
-    // Jaccard similarity: intersection and union must both operate on SETS.
-    // The old code intersected the raw (duplicate-laden) tag list against a
-    // set union, so repeated tags (div, p, ...) inflated the numerator and
-    // produced impossible scores > 1 (e.g. the observed 1.05).
-    const baselineTags = new Set(baselineElements.map(el => el.tag));
-    const currentTags = new Set(currentElements.map(el => el.tag));
 
-    const intersection = [...baselineTags].filter(tag => currentTags.has(tag));
-    const union = new Set([...baselineTags, ...currentTags]);
-
-    return union.size === 0 ? 1 : intersection.length / union.size;
-  }
-  
-  calculateHierarchySimilarity(baseline, current) {
-    // Simple structural comparison - can be enhanced
-    if (!baseline || !current) return 0;
-    return Object.keys(baseline).length === Object.keys(current).length ? 1 : 0.5;
+    // Scored in crawlforge-extractors so the REST API's track_changes reports
+    // the same number for the same pair of pages.
+    return structuralSimilarity(
+      { tags: (baseline.elements || []).map(el => el.tag), depths: baseline.hierarchy },
+      { tags: (current.elements || []).map(el => el.tag), depths: current.hierarchy }
+    );
   }
   
   hammingDistance(str1, str2) {

package/src/core/analysis/ContentAnalyzer.js

@@ -106,24 +106,25 @@ const LANGUAGE_NAMES = {
   'rus': 'Russian',
   'jpn': 'Japanese',
   'kor': 'Korean',
-  'chi': 'Chinese',
-  'ara': 'Arabic',
+  'cmn': 'Chinese',
+  'arb': 'Arabic',
   'hin': 'Hindi',
   'nld': 'Dutch',
   'swe': 'Swedish',
-  'nor': 'Norwegian',
+  'nob': 'Norwegian',
   'dan': 'Danish',
   'fin': 'Finnish',
   'pol': 'Polish',
   'ces': 'Czech',
   'hun': 'Hungarian',
   'tur': 'Turkish',
-  'gre': 'Greek',
+  'ell': 'Greek',
   'heb': 'Hebrew',
   'tha': 'Thai',
   'vie': 'Vietnamese',
   'ind': 'Indonesian',
-  'msa': 'Malay',
+  'zlm': 'Malay',
+  'zsm': 'Malay',
   'tgl': 'Tagalog',
   'ukr': 'Ukrainian',
   'bul': 'Bulgarian',
@@ -282,6 +283,28 @@ export class ContentAnalyzer {
    */
   async detectLanguage(text, options = {}) {
     try {
+      // franc scores the single most common script, so a Chinese, Japanese or
+      // Korean page carrying the usual run of English product names and code
+      // samples is detected as English. Those scripts never appear in
+      // Latin-script prose, so a meaningful share of them settles the question
+      // before trigram scoring gets a say.
+      const letters = (text.match(/\p{L}/gu) || []).length;
+      if (letters > 0) {
+        const han = (text.match(/\p{Script=Han}/gu) || []).length;
+        const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
+        const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
+        if ((han + kana + hangul) / letters >= 0.1) {
+          const code = kana > 0 ? 'jpn' : hangul > han ? 'kor' : 'cmn';
+          return {
+            code,
+            name: LANGUAGE_NAMES[code],
+            confidence: 0.9,
+            alternative: [],
+            detectionMethod: 'script'
+          };
+        }
+      }
+
       // Use franc for language detection
       const detected = franc(text, {
         minLength: 10,

package/src/tools/basic/_fetch.js

@@ -3,6 +3,7 @@
  * Applies an AbortController timeout and a default User-Agent.
  */
 
+import { readBody } from 'crawlforge-extractors';
 import { config } from '../../constants/config.js';
 import { createRequire } from 'module';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
@@ -13,35 +14,6 @@ const _require = createRequire(import.meta.url);
 const _pkg = _require('../../../package.json');
 const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
 
-/**
- * Determine the charset to decode a response body with: Content-Type header
- * first, then a <meta charset> sniff of the first bytes, defaulting to utf-8.
- * @param {Response} response
- * @param {Uint8Array} bytes
- * @returns {string}
- */
-function detectCharset(response, bytes) {
-  const contentType = response.headers?.get?.('content-type') || '';
-  const headerMatch = /charset=["']?([\w-]+)/i.exec(contentType);
-  if (headerMatch) {
-    return headerMatch[1].trim().toLowerCase();
-  }
-
-  // <meta charset> tags must appear within the first 1024 bytes per the
-  // HTML5 spec's prescan algorithm; ASCII-range bytes decode identically
-  // under latin1 regardless of the document's real encoding.
-  const sniffLength = Math.min(bytes.byteLength, 1024);
-  const sniffText = new TextDecoder('latin1').decode(bytes.subarray(0, sniffLength));
-  const metaMatch =
-    /<meta[^>]+charset=["']?([\w-]+)/i.exec(sniffText) ||
-    /<meta[^>]+http-equiv=["']?content-type["']?[^>]*content=["'][^"']*charset=([\w-]+)/i.exec(sniffText);
-  if (metaMatch) {
-    return metaMatch[1].trim().toLowerCase();
-  }
-
-  return 'utf-8';
-}
-
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
  *
@@ -100,46 +72,11 @@ export async function fetchWithTimeout(url, options = {}) {
       throw error;
     }
 
-    // --- Body-size cap ---
-
-    // Early rejection via Content-Length (servers may omit or lie — guard below
-    // handles that case). Optional-chained so non-standard responses (e.g. test
-    // mocks) without a Headers object don't throw.
-    const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
-    if (contentLengthHeader !== null) {
-      const declared = parseInt(contentLengthHeader, 10);
-      if (!isNaN(declared) && declared > maxBodySize) {
-        throw new Error(
-          `Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`
-        );
-      }
-    }
-
-    // Only the streaming byte-count guard requires a readable body. Responses
-    // without a ReadableStream body (already-buffered responses, test mocks)
-    // are returned unchanged so callers' native .text()/.json() still work.
-    if (!response.body || typeof response.body.getReader !== 'function') {
-      return Object.assign(response, { _responseTime: Date.now() - startedAt });
-    }
-
-    // Stream the body and abort if accumulated bytes exceed the cap.
-    const reader = response.body.getReader();
-    const chunks = [];
-    let totalBytes = 0;
-
+    // Reading is delegated to crawlforge-extractors so the REST API applies
+    // the same cap and the same charset handling to the same page.
+    let bodyText;
     try {
-      while (true) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        totalBytes += value.byteLength;
-        if (totalBytes > maxBodySize) {
-          reader.cancel();
-          throw new Error(
-            `Response body too large: exceeded limit of ${maxBodySize} bytes`
-          );
-        }
-        chunks.push(value);
-      }
+      bodyText = await readBody(response, { maxBytes: maxBodySize });
     } catch (error) {
       if (error.name === 'AbortError') {
         throw new Error(`Request timeout after ${timeout}ms`);
@@ -147,29 +84,7 @@ export async function fetchWithTimeout(url, options = {}) {
       throw error;
     }
 
-    // Reassemble the raw bytes in a single pass (totalBytes is already known,
-    // so this is one allocation + one copy per chunk, not the O(n^2) cost of
-    // reallocating/copying the whole buffer on every chunk), then decode using
-    // the response's actual charset (Content-Type header, falling back to a
-    // <meta charset> sniff) instead of always assuming UTF-8.
-    const mergedBytes = new Uint8Array(totalBytes);
-    let offset = 0;
-    for (const chunk of chunks) {
-      mergedBytes.set(chunk, offset);
-      offset += chunk.byteLength;
-    }
-
-    const charset = detectCharset(response, mergedBytes);
-    let bodyText;
-    try {
-      bodyText = new TextDecoder(charset).decode(mergedBytes);
-    } catch {
-      // Unrecognized charset label — fall back to UTF-8 rather than throwing.
-      bodyText = new TextDecoder().decode(mergedBytes);
-    }
-
     // Attach the pre-read text so callers can call .text() on the result.
-    // We wrap it in a minimal compatible object.
     return Object.assign(response, {
       text: () => Promise.resolve(bodyText),
       json: () => Promise.resolve(JSON.parse(bodyText)),

package/src/tools/search/adapters/redditOfficialApi.js

@@ -27,7 +27,7 @@ const TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
 const API_BASE = 'https://oauth.reddit.com';
 
 /** Reddit requires a descriptive, unique User-Agent; generic ones are throttled. */
-const DEFAULT_USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
+const DEFAULT_USER_AGENT = 'CrawlForge-MCP/5.2.1 (+https://www.crawlforge.dev)';
 
 /** Our sort is asc/desc by post date; Reddit listings/search only go newest-first. */
 const REDDIT_SORT = 'new';

package/src/tools/search/redditSearch.js

@@ -47,7 +47,7 @@ const PULLPUSH_BASE = 'https://api.pullpush.io';
  * into a shared bucket (422 "Timeout. Maybe slow down a bit" while curl got
  * 200 for the same URL); with a descriptive UA it answers instantly.
  */
-const USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
+const USER_AGENT = 'CrawlForge-MCP/5.2.1 (+https://www.crawlforge.dev)';
 
 const RedditSearchSchema = z.object({
   query: z.string().min(1).optional(),

package/src/utils/htmlToMarkdown.js

@@ -20,6 +20,38 @@ import { stripHiddenHtml } from './hiddenContent.js';
 
 let _td = null;
 
+// Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
+// whether it converts a table or keeps it as raw HTML.
+function isHeadingRow(tr) {
+  const parent = tr.parentNode;
+  if (!parent) return false;
+  if (parent.nodeName === 'THEAD') return true;
+  const firstTbody =
+    parent.nodeName === 'TBODY' &&
+    (!parent.previousSibling ||
+      (parent.previousSibling.nodeName === 'THEAD' &&
+        /^\s*$/.test(parent.previousSibling.textContent)));
+  return (
+    parent.firstChild === tr &&
+    (parent.nodeName === 'TABLE' || firstTbody) &&
+    Array.prototype.every.call(tr.childNodes, n => n.nodeName === 'TH')
+  );
+}
+
+function isLayoutTable(node) {
+  return (
+    node.nodeName === 'TABLE' &&
+    !(node.rows && node.rows[0] && isHeadingRow(node.rows[0]))
+  );
+}
+
+function isInLayoutTable(node) {
+  for (let p = node.parentNode; p; p = p.parentNode) {
+    if (p.nodeName === 'TABLE') return isLayoutTable(p);
+  }
+  return false;
+}
+
 function getTurndown() {
   if (_td === null) {
     _td = new TurndownService({
@@ -35,6 +67,26 @@ function getTurndown() {
     // Enable GFM extensions (tables, strikethrough, task lists)
     _td.use(gfm);
 
+    // turndown-plugin-gfm only converts a table whose first row is all <th>;
+    // every other table is passed through its `keep` filter as raw HTML. Pages
+    // we scrape are full of layout tables (Hacker News, older sites), so that
+    // leaks <table> markup into a field the caller asked for as markdown.
+    // Rules added here are matched before keep filters, so these reclaim the
+    // tables the plugin skipped and flatten them to their cell content, while
+    // real data tables still reach the plugin and render as pipe tables.
+    _td.addRule('layoutTable', {
+      filter: isLayoutTable,
+      replacement: content => '\n\n' + content.replace(/\n{3,}/g, '\n\n').trim() + '\n\n'
+    });
+    _td.addRule('layoutTableCell', {
+      filter: node => (node.nodeName === 'TH' || node.nodeName === 'TD') && isInLayoutTable(node),
+      replacement: content => (content.trim() ? content.trim() + '  ' : '')
+    });
+    _td.addRule('layoutTableRow', {
+      filter: node => node.nodeName === 'TR' && isInLayoutTable(node),
+      replacement: content => (content.trim() ? content.trim() + '\n' : '')
+    });
+
     // Remove boilerplate elements before converting
     _td.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
   }