crawlforge-mcp-server 5.5.9 → 5.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.5.9",
+  "version": "5.6.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.5.3",
+    "crawlforge-extractors": "^1.6.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.5.9",
+  version: "5.6.0",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/LLMsTxtAnalyzer.js

@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { MapSiteTool } from '../tools/crawl/mapSite.js';
 import { CrawlDeepTool } from '../tools/crawl/crawlDeep.js';
 import { normalizeUrl, getBaseUrl } from '../utils/urlNormalizer.js';
@@ -240,7 +241,7 @@ export class LLMsTxtAnalyzer {
       // Look for API documentation references
       const mainPageResponse = await this.fetchWithTimeout(baseUrl);
       if (mainPageResponse.ok) {
-        const html = await mainPageResponse.text();
+        const html = await readBody(mainPageResponse);
         const $ = load(html);
         
         // Find API documentation links
@@ -490,7 +491,7 @@ export class LLMsTxtAnalyzer {
       const robotsUrl = `${baseUrl}/robots.txt`;
       const response = await this.fetchWithTimeout(robotsUrl);
       if (response.ok) {
-        return await response.text();
+        return await readBody(response);
       }
     } catch {
       // No robots.txt found
@@ -677,7 +678,9 @@ export class LLMsTxtAnalyzer {
       }
 
       const contentType = response.headers.get('content-type') || '';
-      const html = await response.text();
+      // Charset-aware read; .text() decodes UTF-8 only and mangled
+      // ISO-8859-1 page titles in the generated llms.txt.
+      const html = await readBody(response);
       const $ = load(html);
 
       // Check for forms

package/src/core/crawlers/BFSCrawler.js

@@ -1,4 +1,5 @@
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { QueueManager } from '../queue/QueueManager.js';
 import { CacheManager } from '../cache/CacheManager.js';
 import { RateLimiter } from '../../utils/rateLimiter.js';
@@ -366,7 +367,9 @@ export class BFSCrawler {
         throw new Error(`Non-HTML content type: ${contentType}`);
       }
 
-      const html = await response.text();
+      // readBody decodes with the body's real charset — response.text() is
+      // UTF-8-only and mangled ISO-8859-1 pages ("portugu�s" on lua.org).
+      const html = await readBody(response);
       return this.parsePage(html, url);
     } catch (error) {
       clearTimeout(timeoutId);

package/src/skills/agent-skills/crawlforge-structured-extraction/references/templates.md

@@ -12,7 +12,7 @@ to enumerate at runtime. Cost: 1 credit per call.
 | `youtube-video` | YouTube watch page | title, channel, views, likes, published date, description |
 | `reddit-thread` | Reddit post (read from the Arctic Shift archive, not reddit.com, which blocks plain fetchers) | id, title, subreddit, author, score, upvote_ratio, num_comments, posted, body, link_url, flair, removed. Pass `id` to `reddit_search` with `mode:"thread"` for the comment tree |
 | `hacker-news-front-page` | HN front page | ranked stories: title, URL, points, author, comment count |
-| `producthunt-launch` | Product Hunt launch | name, tagline, upvotes, maker, description |
+| `producthunt-launch` | Product Hunt product page (`/products/...`; old `/posts/...` links redirect there) | name, tagline, description, topics, website, followers, reviews_count, reviews_rating. The /products layout has no product-level vote count — followers and reviews are its engagement numbers |
 | `stackoverflow-question` | Stack Overflow question (read from the keyless Stack Exchange API, not the page) | title, body, votes, views, tags, author, asked, answered, accepted_answer_id, answer_count, top 5 answers (accepted first: votes, author, posted, body) |
 | `npm-package` | npm package page | name, version, description, weekly downloads, license, repo link |
 

package/src/tools/advanced/batchScrape/worker.js

@@ -5,6 +5,7 @@
  */
 
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
@@ -36,7 +37,10 @@ export async function fetchUrl(url, options = {}) {
     if (response.status === 429 || response.status === 503) {
       noteRetryAfter(url, response.headers?.get?.('retry-after'));
     }
-    const html = await readBodyCapped(response, maxBodySize);
+    // crawlforge-extractors' readBody: same size cap, but decoded with the
+    // body's real charset — the local reader this replaces decoded everything
+    // as UTF-8, which turned lua.org's ISO-8859-1 "português" into "portugu�s".
+    const html = await readBody(response, { maxBytes: maxBodySize });
     return { response, html, warnings: gate.warnings };
   } catch (error) {
     if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
@@ -50,47 +54,6 @@ export async function fetchUrl(url, options = {}) {
   }
 }
 
-/**
- * Read a Response body as text, enforcing maxBodySize (Content-Length
- * pre-check, then a running byte count while streaming). Falls back to plain
- * .text() for responses without a streamable body (e.g. test mocks).
- */
-async function readBodyCapped(response, maxBodySize) {
-  const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
-  if (contentLengthHeader !== null) {
-    const declared = parseInt(contentLengthHeader, 10);
-    if (!isNaN(declared) && declared > maxBodySize) {
-      throw new Error(`Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`);
-    }
-  }
-
-  if (!response.body || typeof response.body.getReader !== 'function') {
-    return response.text();
-  }
-
-  const reader = response.body.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    totalBytes += value.byteLength;
-    if (totalBytes > maxBodySize) {
-      reader.cancel();
-      throw new Error(`Response body too large: exceeded limit of ${maxBodySize} bytes`);
-    }
-    chunks.push(value);
-  }
-
-  const merged = new Uint8Array(totalBytes);
-  let offset = 0;
-  for (const chunk of chunks) {
-    merged.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return new TextDecoder().decode(merged);
-}
-
 /**
  * Scrape a single URL and return a result object.
  */

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { LLMsTxtAnalyzer } from '../../core/LLMsTxtAnalyzer.js';
 import { Logger } from '../../utils/Logger.js';
 import { getBaseUrl } from '../../utils/urlNormalizer.js';
@@ -382,7 +383,8 @@ export class GenerateLLMsTxtTool {
         headers: { ...gate.headers }
       });
       if (!response.ok) return null;
-      return this.extractHomePageMetadata(await response.text());
+      // Charset-aware read; .text() decodes UTF-8 only.
+      return this.extractHomePageMetadata(await readBody(response));
     } catch {
       return null;
     } finally {

package/src/tools/search/redditSearch.js

@@ -364,12 +364,15 @@ export class RedditSearchTool {
     try {
       data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
     } catch (error) {
-      // Arctic Shift sheds a scoped comment keyword search over its whole
-      // history with the same 422 "Timeout. Maybe slow down a bit" it uses for
-      // throttling (observed live 2026-08-30: r/webdev body=react failed
-      // unbounded and at 7d, answered at 3d). When the caller set no window,
-      // narrow it step by step rather than hand the caller the 422.
-      const narrowable = v.mode === 'comments' && Boolean(v.query) && !v.after;
+      // Arctic Shift sheds a scoped keyword search over its whole history
+      // with the same 422 "Timeout. Maybe slow down a bit" it uses for
+      // throttling (observed live 2026-08-30: r/webdev comments body=react
+      // failed unbounded and at 7d, answered at 3d; 2026-09-01: an
+      // r/selfhosted posts search failed unbounded and answered at 7d). When
+      // the caller set no window, narrow it step by step rather than hand the
+      // caller the 422. Thread mode returned above, so both remaining modes
+      // (posts, comments) narrow.
+      const narrowable = Boolean(v.query) && !v.after;
       if (!error.retryable || !narrowable) throw error;
       let lastError = error;
       for (const window of ['7d', '3d', '1d']) {
@@ -386,7 +389,7 @@ export class RedditSearchTool {
       if (!data) {
         throw new Error(
           `${lastError.message} — Arctic Shift timed out at every window (full history, 7d, 3d, 1d), ` +
-          'so its comment search is under load right now. Retry later, pass a narrower after/before, or scope to one post with link_id.'
+          'so its search is under load right now. Retry later, pass a narrower after/before, or scope to one post with link_id.'
         );
       }
       notes.push(`Arctic Shift timed out searching the full history; results are limited to the last ${windowApplied}. Pass after/before to choose the window.`);

package/src/tools/tracking/trackChanges/differ.js

@@ -3,6 +3,7 @@
  * URL content fetching and history/stat helper functions.
  */
 
+import { readBody } from 'crawlforge-extractors';
 import { safeFetch } from '../../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
@@ -81,7 +82,10 @@ export async function fetchContent(url, options = {}) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }
 
-    const content = await response.text();
+    // Charset-aware read; .text() decodes UTF-8 only, and mojibake here
+    // would both corrupt the reported diff and fake changes on a page whose
+    // bytes never moved.
+    const content = await readBody(response);
 
     return {
       content,
@@ -96,7 +100,11 @@ export async function fetchContent(url, options = {}) {
       }
     };
   } catch (error) {
-    throw new Error(`Failed to fetch content: ${error.message}`);
+    // undici reports every network failure as bare "fetch failed" and puts
+    // the real reason (ENOTFOUND, ECONNREFUSED, timeout) in error.cause —
+    // without it, a dead host and a typoed domain read identically.
+    const cause = error.cause?.message ? ` (${error.cause.message})` : '';
+    throw new Error(`Failed to fetch content: ${error.message}${cause}`);
   }
 }
 

package/src/utils/hiddenContent.js

@@ -112,6 +112,61 @@ function specificity(selector) {
   return ids * 10000 + classes * 100 + elements;
 }
 
+/**
+ * Iterate a stylesheet's top-level rules as {selector, body}, skipping any
+ * block nested inside a rule's body.
+ *
+ * The flat regex this replaces — ([^{}]+)\{([^{}]*)\} — had no concept of
+ * native CSS nesting, so it read a nested rule as a top-level one under its
+ * inner selector. legalblogs.wolterskluwer.com ships
+ *   .cg-splitpane-aside-container.cg-is-closed .cg-splitpane-aside-header-cstm{
+ *     ... h6{display:none;} a{display:none;} }
+ * — links hidden only in a closed sidebar pane — which the stripper read as a
+ * bare a{display:none} and deleted 56 of an article's 68 anchors. Nested rules
+ * are skipped rather than resolved: expanding them into full descendant
+ * selectors needs a real CSS parser, and skipping errs on keeping content.
+ * A rule left unclosed at end of input (stylesheets are size-capped upstream,
+ * so truncation mid-rule is real) is dropped for the same reason.
+ */
+function* topLevelRules(css) {
+  const n = css.length;
+  let i = 0;
+  let selStart = 0;
+  while (i < n) {
+    const ch = css[i];
+    if (ch === '{') {
+      const selector = css.slice(selStart, i);
+      let depth = 1;
+      let j = i + 1;
+      let segStart = j;
+      let body = '';
+      while (j < n && depth > 0) {
+        const c = css[j];
+        if (c === '{') {
+          if (depth === 1) {
+            // The text since the last ';' is the nested rule's selector, not a
+            // declaration of this rule — keep only the completed declarations.
+            const seg = css.slice(segStart, j);
+            body += seg.slice(0, seg.lastIndexOf(';') + 1);
+          }
+          depth++;
+        } else if (c === '}') {
+          depth--;
+          if (depth === 1) segStart = j + 1;
+          else if (depth === 0) body += css.slice(segStart, j);
+        }
+        j++;
+      }
+      if (depth === 0) yield { selector, body };
+      i = j;
+      selStart = j;
+    } else {
+      if (ch === '}') selStart = i + 1; // stray close — resync
+      i++;
+    }
+  }
+}
+
 /**
  * Parse CSS into rules that hide content and rules that re-show it, each
  * carrying the specificity and source order a browser would use to resolve
@@ -125,13 +180,9 @@ export function collectVisibilitySelectors(css) {
   if (!css) return { hide, show };
 
   const flat = stripConditionalBlocks(css);
-  const rule = /([^{}]+)\{([^{}]*)\}/g;
-  let m;
   let order = 0;
 
-  while ((m = rule.exec(flat)) !== null) {
-    const selectorList = m[1];
-    const body = m[2];
+  for (const { selector: selectorList, body } of topLevelRules(flat)) {
     order++;
 
     const display = /(?:^|[;{\s])display\s*:\s*([a-z-]+)/i.exec(body);

package/src/utils/languageDetection.js

@@ -214,12 +214,17 @@ export function detectLanguage(text, options = {}) {
 
     const confidence = Math.min(1, 0.5 + (text.length / 500) * 0.5);
 
+    // francAll scores are similarity relative to the winner (whose own score
+    // is 1), while `confidence` above is length-based — two different scales.
+    // Printed raw, a runner-up could read 0.82 against a winner shown at 0.8.
+    // Scaling by the winner's confidence keeps the ranking and the invariant
+    // that no alternative outranks the pick.
     const alternative = francAll(text, { minLength, whitelist: Object.keys(LANGUAGE_NAMES) })
       .slice(1, 4)
       .map(([code, score]) => ({
         code,
         name: LANGUAGE_NAMES[code] || code,
-        confidence: Math.round(score * 100) / 100
+        confidence: Math.round(score * confidence * 100) / 100
       }));
 
     return {