crawlforge-mcp-server 5.5.1 → 5.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.5.1",
+  "version": "5.5.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.5.1",
+  version: "5.5.3",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1412,7 +1412,13 @@ registerToolIfEnabled("stealth_mode", {
         result = { disabled: true };
         break;
       case 'create_context': {
-        const contextData = await stealthBrowserManager.createStealthContext(stealthConfig);
+        // Forward the tool-level engine the same way operation:"scrape" does —
+        // without it, create_context always ran on chromium whatever the caller
+        // asked for ("playwright" is this tool's public name for chromium).
+        const contextData = await stealthBrowserManager.createStealthContext({
+          ...(stealthConfig || {}),
+          engine: engine === 'camoufox' ? 'camoufox' : 'chromium'
+        });
         // The full fingerprint is ~4 KB of canvas noise arrays and WebGL
         // extension lists no caller acts on. Summarise by default; verbose:true
         // still returns all of it for debugging a detection failure.

package/src/core/StealthBrowserManager.js

@@ -434,9 +434,11 @@ export class StealthBrowserManager {
   async createStealthContext(config = {}) {
     const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
     
-    if (!this.browser) {
-      await this.launchStealthBrowser(validatedConfig);
-    }
+    // Always go through launchStealthBrowser: it returns the running browser
+    // when the engine matches, and closes + relaunches on an engine mismatch.
+    // Guarding on `!this.browser` here skipped that mismatch check, so a
+    // camoufox request silently reused an already-running chromium browser.
+    await this.launchStealthBrowser(validatedConfig);
 
     // Generate fingerprint for this context
     const fingerprint = this.generateAdvancedFingerprint(validatedConfig);

package/src/tools/extract/extractStructured.js

@@ -192,6 +192,14 @@ export class ExtractStructuredTool {
       // the model was shown would null every correct price.
       const fullSource = `${html}\n${textContent}`;
 
+      // A model that saw no answer sometimes fills a required field with a
+      // placeholder ("N/A", "unknown") instead of null. That carries no digits,
+      // so the provenance guard has nothing to check and the retry below never
+      // fired — git-scm.com is the case that found this: Readability drops the
+      // version box, the model answered "N/A", and the page plainly says 2.55.0.
+      const isPlaceholder = (v) => v == null ||
+        (typeof v === 'string' && (v.trim() === '' || /^(n\/?a|none|null|unknown|not (available|found|specified))$/i.test(v.trim())));
+
       /** Run the guard over one extraction and describe what it did. */
       const applyGuard = (result) => {
         const checked = verifyNumericProvenance(result.data || {}, fullSource);
@@ -201,9 +209,14 @@ export class ExtractStructuredTool {
         const nulledRequired = checked.unverified
           .map((entry) => entry.path)
           .filter((path) => (schema.required || []).includes(path));
+        // Retry currency: required fields with no usable answer, whether the
+        // guard nulled them or the model handed back a placeholder.
+        const missingRequired = (schema.required || [])
+          .filter((path) => nulledRequired.includes(path) || isPlaceholder(checked.data?.[path]));
         return {
           checked,
           nulledRequired,
+          missingRequired,
           result: {
             ...result,
             data: checked.data,
@@ -234,7 +247,7 @@ export class ExtractStructuredTool {
         // which is exactly what the guard will accept. Only a strictly better
         // result is kept, so a retry can never make the answer worse.
         const shownText = mainContentText($, html, url) || textContent;
-        if (guarded.nulledRequired.length > 0 && shownText !== textContent && textContent) {
+        if (guarded.missingRequired.length > 0 && shownText !== textContent && textContent) {
           try {
             const llm = this._ensureLLMManager(llmConfig || {});
             const retry = await llm.extractStructured(textContent, schema, {
@@ -243,7 +256,7 @@ export class ExtractStructuredTool {
             });
             if (retry?.method === 'llm') {
               const retryGuarded = applyGuard(retry);
-              if (retryGuarded.nulledRequired.length < guarded.nulledRequired.length) {
+              if (retryGuarded.missingRequired.length < guarded.missingRequired.length) {
                 guarded = retryGuarded;
               }
             }

package/src/tools/scrape/_mainContent.js

@@ -1,14 +1,39 @@
 /**
  * _mainContent.js — Readability main-content extraction, plus recovery of the
- * data tables Readability drops.
+ * content Readability drops.
  *
- * Readability keeps one article candidate and discards everything outside it.
- * On a table-led page that silently loses the payload: Wikipedia's *List of
- * S&P 500 companies* came back with zero table rows at scrape's default
- * onlyMainContent:true, and 505 pipe-table lines with it off. No Readability
- * option recovers them (charThreshold:100 and nbTopCandidates:20 were both
- * probed) — the tables are simply not in the candidate — so they are
- * re-attached afterwards.
+ * Two independent losses are repaired here.
+ *
+ * 1. Readability keeps one article candidate and discards everything outside
+ *    it. On a table-led page that silently loses the payload: Wikipedia's
+ *    *List of S&P 500 companies* came back with zero table rows at scrape's
+ *    default onlyMainContent:true, and 505 pipe-table lines with it off. No
+ *    Readability option recovers them (charThreshold:100 and nbTopCandidates:20
+ *    were both probed) — the tables are simply not in the candidate — so they
+ *    are re-attached afterwards.
+ *
+ * 2. Readability's class regexes are unanchored, so they fire on words their
+ *    author never meant. Two collisions between them deleted every code
+ *    example on next-intl.dev/docs/routing/setup:
+ *
+ *      - `unlikelyCandidates` contains `extra`, which matches inside `nextra-*`
+ *        — the class prefix on every element Nextra emits, and Nextra builds a
+ *        large share of the JS ecosystem's documentation. None of those classes
+ *        match `okMaybeItsACandidate`, so the elements are deleted before
+ *        scoring runs: all 96 inline `<code>` spans and the "setRequestLocale
+ *        is legacy" callout went with them.
+ *      - `negative` contains `hidden`, which matches Tailwind's
+ *        `overflow-hidden`. That is only -25 class weight, but it is enough for
+ *        `_cleanConditionally` to drop the wrapper around all 12 `<pre>` blocks.
+ *
+ *    Both losses were silent, and the second is the dangerous one: the prose
+ *    survived, so the page read as complete while every code example and the
+ *    warning callout were gone, and sentences came back with holes where their
+ *    inline code had been ("In order to use unique pathnames …, can be used to
+ *    handle"). `sanitizeAccidentalClasses` strips just those class tokens
+ *    before Readability sees the document, restoring 11 of the 12 blocks — the
+ *    12th sits inside a collapsed `<details>`, which Readability drops whole
+ *    and which this deliberately does not try to rescue.
  *
  * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
  */
@@ -20,10 +45,108 @@ import { Readability } from '@mozilla/readability';
 // Readability already kept it. Long enough to be unique to that table.
 const SIGNATURE_LENGTH = 120;
 
+// Copied verbatim from @mozilla/readability's REGEXPS. Two of them delete
+// content, and both are unanchored: `unlikelyCandidates` removes an element
+// outright, `negative` docks 25 from its class weight, which is enough for
+// `_cleanConditionally` to drop it. The unit test asserts the real library
+// still behaves this way, so a Readability upgrade that fixes it upstream
+// fails loudly here rather than leaving a no-op pass behind.
+const UNLIKELY_SOURCE =
+  '-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote';
+const NEGATIVE_SOURCE =
+  '-ad-|hidden|^hid$| hid$| hid |^hidden|banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget';
+const REMOVAL_SOURCES = [UNLIKELY_SOURCE, NEGATIVE_SOURCE];
+// Readability's `positive` regex. A token matching it carries a signal worth
+// keeping, so it is never stripped even if it also matches mid-word.
+const POSITIVE_SIGNAL =
+  /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i;
+
+// Utility classes that collide with the `negative` regex on a whole word while
+// meaning something else entirely. Tailwind's `overflow-hidden` clips overflow;
+// it does not mark the element hidden. The mid-word rule below cannot catch
+// these — `hidden` really is a whole segment of `overflow-hidden` — and the
+// cost is real: -25 class weight is enough for `_cleanConditionally` to delete
+// the element, which is what removed every code block on next-intl.dev after
+// the `nextra-` collision had already been handled. Listed explicitly rather
+// than inferred, because `page-footer` and `main-sidebar` are the same shape
+// and *are* genuine.
+const UTILITY_FALSE_POSITIVES = new Set([
+  'overflow-hidden',
+  'overflow-x-hidden',
+  'overflow-y-hidden',
+  'overflow-scroll',
+  'overflow-x-scroll',
+  'overflow-y-scroll',
+  'scroll-smooth',
+  'scroll-auto'
+]);
+
 function normalizeWhitespace(text) {
   return text.replace(/\s+/g, ' ').trim();
 }
 
+/**
+ * True when `token` only ever trips Readability's removal regexes inside a
+ * longer word — `extra` within `nextra`, `hidden` within Tailwind's
+ * `overflow-hidden`.
+ *
+ * Only a *preceding* letter or digit counts as mid-word. A trailing one does
+ * not, because that is how English suffixes attach: rescuing `comments`,
+ * `banners` or `footers` would defeat the regexes on exactly the clutter they
+ * were written to catch, whereas a letter run in front of the match means a
+ * different word entirely.
+ *
+ * @param {string} token - one class name
+ * @returns {boolean}
+ */
+function isAccidentalToken(token) {
+  if (UTILITY_FALSE_POSITIVES.has(token.toLowerCase())) return true;
+  if (POSITIVE_SIGNAL.test(token)) return false;
+  let sawMatch = false;
+  for (const source of REMOVAL_SOURCES) {
+    const scan = new RegExp(source, 'gi');
+    let match;
+    while ((match = scan.exec(token)) !== null) {
+      if (match[0].length === 0) {
+        scan.lastIndex += 1;
+        continue;
+      }
+      sawMatch = true;
+      const before = token[match.index - 1];
+      if (before === undefined || !/[a-z0-9]/i.test(before)) return false;
+    }
+  }
+  return sawMatch;
+}
+
+/**
+ * Strip the class tokens that would make Readability discard an element over a
+ * word the class does not actually contain.
+ *
+ * Only whole tokens that are accidental on every match are removed, so a class
+ * carrying a real signal survives: an element still genuinely marked `sidebar`
+ * or `hidden` is still dropped, as it should be. Ids are left alone — they
+ * anchor in-page links, and the class attribute is what these frameworks put
+ * their names on.
+ *
+ * @param {Document} document - mutated in place
+ * @returns {number} how many elements were sanitized
+ */
+export function sanitizeAccidentalClasses(document) {
+  let sanitized = 0;
+  for (const el of Array.from(document.querySelectorAll('[class]'))) {
+    const className = el.getAttribute('class') || '';
+    const tokens = className.split(/\s+/).filter(Boolean);
+    const kept = tokens.filter((token) => !isAccidentalToken(token));
+    if (kept.length === tokens.length) continue;
+
+    if (kept.length > 0) el.setAttribute('class', kept.join(' '));
+    else el.removeAttribute('class');
+    sanitized += 1;
+  }
+  return sanitized;
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -61,6 +184,7 @@ export function extractMainContent(html, url) {
   let article;
   try {
     const dom = new JSDOM(html, { url });
+    sanitizeAccidentalClasses(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };