crawlforge-mcp-server 5.2.4 → 5.2.5

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.4
+**Current Version:** 5.2.5
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.4",
+  "version": "5.2.5",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,7 +113,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.2.0",
+    "crawlforge-extractors": "^1.2.1",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.4",
+  version: "5.2.5",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/ChangeTracker.js

@@ -312,14 +312,22 @@ export class ChangeTracker extends EventEmitter {
       
       this.emit('changeDetected', changeRecord);
       
+      const ignoredOptions = this.findIgnoredCompareOptions(options, baseline.options);
+
       return {
         hasChanges: significance !== 'none',
         significance,
         changeType: changeRecord.changeType,
-        summary: this.generateChangeSummary(changeAnalysis),
+        summary: this.generateChangeSummary(changeAnalysis, significance),
         details: changeAnalysis,
         metrics: changeRecord.metrics,
-        recommendations: this.generateChangeRecommendations(changeRecord)
+        recommendations: this.generateChangeRecommendations(changeRecord),
+        ...(ignoredOptions.length ? {
+          warnings: [
+            `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
+            `the baseline's options are applied to both sides of the diff. Recreate the baseline to change them.`
+          ]
+        } : {})
       };
       
     } catch (error) {
@@ -328,6 +336,27 @@ export class ChangeTracker extends EventEmitter {
     }
   }
   
+  /**
+   * Report analysis options supplied at compare time that differ from the
+   * baseline's and were therefore not applied.
+   *
+   * Ignoring them is deliberate — both sides of a diff have to be analyzed
+   * identically, and once customSelectors scoped the baseline it no longer
+   * holds the full document to re-scope. But doing it silently let a caller
+   * scope a compare and read the resulting whole-page churn as real change:
+   * the result is byte-identical to an unscoped run, with nothing saying so.
+   *
+   * @param {Object} callerOptions - trackingOptions passed to this compare
+   * @param {Object} baselineOptions - options stored with the baseline
+   * @returns {string[]} - names of the ignored options
+   */
+  findIgnoredCompareOptions(callerOptions = {}, baselineOptions = {}) {
+    return ['granularity', 'customSelectors', 'excludeSelectors'].filter(key => {
+      if (callerOptions[key] === undefined) return false;
+      return JSON.stringify(callerOptions[key]) !== JSON.stringify(baselineOptions[key]);
+    });
+  }
+  
   /**
    * Analyze content structure and create hierarchical hashes
    * @param {string} content - Content to analyze
@@ -701,6 +730,29 @@ export class ChangeTracker extends EventEmitter {
         }
       });
     });
+
+    // Scoping to a tag outside that list (address, td, li, tr, dd) otherwise
+    // indexes ZERO elements: the scoped document is hashed, but nothing in it
+    // matches the allowlist, so every compare sees an empty element map and
+    // can never report an element-level change. Hash the scoped elements
+    // themselves for any tag the loop above does not already cover.
+    if (options.customSelectors?.length) {
+      const alreadyHashed = new Set(importantElements);
+      options.customSelectors.forEach((selector, selectorIndex) => {
+        $(selector).each((index, element) => {
+          const tag = element.tagName?.toLowerCase();
+          if (!tag || alreadyHashed.has(tag)) return;
+
+          const elementKey = `custom_${selectorIndex}_${index}`;
+          analysis.hashes.elements[elementKey] = this.hashContent($(element).html() || '');
+
+          if (options.trackAttributes) {
+            const attributes = element.attribs || {};
+            analysis.hashes.elements[`${elementKey}_attr`] = this.hashContent(JSON.stringify(attributes));
+          }
+        });
+      });
+    }
   }
   
   async analyzeTextLevel($, analysis, options) {
@@ -1160,7 +1212,7 @@ export class ChangeTracker extends EventEmitter {
     return 'text_change';
   }
   
-  generateChangeSummary(changeAnalysis) {
+  generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;
     
     const total = addedElements.length + removedElements.length + modifiedElements.length;
@@ -1171,7 +1223,13 @@ export class ChangeTracker extends EventEmitter {
       added: addedElements.length,
       removed: removedElements.length,
       modified: modifiedElements.length,
-      changeDescription: this.generateChangeDescription(changeAnalysis)
+      // Sub-threshold text noise (a rotating session token, a base64 timestamp)
+      // still lands in textChanges, so the description read "Text content
+      // changed" on a compare that reported hasChanges:false and
+      // totalChanges:0. Defer to the verdict the caller is given.
+      changeDescription: significance === 'none'
+        ? 'No significant changes detected'
+        : this.generateChangeDescription(changeAnalysis)
     };
   }
   

package/src/tools/tracking/trackChanges/index.js

@@ -281,6 +281,7 @@ export class TrackChangesTool extends EventEmitter {
       details: comparisonResult.details,
       metrics: comparisonResult.metrics,
       recommendations: comparisonResult.recommendations,
+      ...(comparisonResult.warnings ? { warnings: comparisonResult.warnings } : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }