crawlforge-mcp-server 4.10.0 → 5.0.1

package/src/tools/tracking/trackChanges/index.js

@@ -14,6 +14,8 @@
  */
 
 import { EventEmitter } from 'events';
+import os from 'os';
+import path from 'path';
 import ChangeTracker from '../../../core/ChangeTracker.js';
 import SnapshotManager from '../../../core/SnapshotManager.js';
 import CacheManager from '../../../core/cache/CacheManager.js';
@@ -31,7 +33,12 @@ export class TrackChangesTool extends EventEmitter {
     this.options = {
       cacheEnabled: true,
       cacheTTL: 3600000,
-      snapshotStorageDir: './snapshots',
+      // Rooted in a stable, non-cwd-dependent base (~/.crawlforge — same
+      // convention as ~/.crawlforge/config.json) rather than process.cwd(),
+      // since MCP clients (e.g. Claude Desktop) may launch the server with a
+      // cwd the process cannot write to (e.g. '/'), which previously made
+      // every snapshot write fail silently.
+      snapshotStorageDir: path.join(os.homedir(), '.crawlforge', 'snapshots'),
       enableRealTimeMonitoring: true,
       maxConcurrentMonitors: 50,
       defaultPollingInterval: 300000,
@@ -64,7 +71,15 @@ export class TrackChangesTool extends EventEmitter {
     this.monitorStore = new MonitorStore({ storageDir: this.options.monitorStorageDir || './monitors' });
     this.scheduler = new MonitorScheduler({ tool: this, store: this.monitorStore });
 
-    this.initialize();
+    // Wired synchronously (no I/O) so no 'error' event emitted by
+    // changeTracker/snapshotManager during the lazy initialize() below (see
+    // ensureInitialized()) can ever be emitted before a listener exists.
+    this._setupEventHandlers();
+
+    // Not started here — construction stays synchronous and side-effect
+    // free. ensureInitialized() lazily creates and memoizes this on first
+    // real use (called from execute()/startScheduler()/runDueOnce()).
+    this._initPromise = null;
   }
 
   /** Wire the MCP server so the goal-judge can use SamplingClient (Ollama-first). */
@@ -74,6 +89,7 @@ export class TrackChangesTool extends EventEmitter {
 
   /** Start the in-process scheduler (called once, by the server). */
   async startScheduler() {
+    await this.ensureInitialized();
     if (this._mcpServer && !this.scheduler.samplingClient) {
       try {
         const { SamplingClient } = await import('../../../core/SamplingClient.js');
@@ -87,6 +103,7 @@ export class TrackChangesTool extends EventEmitter {
 
   /** Fire every due monitor once and exit (the external-cron one-shot path). */
   async runDueOnce() {
+    await this.ensureInitialized();
     if (this._mcpServer && !this.scheduler.samplingClient) {
       try {
         const { SamplingClient } = await import('../../../core/SamplingClient.js');
@@ -96,24 +113,33 @@ export class TrackChangesTool extends EventEmitter {
     return this.scheduler.runDueOnce();
   }
 
-  async initialize() {
-    try {
-      await this.snapshotManager.initialize();
-      this._setupEventHandlers();
-      this.emit('initialized');
-    } catch (error) {
-      this.emit('error', { operation: 'initialize', error: error.message });
-      throw error;
+  /**
+   * Lazily runs (and memoizes) storage initialization, awaited by execute()
+   * and the other top-level entry points below. Replaces the old pattern of
+   * firing initialize() unawaited from the constructor, which could turn a
+   * snapshot-directory failure into an opaque unhandled rejection.
+   */
+  async ensureInitialized() {
+    if (!this._initPromise) {
+      this._initPromise = this.snapshotManager.ensureInitialized().then(() => {
+        this.emit('initialized');
+      });
     }
+    return this._initPromise;
   }
 
   _setupEventHandlers() {
     this.changeTracker.on('changeDetected', async (changeRecord) => {
-      if (changeRecord.significance !== 'none') {
+      // changeRecord.details is diff analysis and carries no page content —
+      // storing `details.current || ''` wrote an empty junk snapshot on every
+      // significant change (compareWithBaseline already snapshots the real
+      // current content). Only store if a future record ever carries content.
+      const current = changeRecord.details?.current;
+      if (changeRecord.significance !== 'none' && current) {
         try {
           await this.snapshotManager.storeSnapshot(
             changeRecord.url,
-            changeRecord.details.current || '',
+            current,
             { changes: changeRecord.details, significance: changeRecord.significance, changeType: changeRecord.changeType }
           );
         } catch (error) {
@@ -129,6 +155,8 @@ export class TrackChangesTool extends EventEmitter {
 
   async execute(params) {
     try {
+      await this.ensureInitialized();
+
       const validated = TrackChangesSchema.parse(params);
       const { operation } = validated;
 
@@ -169,7 +197,7 @@ export class TrackChangesTool extends EventEmitter {
     const baseline = await this.changeTracker.createBaseline(url, sourceContent, trackingOptions);
     let snapshotInfo = null;
     if (enableSnapshots) {
-      snapshotInfo = await this.snapshotManager.storeSnapshot(url, sourceContent, { ...fetchMeta, baseline: true, trackingOptions });
+      snapshotInfo = await this.snapshotManager.storeSnapshot(url, sourceContent, { ...fetchMeta, baseline: true, trackingOptions }, { enableCompression: storageOptions.compressionEnabled });
     }
 
     return {
@@ -186,9 +214,34 @@ export class TrackChangesTool extends EventEmitter {
     };
   }
 
+  // Rebuild the in-memory baseline from the newest persisted snapshot, so
+  // compare works across processes (fresh CLI runs, server restarts). Same
+  // fail-soft mechanism as MonitorScheduler._ensureBaseline: no usable
+  // snapshot means compare still reports "No baseline" for a genuine first run.
+  async rehydrateBaseline(url, trackingOptions = {}) {
+    if (this.changeTracker?.snapshots?.has(url)) return;
+    try {
+      // limit > 1: existing stores contain empty junk snapshots (from the old
+      // changeDetected handler) that can tie on timestamp with the real one —
+      // take the newest snapshot that actually has content.
+      const q = await this.snapshotManager.querySnapshots({ url, limit: 5, includeContent: true });
+      for (const snap of q?.snapshots ?? []) {
+        let content = snap?.content;
+        if (Buffer.isBuffer(content)) content = content.toString('utf8');
+        if (content && typeof content === 'string') {
+          await this.changeTracker.createBaseline(url, content, trackingOptions);
+          return;
+        }
+      }
+    } catch {
+      /* no usable snapshot — caller's compare will surface "No baseline" */
+    }
+  }
+
   async compareWithBaseline(params) {
     const { url, content, html, trackingOptions, storageOptions = {}, notificationOptions } = params;
     const enableSnapshots = storageOptions.enableSnapshots !== false;
+    await this.rehydrateBaseline(url, trackingOptions);
 
     let currentContent = content || html;
     let fetchMeta = {};
@@ -199,13 +252,13 @@ export class TrackChangesTool extends EventEmitter {
     }
     if (!currentContent || typeof currentContent !== 'string') throw new Error('Invalid content');
 
-    const comparisonResult = await this.changeTracker.compareWithBaseline(url, currentContent, trackingOptions);
+    const comparisonResult = await this.changeTracker.compareWithBaseline(url, currentContent, trackingOptions, storageOptions);
 
     let snapshotInfo = null;
     if (comparisonResult.hasChanges && enableSnapshots) {
       snapshotInfo = await this.snapshotManager.storeSnapshot(url, currentContent, {
         ...fetchMeta, changes: comparisonResult.summary, significance: comparisonResult.significance
-      });
+      }, { enableCompression: storageOptions.compressionEnabled });
     }
 
     if (comparisonResult.hasChanges && notificationOptions) {
@@ -339,7 +392,10 @@ export class TrackChangesTool extends EventEmitter {
     const monitorId = scheduledMonitorOptions?.monitorId;
     if (monitorId) {
       const result = await this.scheduler.stopMonitor(monitorId);
-      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: result.stopped, timestamp: Date.now() };
+      if (!result.stopped) {
+        return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}`, timestamp: Date.now() };
+      }
+      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
     }
     if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
     const result = await this.scheduler.stopByUrl(url);
@@ -421,6 +477,17 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async shutdown() {
+    // ensureInitialized() is lazy (see above) and awaits
+    // this.snapshotManager.ensureInitialized(), which is itself lazy. If a
+    // caller triggered initialization and then immediately calls shutdown(),
+    // we must wait for that in-flight init to finish first — otherwise its
+    // cleanup timer could start after snapshotManager.shutdown() already
+    // tried to stop it, leaking a live timer. If nothing ever triggered
+    // initialization, _initPromise is still null and there's nothing to
+    // wait for.
+    if (this._initPromise) {
+      await this._initPromise.catch(() => {});
+    }
     this.stopAllMonitoring();
     this.scheduler?.stopAll();
     await this.snapshotManager.shutdown();
@@ -469,17 +536,44 @@ export class TrackChangesTool extends EventEmitter {
 
 export default TrackChangesTool;
 
-// Singleton instance — kept for backward-compat with any code that imports it directly
-export const trackChangesTool = new TrackChangesTool();
-trackChangesTool.name = 'track_changes';
-trackChangesTool.validateParameters = (params) => TrackChangesSchema.parse(params);
-trackChangesTool.description = 'Track and analyze content changes with baseline capture, comparison, and monitoring capabilities';
-trackChangesTool.inputSchema = {
-  type: 'object',
-  properties: {
-    url: { type: 'string', description: 'URL to track for changes' },
-    operation: { type: 'string', description: 'Operation to perform: create_baseline, compare, monitor, get_history, get_stats' },
-    content: { type: 'string', description: 'Content to analyze or compare' }
+// Singleton instance — kept for backward-compat with any code that imports it
+// directly. Built lazily, on first property access, rather than eagerly at
+// module-import time: the server (server.js) constructs and owns its own
+// TrackChangesTool instance, so an eager `new TrackChangesTool()` here spun
+// up a second, never-shut-down ChangeTracker/SnapshotManager/CacheManager/
+// MonitorStore/MonitorScheduler (and touched the filesystem) merely by
+// importing this module. Nothing in this codebase currently imports this
+// named export directly, but it's preserved for backward-compat.
+let _singleton = null;
+function _getTrackChangesToolSingleton() {
+  if (!_singleton) {
+    _singleton = new TrackChangesTool();
+    _singleton.name = 'track_changes';
+    _singleton.validateParameters = (params) => TrackChangesSchema.parse(params);
+    _singleton.description = 'Track and analyze content changes with baseline capture, comparison, and monitoring capabilities';
+    _singleton.inputSchema = {
+      type: 'object',
+      properties: {
+        url: { type: 'string', description: 'URL to track for changes' },
+        operation: { type: 'string', description: 'Operation to perform: create_baseline, compare, monitor, get_history, get_stats' },
+        content: { type: 'string', description: 'Content to analyze or compare' }
+      },
+      required: ['url']
+    };
+  }
+  return _singleton;
+}
+
+export const trackChangesTool = new Proxy({}, {
+  get(_target, prop) {
+    const instance = _getTrackChangesToolSingleton();
+    const value = Reflect.get(instance, prop, instance);
+    return typeof value === 'function' ? value.bind(instance) : value;
   },
-  required: ['url']
-};
+  set(_target, prop, value) {
+    return Reflect.set(_getTrackChangesToolSingleton(), prop, value);
+  },
+  has(_target, prop) {
+    return Reflect.has(_getTrackChangesToolSingleton(), prop);
+  }
+});

package/src/tools/tracking/trackChanges/schema.js

@@ -56,7 +56,7 @@ export const TrackChangesSchema = z.object({
     enableWebhook: z.boolean().default(false),
     webhookUrl: z.string().url().optional(),
     webhookSecret: z.string().optional()
-  }).optional(),
+  }).optional().default({}),
 
   storageOptions: z.object({
     enableSnapshots: z.boolean().default(true),
@@ -73,7 +73,7 @@ export const TrackChangesSchema = z.object({
     endTime: z.number().optional(),
     includeContent: z.boolean().default(false),
     significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
-  }).optional(),
+  }).optional().default({}),
 
   notificationOptions: z.object({
     email: z.object({

package/src/utils/CircuitBreaker.js

@@ -26,9 +26,11 @@ export class CircuitBreaker {
     this.monitoringWindow = monitoringWindow;
     this.errorThresholdPercentage = errorThresholdPercentage;
     this.minimumThroughput = minimumThroughput;
-    this.onStateChange = onStateChange;
-    this.onFailure = onFailure;
-    this.onSuccess = onSuccess;
+    // Stored under distinct names so they don't shadow the onStateChange/
+    // onFailure/onSuccess prototype methods below (execute() calls those).
+    this.onStateChangeCallback = onStateChange;
+    this.onFailureCallback = onFailure;
+    this.onSuccessCallback = onSuccess;
     this.name = name;
 
     // Circuit state per service endpoint
@@ -138,8 +140,8 @@ export class CircuitBreaker {
     }
 
     // Call success callback
-    if (this.onSuccess) {
-      this.onSuccess(serviceId, duration);
+    if (this.onSuccessCallback) {
+      this.onSuccessCallback(serviceId, duration);
     }
   }
 
@@ -166,8 +168,8 @@ export class CircuitBreaker {
     }
 
     // Call failure callback
-    if (this.onFailure) {
-      this.onFailure(serviceId, error, duration);
+    if (this.onFailureCallback) {
+      this.onFailureCallback(serviceId, error, duration);
     }
   }
 
@@ -221,8 +223,8 @@ export class CircuitBreaker {
     }
 
     // Call state change callback
-    if (this.onStateChange) {
-      this.onStateChange(serviceId, oldState, newState, circuit);
+    if (this.onStateChangeCallback) {
+      this.onStateChangeCallback(serviceId, oldState, newState, circuit);
     }
 
     // Start health monitoring for open circuits

package/src/utils/contentUtils.js

@@ -93,61 +93,74 @@ export class HTMLCleaner {
 
     let text = '';
 
-    $('body').find('*').each((_, element) => {
-      const $element = $(element);
-      const tagName = element.tagName.toLowerCase();
-
-      switch (tagName) {
-        case 'p':
-        case 'div':
-          if (extractOptions.preserveParagraphs) {
-            text += '\n\n' + $element.text().trim();
-          } else {
-            text += ' ' + $element.text().trim();
-          }
-          break;
-        case 'br':
-          if (extractOptions.preserveLineBreaks) {
-            text += '\n';
-          }
-          break;
-        case 'h1':
-        case 'h2':
-        case 'h3':
-        case 'h4':
-        case 'h5':
-        case 'h6':
-          text += '\n\n' + $element.text().trim().toUpperCase() + '\n';
-          break;
-        case 'a':
-          if (extractOptions.includeLinks) {
-            const href = $element.attr('href');
-            const linkText = $element.text().trim();
-            text += ` ${linkText}${href ? ` (${href})` : ''}`;
-          } else {
-            text += ' ' + $element.text().trim();
-          }
-          break;
-        case 'img':
-          if (extractOptions.includeImageAlt) {
-            const alt = $element.attr('alt');
-            if (alt) {
-              text += ` [Image: ${alt}]`;
+    // Walk direct children recursively (rather than $('body').find('*'),
+    // which flattens every descendant) so a block element's text isn't
+    // captured once via $element.text() (which includes nested content) and
+    // then again when the walker separately visits its nested elements.
+    function walk($el) {
+      $el.contents().each((_, node) => {
+        if (node.type === 'text') {
+          const value = (node.data || '').trim();
+          if (value) text += ' ' + value;
+          return;
+        }
+        if (node.type !== 'tag') return;
+
+        const $node = $(node);
+        const tagName = node.tagName.toLowerCase();
+
+        switch (tagName) {
+          case 'p':
+          case 'div':
+            text += extractOptions.preserveParagraphs ? '\n\n' : ' ';
+            walk($node);
+            break;
+          case 'br':
+            if (extractOptions.preserveLineBreaks) {
+              text += '\n';
             }
-          }
-          break;
-        case 'li':
-          text += '\n• ' + $element.text().trim();
-          break;
-        default:
-          // For other elements, just extract text
-          if ($element.children().length === 0) {
-            text += ' ' + $element.text().trim();
-          }
-      }
-    });
+            break;
+          case 'h1':
+          case 'h2':
+          case 'h3':
+          case 'h4':
+          case 'h5':
+          case 'h6':
+            text += '\n\n' + $node.text().trim().toUpperCase() + '\n';
+            break;
+          case 'a':
+            if (extractOptions.includeLinks) {
+              const href = $node.attr('href');
+              const linkText = $node.text().trim();
+              text += ` ${linkText}${href ? ` (${href})` : ''}`;
+            } else {
+              walk($node);
+            }
+            break;
+          case 'img':
+            if (extractOptions.includeImageAlt) {
+              const alt = $node.attr('alt');
+              if (alt) {
+                text += ` [Image: ${alt}]`;
+              }
+            }
+            break;
+          case 'li':
+            text += '\n• ';
+            walk($node);
+            break;
+          default:
+            walk($node);
+        }
+      });
+    }
+
+    walk($('body'));
 
-    return text.replace(/\s+/g, ' ').replace(/\n\s+/g, '\n').trim();
+    // Collapse horizontal whitespace only — collapsing all whitespace
+    // (including newlines) here would erase the line breaks/paragraphs the
+    // options above were just asked to preserve.
+    return text.replace(/[ \t]+/g, ' ').replace(/[ \t]*\n[ \t]*/g, '\n').trim();
   }
 }
 

package/src/utils/secretMask.js

@@ -6,7 +6,7 @@
  *   logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
  */
 
-const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|private[_-]?key|access[_-]?key|proxy_url|proxyurl/i;
+const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
 
 const MASK = '[REDACTED]';
 const PARTIAL_MASK_LEN = 4; // show last N chars of long secrets

package/src/utils/sitemapParser.js

@@ -392,19 +392,21 @@ export class SitemapParser {
         return null;
       }
 
-      const contentType = response.headers.get('content-type') || '';
-      const contentEncoding = response.headers.get('content-encoding') || '';
-      
+      // fetch (undici) transparently decompresses a gzip Content-Encoding
+      // while leaving the response header intact, so that header can't be
+      // trusted to decide whether the body still needs gunzipping. Sniff the
+      // actual bytes instead: a real gzip payload starts with the 0x1f 0x8b
+      // magic number regardless of what the headers claim.
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const isGzipped = buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
+
       let content;
-      
-      // Handle compressed content
-      if (url.endsWith('.gz') || contentEncoding.includes('gzip')) {
-        const buffer = await response.arrayBuffer();
-        const decompressed = await gunzip(Buffer.from(buffer));
+      if (isGzipped) {
+        const decompressed = await gunzip(buffer);
         content = decompressed.toString('utf8');
         this.stats.compressionSavings += buffer.byteLength - decompressed.length;
       } else {
-        content = await response.text();
+        content = buffer.toString('utf8');
       }
 
       return content;