crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/core/SnapshotManager.js

@@ -6,6 +6,7 @@
 
 import { promises as fs } from 'fs';
 import path from 'path';
+import os from 'os';
 import { createHash } from 'crypto';
 import { gzip, gunzip } from 'zlib';
 import { promisify } from 'util';
@@ -77,10 +78,16 @@ export class SnapshotManager extends EventEmitter {
   constructor(options = {}) {
     super();
     
+    // Default storage location is rooted in a stable, non-cwd-dependent base
+    // (~/.crawlforge — same convention as ~/.crawlforge/config.json) rather
+    // than process.cwd(), since MCP clients (e.g. Claude Desktop) may launch
+    // the server with a cwd the process cannot write to (e.g. '/').
+    const defaultBaseDir = path.join(os.homedir(), '.crawlforge', 'snapshots');
+
     this.options = {
-      storageDir: options.storageDir || './snapshots',
-      metadataDir: options.metadataDir || './snapshots/metadata',
-      tempDir: options.tempDir || './snapshots/temp',
+      storageDir: options.storageDir || defaultBaseDir,
+      metadataDir: options.metadataDir || path.join(defaultBaseDir, 'metadata'),
+      tempDir: options.tempDir || path.join(defaultBaseDir, 'temp'),
       enableCompression: options.enableCompression !== false,
       enableDeltaStorage: options.enableDeltaStorage !== false,
       enableEncryption: options.enableEncryption || false,
@@ -88,6 +95,12 @@ export class SnapshotManager extends EventEmitter {
       maxConcurrentOperations: options.maxConcurrentOperations || 10,
       cacheEnabled: options.cacheEnabled !== false,
       cacheSize: options.cacheSize || 100,
+      // metadataCache holds one (now content-stripped) entry per stored
+      // snapshot and doubles as the in-memory index querySnapshots/
+      // cleanupSnapshots scan — so it can't be capped as tightly as
+      // snapshotCache without breaking query/retention correctness. This is
+      // a safety ceiling against truly unbounded growth, not a hot-cache size.
+      metadataCacheSize: options.metadataCacheSize || 10000,
       ...options
     };
     
@@ -123,37 +136,48 @@ export class SnapshotManager extends EventEmitter {
     
     // Cleanup timer
     this.cleanupTimer = null;
-    
-    this.initialize();
+
+    // Not started here — construction must stay synchronous and side-effect
+    // free (no directories touched, nothing to leave unhandled-rejected).
+    // ensureInitialized() lazily creates and memoizes this on first real use.
+    this._initPromise = null;
   }
-  
+
+  /**
+   * Lazily runs (and memoizes) initialize(), awaited by every public
+   * storage-touching method below. Replaces the old pattern of firing
+   * initialize() unawaited from the constructor, which could turn a
+   * directory-creation failure into an opaque unhandled rejection (emit()ing
+   * 'error' before any caller has had a chance to attach a listener).
+   */
+  async ensureInitialized() {
+    if (!this._initPromise) {
+      this._initPromise = this.initialize();
+    }
+    return this._initPromise;
+  }
+
   async initialize() {
-    try {
-      // Create storage directories
-      await this.createDirectories();
-      
-      // Load existing snapshot metadata
-      await this.loadMetadata();
-      
-      // Start cleanup timer if enabled
-      if (this.retentionPolicy.autoCleanup) {
-        this.startCleanupTimer();
-      }
-      
-      // Initialize cache
-      if (this.options.cacheEnabled) {
-        await this.initializeCache();
-      }
-      
-      this.emit('initialized', {
-        totalSnapshots: this.stats.totalSnapshots,
-        storageSize: this.stats.totalStorageSize
-      });
-      
-    } catch (error) {
-      this.emit('error', { operation: 'initialize', error: error.message });
-      throw error;
+    // Create storage directories
+    await this.createDirectories();
+
+    // Load existing snapshot metadata
+    await this.loadMetadata();
+
+    // Start cleanup timer if enabled
+    if (this.retentionPolicy.autoCleanup) {
+      this.startCleanupTimer();
+    }
+
+    // Initialize cache
+    if (this.options.cacheEnabled) {
+      await this.initializeCache();
     }
+
+    this.emit('initialized', {
+      totalSnapshots: this.stats.totalSnapshots,
+      storageSize: this.stats.totalStorageSize
+    });
   }
   
   /**
@@ -168,6 +192,8 @@ export class SnapshotManager extends EventEmitter {
     const operationId = this.generateOperationId();
 
     try {
+      await this.ensureInitialized();
+
       // Validate content is not null/undefined
       if (content === null || content === undefined) {
         throw new Error('Content cannot be null or undefined');
@@ -182,13 +208,7 @@ export class SnapshotManager extends EventEmitter {
 
       const snapshotId = this.generateSnapshotId(url, metadata.timestamp || Date.now());
       const contentHash = this.hashContent(content);
-      
-      // Check for similar existing snapshots for delta storage
-      let deltaInfo = null;
-      if (this.retentionPolicy.enableDeltaStorage) {
-        deltaInfo = await this.findSimilarSnapshot(url, contentHash, content);
-      }
-      
+
       // Prepare snapshot data
       const snapshot = {
         id: snapshotId,
@@ -217,28 +237,16 @@ export class SnapshotManager extends EventEmitter {
       
       let finalContent = content;
       let isCompressed = false;
-      let isDelta = false;
-      
-      // Apply delta storage if similar snapshot found (skip if base content is null, e.g. exact hash match)
-      if (deltaInfo && deltaInfo.content && deltaInfo.similarity > this.retentionPolicy.deltaThreshold) {
-        const deltaData = this.createDelta(deltaInfo.content, content);
-        if (deltaData.length < content.length * 0.7) { // Only use delta if it's significantly smaller
-          finalContent = deltaData;
-          isDelta = true;
-          
-          snapshot.delta = {
-            enabled: true,
-            baseSnapshotId: deltaInfo.snapshotId,
-            deltaData: deltaData,
-            deltaSize: deltaData.length
-          };
-          
-          this.stats.deltaSnapshots++;
-        }
-      }
-      
-      // Apply compression if enabled and above threshold
-      if (this.options.enableCompression && 
+      // Delta storage is intentionally disabled: createDelta() never stored real
+      // diff data, so retrieval silently returned the wrong (base) content.
+      // Every snapshot is now persisted in full (optionally gzip-compressed below).
+      const isDelta = false;
+
+      // Apply compression if enabled (per-call `options.enableCompression`
+      // overrides the instance default when explicitly provided) and above
+      // threshold.
+      const compressionEnabled = options.enableCompression ?? this.options.enableCompression;
+      if (compressionEnabled &&
           finalContent.length > this.retentionPolicy.compressionThreshold) {
         
         const compressed = await gzipAsync(finalContent);
@@ -316,12 +324,14 @@ export class SnapshotManager extends EventEmitter {
    */
   async retrieveSnapshot(snapshotId, options = {}) {
     const operationId = this.generateOperationId();
-    
+
     try {
-      this.activeOperations.set(operationId, { 
-        type: 'retrieve', 
-        snapshotId, 
-        startTime: Date.now() 
+      await this.ensureInitialized();
+
+      this.activeOperations.set(operationId, {
+        type: 'retrieve',
+        snapshotId,
+        startTime: Date.now()
       });
       
       // Check cache first
@@ -394,6 +404,8 @@ export class SnapshotManager extends EventEmitter {
    */
   async querySnapshots(query = {}) {
     try {
+      await this.ensureInitialized();
+
       const validated = QuerySchema.parse(query);
       
       // Load all metadata that matches URL filter
@@ -531,6 +543,8 @@ export class SnapshotManager extends EventEmitter {
     };
     
     try {
+      await this.ensureInitialized();
+
       for (const snapshotId of ids) {
         try {
           const metadata = await this.loadSnapshotMetadata(snapshotId);
@@ -579,8 +593,10 @@ export class SnapshotManager extends EventEmitter {
    */
   async cleanupSnapshots() {
     const startTime = Date.now();
-    
+
     try {
+      await this.ensureInitialized();
+
       const cleanupResults = {
         deletedCount: 0,
         freedSpace: 0,
@@ -698,29 +714,60 @@ export class SnapshotManager extends EventEmitter {
     await fs.unlink(filePath);
   }
   
-  async storeMetadata(snapshotId, metadata) {
+  /**
+   * Strips the full page `content` (and any legacy `delta.deltaData`) from a
+   * snapshot before it is persisted to a .meta file or cached in
+   * metadataCache — content lives only in the .snap file. Idempotent: safe
+   * to call on an object that's already stripped.
+   */
+  _stripHeavyFields(snapshot) {
+    const { content, ...rest } = snapshot;
+    if (rest.delta && rest.delta.deltaData !== undefined) {
+      const { deltaData, ...deltaRest } = rest.delta;
+      rest.delta = deltaRest;
+    }
+    return rest;
+  }
+
+  /**
+   * Bounded insert into metadataCache (same LRU-eviction shape as
+   * updateCache()/snapshotCache, sized separately via metadataCacheSize —
+   * see the constructor comment for why the two caches need different sizes).
+   */
+  _cacheMetadata(snapshotId, metadata) {
+    if (!this.metadataCache.has(snapshotId) && this.metadataCache.size >= this.options.metadataCacheSize) {
+      const oldestKey = this.metadataCache.keys().next().value;
+      this.metadataCache.delete(oldestKey);
+    }
+    this.metadataCache.set(snapshotId, metadata);
+  }
+
+  async storeMetadata(snapshotId, snapshot) {
     const filePath = path.join(this.options.metadataDir, `${snapshotId}.meta`);
+    const metadata = this._stripHeavyFields(snapshot);
     await fs.writeFile(filePath, JSON.stringify(metadata, null, 2), 'utf8');
-    
+
     // Update in-memory cache
-    this.metadataCache.set(snapshotId, metadata);
+    this._cacheMetadata(snapshotId, metadata);
   }
-  
+
   async loadSnapshotMetadata(snapshotId) {
     // Check cache first
     if (this.metadataCache.has(snapshotId)) {
       return this.metadataCache.get(snapshotId);
     }
-    
+
     // Load from disk
     try {
       const filePath = path.join(this.options.metadataDir, `${snapshotId}.meta`);
-      const content = await fs.readFile(filePath, 'utf8');
-      const metadata = JSON.parse(content);
-      
+      const raw = await fs.readFile(filePath, 'utf8');
+      // Defensive strip: .meta files written before this fix may still embed
+      // full page content — never let a legacy fat file re-pin it in memory.
+      const metadata = this._stripHeavyFields(JSON.parse(raw));
+
       // Cache it
-      this.metadataCache.set(snapshotId, metadata);
-      
+      this._cacheMetadata(snapshotId, metadata);
+
       return metadata;
     } catch (error) {
       return null;
@@ -780,85 +827,16 @@ export class SnapshotManager extends EventEmitter {
     return hash.digest('hex');
   }
   
-  async findSimilarSnapshot(url, contentHash, content) {
-    // Find recent snapshots for the same URL
-    const recentSnapshots = await this.querySnapshots({
-      url,
-      limit: 10,
-      sortBy: 'timestamp',
-      sortOrder: 'desc',
-      includeContent: false
-    });
-    
-    for (const snapshot of recentSnapshots.snapshots) {
-      if (snapshot.metadata.contentHash === contentHash) {
-        // Exact match
-        return {
-          snapshotId: snapshot.id,
-          similarity: 1.0,
-          content: null
-        };
-      }
-      
-      // Load content for similarity comparison
-      const fullSnapshot = await this.retrieveSnapshot(snapshot.id, { includeContent: true });
-      const similarity = this.calculateContentSimilarity(content, fullSnapshot.content);
-      
-      if (similarity > this.retentionPolicy.deltaThreshold) {
-        return {
-          snapshotId: snapshot.id,
-          similarity,
-          content: fullSnapshot.content
-        };
-      }
-    }
-    
-    return null;
-  }
-  
-  calculateContentSimilarity(content1, content2) {
-    // Simple similarity calculation based on content length difference
-    // In production, you might want to use more sophisticated algorithms
-    const len1 = content1.length;
-    const len2 = content2.length;
-    
-    if (len1 === 0 && len2 === 0) return 1.0;
-    if (len1 === 0 || len2 === 0) return 0.0;
-    
-    const lengthSimilarity = 1 - Math.abs(len1 - len2) / Math.max(len1, len2);
-    
-    // Additional similarity checks can be added here
-    // For example, using diff algorithms or content hashing
-    
-    return lengthSimilarity;
-  }
-  
-  createDelta(baseContent, currentContent) {
-    // Simple delta implementation - in production, consider using proper diff libraries
-    // This is a placeholder that would create a compressed diff
-    const deltaObject = {
-      type: 'diff',
-      base: baseContent.length,
-      current: currentContent.length,
-      // In a real implementation, you'd store the actual diff data
-      operations: [] // diff operations would go here
-    };
-    
-    return JSON.stringify(deltaObject);
-  }
-  
+  /**
+   * Legacy delta snapshots (written before delta storage was disabled — see
+   * storeSnapshot) stored only a byte-count stub with no real diff data, so
+   * their original content is unrecoverable. Fail loudly instead of silently
+   * returning the base snapshot's (wrong) content.
+   */
   applyDelta(baseContent, deltaData) {
-    try {
-      const delta = JSON.parse(deltaData);
-      
-      // In a real implementation, you'd apply the diff operations
-      // For now, return the base content as a fallback
-      return baseContent;
-    } catch (error) {
-      throw new Error(`Failed to apply delta: ${error.message}`);
-    }
+    throw new Error('Cannot reconstruct legacy delta snapshot: no diff data was stored for it. Re-fetch or re-create this snapshot from the source.');
   }
-  
+
   async calculateChangeMetrics(previousSnapshot, currentSnapshot) {
     // Calculate various change metrics between snapshots
     const metrics = {
@@ -908,16 +886,17 @@ export class SnapshotManager extends EventEmitter {
   }
   
   async initializeCache() {
-    // Pre-load recent snapshots into cache
-    const recentSnapshots = await this.querySnapshots({
-      limit: Math.min(this.options.cacheSize, 50),
-      sortBy: 'timestamp',
-      sortOrder: 'desc',
-      includeContent: false
-    });
-    
-    for (const snapshot of recentSnapshots.snapshots) {
-      this.metadataCache.set(snapshot.id, snapshot);
+    // Reads metadataCache directly (already fully populated by loadMetadata(),
+    // which runs earlier in initialize()) rather than going through
+    // querySnapshots() — querySnapshots() now awaits ensureInitialized(),
+    // which would deadlock against the in-flight initialize() call this
+    // method is itself called from.
+    const recent = Array.from(this.metadataCache.values())
+      .sort((a, b) => b.metadata.timestamp - a.metadata.timestamp)
+      .slice(0, Math.min(this.options.cacheSize, 50));
+
+    for (const snapshot of recent) {
+      this._cacheMetadata(snapshot.id, snapshot);
     }
   }
   
@@ -976,6 +955,12 @@ export class SnapshotManager extends EventEmitter {
       clearInterval(this.cleanupTimer);
     }
     
+    // .unref() so this timer never blocks process exit on its own — matches
+    // CacheManager's cleanupTimer/monitoringTimer. Without it, any process
+    // that merely imports a module holding a live SnapshotManager (e.g. the
+    // trackChangesTool singleton exported by trackChanges/index.js, which is
+    // never explicitly shut down) hangs forever, since a real server always
+    // has other things keeping it alive regardless.
     this.cleanupTimer = setInterval(async () => {
       try {
         await this.cleanupSnapshots();
@@ -983,8 +968,9 @@ export class SnapshotManager extends EventEmitter {
         this.emit('error', { operation: 'scheduledCleanup', error: error.message });
       }
     }, this.retentionPolicy.cleanupInterval);
+    if (typeof this.cleanupTimer.unref === 'function') this.cleanupTimer.unref();
   }
-  
+
   stopCleanupTimer() {
     if (this.cleanupTimer) {
       clearInterval(this.cleanupTimer);
@@ -1027,8 +1013,19 @@ export class SnapshotManager extends EventEmitter {
   }
   
   async shutdown() {
+    // initialize() is lazy (see ensureInitialized()) and only starts the
+    // cleanup timer partway through (after the async createDirectories/
+    // loadMetadata steps). If a caller triggered initialization (via any
+    // public method) and then immediately calls shutdown(), we must wait for
+    // that in-flight init to finish before stopping the timer — otherwise it
+    // could start AFTER stopCleanupTimer() already ran, leaking a live timer.
+    // If nothing ever triggered initialization, _initPromise is still null
+    // and there's nothing to wait for (no directories/timer were created).
+    if (this._initPromise) {
+      await this._initPromise.catch(() => {});
+    }
     this.stopCleanupTimer();
-    
+
     // Wait for active operations to complete
     const maxWaitTime = 30000; // 30 seconds
     const startTime = Date.now();

package/src/core/StealthBrowserManager.js

@@ -251,6 +251,26 @@ export class StealthBrowserManager {
       return this.browser;
     }
 
+    // Guard against concurrent callers both seeing this.browser === null and
+    // both launching a Chromium/Camoufox process — the second assignment to
+    // this.browser would overwrite the first, orphaning it. Callers that
+    // arrive while a launch is already in flight await the same promise.
+    if (this._launchPromise) {
+      return this._launchPromise;
+    }
+    this._launchPromise = this._doLaunchStealthBrowser(validatedConfig);
+    try {
+      return await this._launchPromise;
+    } finally {
+      this._launchPromise = null;
+    }
+  }
+
+  /**
+   * Actual browser launch, guarded by launchStealthBrowser's in-flight
+   * promise so only one launch can be in progress at a time.
+   */
+  async _doLaunchStealthBrowser(validatedConfig) {
     // C2: delegate to CamoufoxAdapter when engine === 'camoufox'
     if (validatedConfig.engine === 'camoufox') {
       const adapter = new CamoufoxAdapter();
@@ -2015,8 +2035,10 @@ export class CamoufoxAdapter extends BrowserEngine {
 // CRAWLFORGE_BROWSER_BACKEND=local  → LocalPlaywrightBackend (default, current behavior)
 // CRAWLFORGE_BROWSER_BACKEND=browserbase → BrowserBaseBackend via CDP
 //
-// Graceful fallback: if BrowserBaseBackend fails to connect (no API key, network error,
-// quota exceeded), StealthBrowserManager.getBrowserBackend() falls back to local.
+// Graceful fallback: resolveBrowserBackend() below falls back to LocalPlaywrightBackend
+// when CRAWLFORGE_BROWSER_BACKEND=browserbase but BROWSERBASE_API_KEY is unset.
+// NOTE: resolveBrowserBackend() is exported but not currently called anywhere in
+// StealthBrowserManager's own launch path — this backend is defined but unwired.
 
 /**
  * BrowserBackend interface (D3.4).
@@ -2101,7 +2123,7 @@ export class BrowserBaseBackend extends BrowserBackend {
 
     if (!sessionRes.ok) {
       const err = await sessionRes.text().catch(() => '');
-      throw new Error();
+      throw new Error(`BrowserBase session create failed: HTTP ${sessionRes.status} ${err}`);
     }
 
     const session = await sessionRes.json();

package/src/core/WebhookDispatcher.js

@@ -8,6 +8,7 @@ import { EventEmitter } from 'events';
 import { promises as fs } from 'fs';
 import path from 'path';
 import RetryManager from '../utils/RetryManager.js';
+import { safeFetch } from '../utils/ssrfGuard.js';
 
 export class WebhookDispatcher extends EventEmitter {
   constructor(options = {}) {
@@ -378,12 +379,6 @@ export class WebhookDispatcher extends EventEmitter {
     headers['X-Webhook-ID'] = event.id;
     headers['X-Webhook-Timestamp'] = event.timestamp.toString();
 
-    // Add HMAC signature if secret provided
-    if (config.signingSecret) {
-      const signature = this.generateSignature(event.payload, config.signingSecret);
-      headers['X-Webhook-Signature'] = signature;
-    }
-
     // Create request body
     const body = JSON.stringify({
       event: event.type,
@@ -393,17 +388,28 @@ export class WebhookDispatcher extends EventEmitter {
       metadata: event.metadata
     });
 
+    // Add HMAC signature if secret provided — sign the exact body being sent
+    // so receivers verifying over the raw request body get a matching digest.
+    if (config.signingSecret) {
+      const signature = this.generateSignature(body, config.signingSecret);
+      headers['X-Webhook-Signature'] = signature;
+    }
+
     // Execute with retry logic
     const result = await this.retryManager.execute(async () => {
-      const response = await fetch(event.url, {
+      const response = await safeFetch(event.url, {
         method: 'POST',
         headers,
         body,
-        timeout: config.timeout
+        // fetch/undici has no `timeout` RequestInit option — it was silently
+        // ignored, so a hung endpoint stalled the whole delivery queue.
+        signal: AbortSignal.timeout(config.timeout)
       });
 
       if (!response.ok) {
-        throw new Error('HTTP ' + response.status + ': ' + response.statusText);
+        const httpError = new Error('HTTP ' + response.status + ': ' + response.statusText);
+        httpError.response = { status: response.status };
+        throw httpError;
       }
 
       return response;
@@ -426,12 +432,11 @@ export class WebhookDispatcher extends EventEmitter {
 
   /**
    * Generate HMAC signature for webhook security
-   * @param {Object} payload - Webhook payload
+   * @param {string} body - Serialized request body (exact string being sent)
    * @param {string} secret - Signing secret
    * @returns {string} HMAC signature
    */
-  generateSignature(payload, secret) {
-    const body = JSON.stringify(payload);
+  generateSignature(body, secret) {
     const hmac = crypto.createHmac('sha256', secret);
     hmac.update(body);
     return 'sha256=' + hmac.digest('hex');
@@ -542,9 +547,9 @@ export class WebhookDispatcher extends EventEmitter {
 
     try {
       const startTime = Date.now();
-      const response = await fetch(url, {
+      const response = await safeFetch(url, {
         method: 'HEAD',
-        timeout: config.timeout / 2, // Use half timeout for health checks
+        signal: AbortSignal.timeout(config.timeout / 2), // Use half timeout for health checks
         headers: {
           'User-Agent': 'WebhookDispatcher-HealthCheck/1.0'
         }