crawlforge-mcp-server 5.1.0 → 5.2.0

package/src/core/ChangeTracker.js

@@ -48,6 +48,60 @@ const ChangeComparisonSchema = z.object({
 
 const ChangeSignificance = z.enum(['none', 'minor', 'moderate', 'major', 'critical']);
 
+// Bounds that keep a compare response usable. An unscoped Amazon product page
+// produced a 5.6MB payload — 4MB of it a single line_diff holding the entire
+// document twice — which overflows the MCP response limit on every comparison.
+const MAX_DIFF_ENTRIES = 200;
+const MAX_DIFF_VALUE_CHARS = 2000;
+
+// Significance ordering, used to raise a level without ever lowering it.
+const SIGNIFICANCE_ORDER = ['none', 'minor', 'moderate', 'major', 'critical'];
+
+/**
+ * A monetary amount carries meaning that its size on the page does not.
+ * Significance is otherwise purely volumetric — how much of the document
+ * changed — so a price is scored by how many characters it occupies. Tracking
+ * a price block, a rise from $19.99 to $99.99 scored "minor", below the default
+ * "moderate" notification threshold; untracked, the same change did not
+ * register as a change at all.
+ *
+ * Only currency-tagged numbers count. Treating every number this way would fire
+ * on view counters, timestamps and review totals, which is the opposite failure.
+ */
+const MONETARY_PATTERN =
+  /[$£€¥₹]\s?\d[\d,]*(?:\.\d{1,2})?|\b\d[\d,]*(?:\.\d{1,2})?\s?(?:USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|INR)\b/gi;
+
+/** A price change at or above this fraction is major rather than moderate. */
+const MAJOR_VALUE_CHANGE = 0.2;
+
+/**
+ * Parse the numeric amount out of a matched monetary string.
+ * Commas are read as thousands separators; a European decimal comma is
+ * ambiguous here and is not guessed at, so such a value simply reads as
+ * changed rather than being scored by magnitude.
+ * @param {string} raw
+ * @returns {number|null}
+ */
+function parseMonetaryAmount(raw) {
+  const amount = Number.parseFloat(raw.replace(/[^\d.,]/g, '').replace(/,/g, ''));
+  return Number.isFinite(amount) ? amount : null;
+}
+
+/**
+ * Monetary amounts in document order.
+ * @param {string} text
+ * @returns {Array<{raw: string, amount: number}>}
+ */
+function extractMonetaryValues(text) {
+  if (!text) return [];
+  const values = [];
+  for (const match of String(text).matchAll(MONETARY_PATTERN)) {
+    const amount = parseMonetaryAmount(match[0]);
+    if (amount !== null) values.push({ raw: match[0].trim(), amount });
+  }
+  return values;
+}
+
 export class ChangeTracker extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -156,7 +210,8 @@ export class ChangeTracker extends EventEmitter {
         contentHash: contentAnalysis.hashes.page,
         sections: Object.keys(contentAnalysis.hashes.sections).length,
         elements: Object.keys(contentAnalysis.hashes.elements).length,
-        createdAt: baseline.timestamp
+        createdAt: baseline.timestamp,
+        ...(contentAnalysis.warnings ? { warnings: contentAnalysis.warnings } : {})
       };
       
     } catch (error) {
@@ -294,13 +349,36 @@ export class ChangeTracker extends EventEmitter {
     
     try {
       // Parse HTML if available
-      const $ = load(content);
-      
+      let $ = load(content);
+
       // Remove excluded elements
       options.excludeSelectors?.forEach(selector => {
         $(selector).remove();
       });
-      
+
+      // Narrow the working document to customSelectors so hashing, similarity
+      // and text diffs all operate on the same subtree. Previously these
+      // selectors only added extra section hashes while every comparison still
+      // ran over the whole page, so document-level churn (session tokens,
+      // CSP nonces, rotating ad ids) registered as changes no matter how
+      // tightly the caller scoped.
+      if (options.customSelectors?.length) {
+        const scoped = options.customSelectors
+          .flatMap(selector => $(selector).toArray().map(element => $.html(element)))
+          .join('\n');
+
+        if (scoped) {
+          $ = load(scoped);
+          analysis.originalContent = scoped;
+        } else {
+          // Falling back to the full document keeps a bad selector from
+          // silently tracking nothing, but the caller needs to know.
+          analysis.warnings = [
+            `customSelectors matched no elements (${options.customSelectors.join(', ')}); tracked the full document instead`
+          ];
+        }
+      }
+
       // Analyze at different granularities
       switch (options.granularity) {
         case 'element':
@@ -324,8 +402,8 @@ export class ChangeTracker extends EventEmitter {
       // Extract metadata
       analysis.metadata = this.extractMetadata($, options);
       
-      // Calculate statistics
-      analysis.statistics = this.calculateContentStatistics(content, $);
+      // Calculate statistics over the scoped content, matching what is hashed
+      analysis.statistics = this.calculateContentStatistics(analysis.originalContent, $);
       
     } catch (error) {
       // Fallback to plain text analysis
@@ -350,7 +428,10 @@ export class ChangeTracker extends EventEmitter {
   async detectChanges(baseline, current, options = {}) {
     const changes = {
       similarity: 0,
-      structuralSimilarity: 0,
+      // null rather than 0: a structural score is only produced when
+      // trackStructure is on, and 0 is a real score meaning "the structure
+      // changed completely".
+      structuralSimilarity: null,
       addedElements: [],
       removedElements: [],
       modifiedElements: [],
@@ -413,6 +494,15 @@ export class ChangeTracker extends EventEmitter {
       );
     }
     
+    // Detect monetary value changes. Scored by magnitude rather than by how
+    // much of the page they occupy, so a price change is not diluted away.
+    if (options.trackText !== false) {
+      changes.valueChanges = this.detectValueChanges(
+        baseline.originalContent,
+        current.originalContent
+      );
+    }
+
     // Detect link changes
     if (options.trackLinks) {
       changes.linkChanges = this.detectLinkChanges(
@@ -432,6 +522,49 @@ export class ChangeTracker extends EventEmitter {
     return changes;
   }
   
+  /**
+   * Compare the monetary amounts in two versions of the tracked content.
+   *
+   * Amounts are paired in document order. When the two versions hold different
+   * counts the set of prices itself changed (an item sold out, a sale price
+   * appeared), which is reported as a change even though no single pair can be
+   * measured.
+   *
+   * @param {string} baselineText
+   * @param {string} currentText
+   * @returns {{changes: Array, countChanged: boolean, maxRelativeChange: number}|null}
+   *   null when no monetary value changed
+   */
+  detectValueChanges(baselineText, currentText) {
+    const before = extractMonetaryValues(baselineText);
+    const after = extractMonetaryValues(currentText);
+    if (before.length === 0 && after.length === 0) return null;
+
+    const changes = [];
+    const pairs = Math.min(before.length, after.length);
+    for (let i = 0; i < pairs; i++) {
+      if (before[i].amount === after[i].amount) continue;
+      const base = Math.abs(before[i].amount);
+      const relativeChange = base > 0
+        ? Math.abs(after[i].amount - before[i].amount) / base
+        : 1;
+      changes.push({
+        before: before[i].raw,
+        after: after[i].raw,
+        relativeChange: Math.round(relativeChange * 1000) / 1000
+      });
+    }
+
+    const countChanged = before.length !== after.length;
+    if (changes.length === 0 && !countChanged) return null;
+
+    return {
+      changes: changes.slice(0, MAX_DIFF_ENTRIES),
+      countChanged,
+      maxRelativeChange: changes.reduce((max, c) => Math.max(max, c.relativeChange), 0)
+    };
+  }
+
   /**
    * Calculate change significance score
    * @param {Object} changeAnalysis - Change analysis results
@@ -482,17 +615,33 @@ export class ChangeTracker extends EventEmitter {
     }
     
     // Determine significance level
+    let level;
     if (significanceScore < thresholds.minor) {
-      return 'none';
+      level = 'none';
     } else if (significanceScore < thresholds.moderate) {
-      return 'minor';
+      level = 'minor';
     } else if (significanceScore < thresholds.major) {
-      return 'moderate';
+      level = 'moderate';
     } else if (significanceScore < 0.9) {
-      return 'major';
+      level = 'major';
     } else {
-      return 'critical';
+      level = 'critical';
     }
+
+    // The score above measures how much of the page changed. A price change is
+    // significant because of what it is, not how many characters it takes up,
+    // so a monetary change raises the level to at least "moderate" — the
+    // default notification threshold, which it previously fell below. This only
+    // ever raises the level; a large structural change stays major.
+    const valueChanges = changeAnalysis.valueChanges;
+    if (valueChanges) {
+      const floor = valueChanges.maxRelativeChange >= MAJOR_VALUE_CHANGE ? 'major' : 'moderate';
+      if (SIGNIFICANCE_ORDER.indexOf(floor) > SIGNIFICANCE_ORDER.indexOf(level)) {
+        level = floor;
+      }
+    }
+
+    return level;
   }
   
   // Content Analysis Methods
@@ -752,21 +901,65 @@ export class ChangeTracker extends EventEmitter {
     if (wordDiffChanges.length > 0) {
       textChanges.push({
         type: 'word_diff',
-        changes: wordDiffChanges
+        changes: this.capDiffPayload(wordDiffChanges)
       });
     }
-    
-    // Line-level diff for structured content
-    const lineDiff = diffLines(baselineContent, currentContent);
-    if (lineDiff.some(part => part.added || part.removed)) {
-      textChanges.push({
-        type: 'line_diff',
-        changes: lineDiff.filter(part => part.added || part.removed)
-      });
+
+    // Line-level diff for structured content. With ignoreWhitespace (the
+    // default) the whole document collapses onto a single line, so diffLines
+    // degenerates into "remove everything, add everything" — a payload twice
+    // the page size describing what word_diff already pinpointed. Only run it
+    // when the content genuinely has line structure.
+    const hasLineStructure = baselineContent.includes('\n') || currentContent.includes('\n');
+    if (hasLineStructure) {
+      const lineDiff = diffLines(baselineContent, currentContent);
+      const lineDiffChanges = lineDiff.filter(part => part.added || part.removed);
+      if (lineDiffChanges.length > 0) {
+        textChanges.push({
+          type: 'line_diff',
+          changes: this.capDiffPayload(lineDiffChanges)
+        });
+      }
     }
-    
+
     return textChanges;
   }
+
+  /**
+   * Bound a diff payload so a large page cannot produce a multi-megabyte
+   * response. Keeps the first maxEntries changes, truncates any oversized
+   * value, and appends a marker describing what was dropped so callers never
+   * mistake a truncated diff for a complete one.
+   * @param {Array} changes - Diff parts from diffWords/diffLines
+   * @param {Object} limits - Optional maxEntries / maxValueChars overrides
+   * @returns {Array} - Bounded diff parts
+   */
+  capDiffPayload(changes, limits = {}) {
+    const maxEntries = limits.maxEntries ?? MAX_DIFF_ENTRIES;
+    const maxValueChars = limits.maxValueChars ?? MAX_DIFF_VALUE_CHARS;
+
+    const capped = changes.slice(0, maxEntries).map(part => {
+      if (typeof part.value === 'string' && part.value.length > maxValueChars) {
+        return {
+          ...part,
+          value: part.value.slice(0, maxValueChars),
+          truncated: true,
+          omittedChars: part.value.length - maxValueChars
+        };
+      }
+      return part;
+    });
+
+    const omittedEntries = changes.length - capped.length;
+    if (omittedEntries > 0) {
+      capped.push({
+        omittedEntries,
+        note: `${omittedEntries} further changes omitted; scope the comparison with customSelectors to see them`
+      });
+    }
+
+    return capped;
+  }
   
   detectLinkChanges(baselineLinks, currentLinks) {
     const changes = {

package/src/core/ResearchOrchestrator.js

@@ -91,8 +91,9 @@ export class ResearchOrchestrator extends EventEmitter {
     
     // Initialize LLM Manager for AI-powered research
     this.llmManager = new LLMManager(options.llmConfig || {});
+    // Provisional: conductResearch() re-resolves this with an async probe.
     this.enableLLMFeatures = this.llmManager.isAvailable();
-    
+
     if (this.enableLLMFeatures) {
       this.logger.info('LLM-powered research features enabled');
     } else {
@@ -146,10 +147,15 @@ export class ResearchOrchestrator extends EventEmitter {
     const startTime = Date.now();
     
     this.initializeResearchSession(sessionId, topic, startTime);
-    
+
+    // Settle LLM availability before the first LLM-gated stage. The constructor
+    // can only read it synchronously, and Ollama — which needs no API key — can
+    // only be confirmed by a probe.
+    this.enableLLMFeatures = await this.llmManager.ready();
+
     try {
       this.logger.info('Starting deep research', { sessionId, topic, options });
-      
+
       // Stage 1: Initial topic exploration and query expansion
       const expandedQueries = await this.expandResearchTopic(topic, options);
       this.researchState.currentDepth = 1;

package/src/core/SamplingClient.js

@@ -11,8 +11,7 @@
  *   4. Error
  */
 
-const OLLAMA_DEFAULT_MODEL = 'llama3.2';
-const OLLAMA_BASE_URL = () => (process.env.OLLAMA_BASE_URL || 'http://localhost:11434').replace(/\/$/, '');
+import { ollamaBaseUrl as OLLAMA_BASE_URL, ollamaHeaders, selectOllamaModel } from '../utils/ollamaConfig.js';
 
 /**
  * Attempt an Ollama completion.
@@ -21,11 +20,11 @@ const OLLAMA_BASE_URL = () => (process.env.OLLAMA_BASE_URL || 'http://localhost:
  * @returns {Promise<string>}
  */
 async function tryOllama(prompt, { model, maxTokens } = {}) {
-  const ollamaModel = model || process.env.OLLAMA_DEFAULT_MODEL || OLLAMA_DEFAULT_MODEL;
+  const ollamaModel = model || await selectOllamaModel();
   const url = `${OLLAMA_BASE_URL()}/api/generate`;
   const res = await fetch(url, {
     method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
+    headers: ollamaHeaders({ 'Content-Type': 'application/json' }),
     body: JSON.stringify({
       model: ollamaModel,
       prompt,
@@ -178,7 +177,7 @@ export class SamplingClient {
     const result = { ollama: false, openai: false, anthropic: false, sampling: false };
 
     try {
-      const res = await fetch(`${OLLAMA_BASE_URL()}/api/tags`, { signal: AbortSignal.timeout(3000) });
+      const res = await fetch(`${OLLAMA_BASE_URL()}/api/tags`, { headers: ollamaHeaders(), signal: AbortSignal.timeout(3000) });
       result.ollama = res.ok;
     } catch (_) { /* unavailable */ }
 

package/src/core/StealthBrowserManager.js

@@ -64,15 +64,7 @@ export class StealthBrowserManager {
   constructor(options = {}) {
     this.browser = null;
     this._maxContexts = parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10);
-    this.contexts = new BrowserContextPool({
-      maxContexts: this._maxContexts,
-      periodicRefreshAfter: 200,
-      closeIdleAfterMs: 30 * 60 * 1000,
-      waitTimeoutMs: 10_000,
-      onContextExpired: (contextId) => {
-        this.fingerprints.delete(contextId);
-      }
-    });
+    this.contexts = this._createContextPool();
     // D2.2: fingerprints Map is capped at _maxContexts to prevent unbounded growth.
     // Oldest entries are evicted when the cap is exceeded (insertion order via Map).
     this.fingerprints = new Map();
@@ -234,6 +226,22 @@ export class StealthBrowserManager {
     ];
   }
 
+  /**
+   * Build the context pool. Also used by cleanup(): the pool's destroy()
+   * permanently stops its idle timer, so a destroyed pool must be replaced.
+   */
+  _createContextPool() {
+    return new BrowserContextPool({
+      maxContexts: this._maxContexts,
+      periodicRefreshAfter: 200,
+      closeIdleAfterMs: 30 * 60 * 1000,
+      waitTimeoutMs: 10_000,
+      onContextExpired: (contextId) => {
+        this.fingerprints.delete(contextId);
+      }
+    });
+  }
+
   /**
    * Launch stealth browser with anti-detection configurations.
    * C2: honours config.engine — 'chromium' (default) or 'camoufox' (Firefox-based).
@@ -241,6 +249,12 @@ export class StealthBrowserManager {
   async launchStealthBrowser(config = {}) {
     const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
 
+    // A Chromium that was OOM-killed or crashed doesn't error on reuse — its
+    // protocol calls hang. Detect the corpse and relaunch instead.
+    if (this.browser && !this.browser.isConnected()) {
+      this.browser = null;
+    }
+
     // C2: if the requested engine differs from the running browser, tear it down first.
     if (this.browser && this._launchedEngine && this._launchedEngine !== validatedConfig.engine) {
       await this.browser.close().catch(() => {});
@@ -368,8 +382,11 @@ export class StealthBrowserManager {
       stealthArgs.push(`--proxy-server=${currentProxy}`);
     }
 
-    this.browser = await chromium.launch({
+    const browser = await chromium.launch({
       headless: true,
+      // Hosted images set this to their system Chromium (Playwright itself
+      // never reads it) — see Dockerfile.
+      executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
       args: stealthArgs,
       ignoreDefaultArgs: [
         '--enable-blink-features=IdleDetection',
@@ -377,6 +394,16 @@ export class StealthBrowserManager {
       ]
     });
 
+    // If this Chromium dies (OOM kill, crash), drop the handle so the next
+    // call relaunches instead of reusing a corpse. The identity guard keeps a
+    // late event from an old instance from nulling a newer one.
+    browser.on('disconnected', () => {
+      if (this.browser === browser) {
+        this.browser = null;
+      }
+    });
+    this.browser = browser;
+
     return this.browser;
   }
 
@@ -1864,8 +1891,20 @@ export class StealthBrowserManager {
    * Close all contexts and browser
    */
   async cleanup() {
-    // Close all contexts via pool (handles idle timer cleanup + wait queue drain)
-    await this.contexts.destroy();
+    // A wedged Chromium doesn't error on close() — it hangs. Race each close
+    // against a short deadline so cleanup always finishes inside callers'
+    // timeout windows and works as a remote unwedge lever.
+    const withDeadline = (promise, ms) =>
+      Promise.race([
+        promise.then(() => true, () => true),
+        new Promise((resolve) => setTimeout(() => resolve(false), ms))
+      ]);
+
+    // Close all contexts via pool (handles idle timer cleanup + wait queue
+    // drain). destroy() permanently stops the pool's idle timer, so recreate
+    // the pool afterwards or idle reaping is dead for the process lifetime.
+    await withDeadline(this.contexts.destroy(), 5000);
+    this.contexts = this._createContextPool();
     this.fingerprints.clear();
 
     // Reset human behavior simulator
@@ -1874,14 +1913,18 @@ export class StealthBrowserManager {
       this.humanBehaviorSimulator = null;
     }
 
-    // Close browser
+    // Close browser; if close hangs, kill the process so the OS reclaims it.
     if (this.browser) {
-      try {
-        await this.browser.close();
-      } catch (error) {
-        console.warn('Failed to close browser:', error.message);
-      }
+      const browser = this.browser;
       this.browser = null;
+      const closed = await withDeadline(browser.close(), 5000);
+      if (!closed) {
+        try {
+          browser.process()?.kill('SIGKILL');
+        } catch {
+          // Process already gone.
+        }
+      }
     }
   }
 
@@ -2128,6 +2171,9 @@ export class LocalPlaywrightBackend extends BrowserBackend {
     const { chromium } = await import('playwright');
     return chromium.launch({
       headless: config.headless !== false,
+      // Hosted images set this to their system Chromium (Playwright itself
+      // never reads it) — see Dockerfile.
+      executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
       ...config.launchOptions
     });
   }

package/src/core/cache/CacheManager.js

@@ -3,6 +3,7 @@ import { promises as fs } from 'fs';
 import path from 'path';
 import crypto from 'crypto';
 import { EventEmitter } from 'events';
+import { config } from '../../constants/config.js';
 
 export class CacheManager extends EventEmitter {
   constructor(options = {}) {
@@ -11,8 +12,12 @@ export class CacheManager extends EventEmitter {
     const {
       maxSize = 1000,
       ttl = 3600000, // 1 hour default
-      diskCacheDir = './cache',
-      enableDiskCache = true,
+      // CACHE_DIR and CACHE_ENABLE_DISK have always been documented knobs in
+      // constants/config.js, but nothing read them: the disk cache was hard-
+      // wired to ./cache and always on. Defaults are unchanged ('./cache',
+      // true) — the env vars now actually do what they say.
+      diskCacheDir = config.performance.cacheDir,
+      enableDiskCache = config.performance.cacheEnableDisk,
       enableCacheWarming = false,
       warmingBatchSize = 10,
       enableMonitoring = true,

package/src/core/crawlers/BFSCrawler.js

@@ -24,7 +24,8 @@ export class BFSCrawler {
       domainFilter = null,
       enableLinkAnalysis = true,
       linkAnalyzerOptions = {},
-      sessionContext = null
+      sessionContext = null,
+      cacheEnabled = true
     } = options;
 
     this.maxDepth = maxDepth;
@@ -46,7 +47,14 @@ export class BFSCrawler {
     this.linkAnalyzer = enableLinkAnalysis ? new LinkAnalyzer(linkAnalyzerOptions) : null;
     
     this.queue = new QueueManager({ concurrency, timeout });
-    this.cache = new CacheManager({ ttl: 3600000 }); // 1 hour cache
+    // Page-body cache for this crawler only: it is destroyed with the
+    // instance, so it stays in memory. Persisting it made bodies outlive the
+    // crawl by an hour and cross process boundaries, which is neither what
+    // destroy() below implies nor what a caller passing cacheEnabled:false
+    // could switch off.
+    this.cache = cacheEnabled
+      ? new CacheManager({ ttl: 3600000, enableDiskCache: false })
+      : null;
     // C1: per-domain rate-limiter map — reuse existing limiter when
     // effectiveRateLimit hasn't changed, rather than recreating it on every URL.
     this.rateLimiter = new RateLimiter({ requestsPerSecond: 10 });
@@ -179,8 +187,8 @@ export class BFSCrawler {
 
     try {
       // Check cache first
-      const cacheKey = this.cache.generateKey(normalizedUrl);
-      let pageData = await this.cache.get(cacheKey);
+      const cacheKey = this.cache ? this.cache.generateKey(normalizedUrl) : null;
+      let pageData = cacheKey ? await this.cache.get(cacheKey) : null;
 
       if (!pageData) {
         // Apply domain-specific rate limiting
@@ -205,7 +213,7 @@ export class BFSCrawler {
         pageData = await this.fetchPage(normalizedUrl);
         
         // Cache the result
-        await this.cache.set(cacheKey, pageData);
+        if (cacheKey) await this.cache.set(cacheKey, pageData);
       }
 
       // Process links for analysis
@@ -425,7 +433,7 @@ export class BFSCrawler {
       visited: this.visited.size,
       results: this.results.length,
       errors: this.errors.length,
-      cacheStats: this.cache.getStats(),
+      cacheStats: this.cache ? this.cache.getStats() : null,
       queueStats: this.queue.getStats(),
       rateLimitStats: this.rateLimiter.getStats(),
       domainFilterStats: filterStats,