crawlforge-mcp-server 3.0.17 → 3.3.1

package/src/tools/search/ranking/SearchResultCache.js

@@ -0,0 +1,52 @@
+/**
+ * SearchResultCache — unified cache layer for search ranking and deduplication.
+ *
+ * Both ResultRanker and ResultDeduplicator previously held separate CacheManager
+ * instances with identical TTL configuration. This module provides a single
+ * shared cache they can both use, halving the number of LRU cache instances
+ * created per SearchWebTool instantiation.
+ *
+ * Usage:
+ *   const cache = new SearchResultCache({ ttl: 3600000 });
+ *   // pass to ResultRanker and ResultDeduplicator via options.sharedCache
+ */
+
+import { CacheManager } from '../../../core/cache/CacheManager.js';
+
+export class SearchResultCache {
+  /**
+   * @param {Object} [options]
+   * @param {number} [options.ttl=3600000]  — cache TTL in milliseconds
+   * @param {boolean} [options.enabled=true] — disable to skip caching
+   */
+  constructor(options = {}) {
+    const { ttl = 3600000, enabled = true } = options;
+    this.enabled = enabled;
+    this._cache = enabled ? new CacheManager({ ttl }) : null;
+  }
+
+  /** Retrieve a cached value by key (returns undefined on miss or when disabled). */
+  async get(key) {
+    if (!this.enabled || !this._cache) return undefined;
+    return this._cache.get(key);
+  }
+
+  /** Store a value under the given key. */
+  async set(key, value) {
+    if (!this.enabled || !this._cache) return;
+    return this._cache.set(key, value);
+  }
+
+  /** Generate a deterministic cache key from an arbitrary descriptor object. */
+  generateKey(namespace, descriptor) {
+    if (!this._cache) return null;
+    return this._cache.generateKey(namespace, descriptor);
+  }
+
+  /** Return underlying cache stats (or null when disabled). */
+  getStats() {
+    return this._cache ? this._cache.getStats() : null;
+  }
+}
+
+export default SearchResultCache;

package/src/tools/search/searchWeb.js

@@ -4,11 +4,14 @@ import { CacheManager } from '../../core/cache/CacheManager.js';
 import { QueryExpander } from './queryExpander.js';
 import { ResultRanker } from './ranking/ResultRanker.js';
 import { ResultDeduplicator } from './ranking/ResultDeduplicator.js';
+import { SearchResultCache } from './ranking/SearchResultCache.js';
 import LocalizationManager from '../../core/LocalizationManager.js';
 import { isCreatorModeVerified } from '../../core/creatorMode.js';
+import { searchViaSearxng } from './providers/searxng.js';
 
 const SearchWebSchema = z.object({
   query: z.string().min(1),
+  provider: z.enum(['crawlforge', 'searxng']).optional().default('crawlforge'),
   limit: z.number().min(1).max(100).optional().default(10),
   offset: z.number().min(0).optional().default(0),
   lang: z.string().optional().default('en'),
@@ -92,13 +95,16 @@ export class SearchWebTool {
     }
 
     this.cache = cacheEnabled ? new CacheManager({ ttl: cacheTTL }) : null;
-    
+
     // Initialize query expander
     this.queryExpander = new QueryExpander(expanderOptions);
-    
-    // Initialize ranking and deduplication systems
-    this.resultRanker = new ResultRanker({ cacheEnabled, cacheTTL, ...rankingOptions });
-    this.resultDeduplicator = new ResultDeduplicator({ cacheEnabled, cacheTTL, ...deduplicationOptions });
+
+    // Shared cache for ranking + deduplication — avoids two separate LRU instances
+    const sharedRankingCache = new SearchResultCache({ ttl: cacheTTL, enabled: cacheEnabled });
+
+    // Initialize ranking and deduplication systems (both share the same cache)
+    this.resultRanker = new ResultRanker({ cacheEnabled, cacheTTL, sharedCache: sharedRankingCache, ...rankingOptions });
+    this.resultDeduplicator = new ResultDeduplicator({ cacheEnabled, cacheTTL, sharedCache: sharedRankingCache, ...deduplicationOptions });
     
     // Initialize localization manager
     this.localizationManager = new LocalizationManager({
@@ -110,7 +116,13 @@ export class SearchWebTool {
   async execute(params) {
     try {
       const validated = SearchWebSchema.parse(params);
-      
+
+      // --- SearXNG provider short-circuit ---
+      if (validated.provider === 'searxng') {
+        return await this._executeViaSearxng(validated);
+      }
+      // --- end SearXNG short-circuit ---
+
       // Apply localization if specified
       let localizedParams = validated;
       if (validated.localization) {
@@ -336,6 +348,100 @@ export class SearchWebTool {
     }
   }
 
+  /**
+   * Execute search via a self-hosted SearXNG instance.
+   * Results are normalised to the same shape as the CrawlForge/Google path.
+   *
+   * @param {Object} validated - Parsed & validated parameters from SearchWebSchema
+   * @returns {Promise<Object>} Standard search_web response object
+   */
+  async _executeViaSearxng(validated) {
+    // page is 1-based; offset is 0-based items, so map via limit
+    const page = Math.floor(validated.offset / validated.limit) + 1;
+
+    const adapterResult = await searchViaSearxng({
+      query: validated.query,
+      limit: validated.limit,
+      page,
+      safeSearch: validated.safe_search,
+      language: validated.lang
+    });
+
+    // Run through shared post-processing (deduplication, ranking)
+    let processedResults = await this.processResults(adapterResult);
+
+    let deduplicationInfo = null;
+    if (validated.enable_deduplication && processedResults.length > 1) {
+      const dedupeOptions = validated.deduplication_thresholds
+        ? { thresholds: validated.deduplication_thresholds }
+        : {};
+      const originalCount = processedResults.length;
+      processedResults = await this.resultDeduplicator.deduplicateResults(
+        processedResults,
+        dedupeOptions
+      );
+      deduplicationInfo = {
+        originalCount,
+        finalCount: processedResults.length,
+        duplicatesRemoved: originalCount - processedResults.length,
+        deduplicationRate:
+          ((originalCount - processedResults.length) / originalCount * 100).toFixed(1) + '%'
+      };
+    }
+
+    let rankingInfo = null;
+    if (validated.enable_ranking && processedResults.length > 1) {
+      const rankingOptions = validated.ranking_weights
+        ? { weights: validated.ranking_weights }
+        : {};
+      processedResults = await this.resultRanker.rankResults(
+        processedResults,
+        validated.query,
+        rankingOptions
+      );
+      rankingInfo = {
+        algorithmsUsed: ['bm25', 'semantic', 'authority', 'freshness'],
+        weightsApplied: this.resultRanker.options.weights,
+        totalResults: processedResults.length
+      };
+    }
+
+    if (!validated.include_ranking_details) {
+      processedResults = processedResults.map(({ rankingDetails, ...r }) => r);
+    }
+    if (!validated.include_deduplication_details) {
+      processedResults = processedResults.map(({ deduplicationInfo: _d, ...r }) => r);
+    }
+
+    return {
+      query: validated.query,
+      results: processedResults,
+      total_results: adapterResult.searchInformation?.totalResults || 0,
+      search_time: adapterResult.searchInformation?.searchTime || 0,
+      offset: validated.offset,
+      limit: validated.limit,
+      cached: false,
+      provider: {
+        name: 'searxng',
+        backend: 'SearXNG (self-hosted)',
+        instanceUrl: process.env.CRAWLFORGE_SEARXNG_URL || null,
+        capabilities: {
+          requiresApiKey: false,
+          supportsPagination: true,
+          supportsLanguageFilter: true,
+          supportsSafeSearch: true
+        }
+      },
+      localization: null,
+      processing: {
+        ranking: rankingInfo,
+        deduplication: deduplicationInfo,
+        query_expansion: null,
+        localization_applied: false
+      }
+    };
+  }
+
   async processResults(searchResults) {
     if (!searchResults.items || searchResults.items.length === 0) {
       return [];

package/src/tools/tracking/trackChanges/differ.js

@@ -0,0 +1,98 @@
+/**
+ * TrackChanges — differ module.
+ * URL content fetching and history/stat helper functions.
+ */
+
+/**
+ * Fetch the HTML/text content of a URL with change-tracking headers.
+ * @param {string} url
+ * @returns {Promise<{ content: string, metadata: Object }>}
+ */
+export async function fetchContent(url) {
+  try {
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate',
+        'Cache-Control': 'no-cache'
+      },
+      timeout: 30000
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const content = await response.text();
+
+    return {
+      content,
+      metadata: {
+        statusCode: response.status,
+        contentType: response.headers.get('content-type'),
+        contentLength: content.length,
+        lastModified: response.headers.get('last-modified'),
+        etag: response.headers.get('etag'),
+        fetchedAt: Date.now()
+      }
+    };
+  } catch (error) {
+    throw new Error(`Failed to fetch content: ${error.message}`);
+  }
+}
+
+/**
+ * Merge change-tracker history entries with snapshot history entries.
+ * Deduplicates by timestamp proximity (within 60 s).
+ */
+export function mergeHistoryData(changeHistory, snapshotHistory) {
+  const merged = [];
+
+  changeHistory.forEach(entry => {
+    merged.push({ ...entry, source: 'change_tracker', hasSnapshot: false });
+  });
+
+  snapshotHistory.forEach(entry => {
+    const existing = merged.find(m => Math.abs(m.timestamp - entry.timestamp) < 60000);
+    if (existing) {
+      existing.hasSnapshot = true;
+      existing.snapshotId = entry.snapshotId;
+    } else {
+      merged.push({ ...entry, source: 'snapshot', hasSnapshot: true });
+    }
+  });
+
+  return merged.sort((a, b) => b.timestamp - a.timestamp);
+}
+
+/** Return true if entry.significance is at or above the filter level. */
+export function matchesSignificanceFilter(entry, filter) {
+  const levels = ['none', 'minor', 'moderate', 'major', 'critical'];
+  return levels.indexOf(entry.significance || 'none') >= levels.indexOf(filter);
+}
+
+/** Return true if significance meets the notification threshold. */
+export function meetsNotificationThreshold(significance, threshold) {
+  const levels = ['none', 'minor', 'moderate', 'major', 'critical'];
+  return levels.indexOf(significance) >= levels.indexOf(threshold);
+}
+
+export function calculateAverageInterval(changeHistory) {
+  if (changeHistory.length < 2) return null;
+  let total = 0;
+  for (let i = 1; i < changeHistory.length; i++) {
+    total += changeHistory[i - 1].timestamp - changeHistory[i].timestamp;
+  }
+  return total / (changeHistory.length - 1);
+}
+
+export function calculateSignificanceDistribution(changeHistory) {
+  const dist = { none: 0, minor: 0, moderate: 0, major: 0, critical: 0 };
+  changeHistory.forEach(entry => {
+    const sig = entry.significance || 'none';
+    if (Object.prototype.hasOwnProperty.call(dist, sig)) dist[sig]++;
+  });
+  return dist;
+}