crawlforge-mcp-server 3.0.1 → 3.0.3

package/src/core/ChangeTracker.js

@@ -1,10 +1,8 @@
 import crypto from "crypto";
 /**
- * ChangeTracker - Enhanced Content Change Detection and Analysis (Phase 2.4)
+ * ChangeTracker - Content Change Detection and Analysis
  * Implements hierarchical content hashing (page → sections → elements)
- * with differential comparison engine, change significance scoring,
- * scheduled monitoring, advanced comparison engine, alert system,
- * and historical analysis capabilities
+ * with differential comparison engine and change significance scoring
  */
 
 import { createHash } from 'crypto';
@@ -12,10 +10,6 @@ import { z } from 'zod';
 import { EventEmitter } from 'events';
 import { load } from 'cheerio';
 import { diffWords, diffLines, diffChars } from 'diff';
-import * as cron from 'node-cron';
-import fs from 'fs/promises';
-import path from 'path';
-import fetch from 'node-fetch';
 
 const ChangeTrackingSchema = z.object({
   url: z.string().url(),
@@ -83,23 +77,7 @@ export class ChangeTracker extends EventEmitter {
     this.baselineContent = new Map();
     this.changeNotifications = new Map();
     this.snapshotManager = new Map();
-    
-    // Phase 2.4 Enhanced Features
-    this.scheduledMonitors = new Map(); // Cron-based monitoring
-    this.monitoringTemplates = new Map(); // Reusable monitoring configurations
-    this.alertRules = new Map(); // Custom alert rules and conditions
-    this.alertHistory = new Map(); // Alert notification history
-    this.trendAnalysis = new Map(); // Pattern recognition data
-    this.visualRegression = new Map(); // Visual diff storage
-    this.alertThrottling = new Map(); // Alert rate limiting
-    this.semanticDiffCache = new Map(); // Semantic analysis cache
-    this.monitoringDashboard = {
-      status: 'initialized',
-      monitors: new Map(),
-      alerts: [],
-      trends: {}
-    };
-    // Enhanced Statistics
+    // Statistics
     this.stats = {
       pagesTracked: 0,
       changesDetected: 0,
@@ -109,16 +87,7 @@ export class ChangeTracker extends EventEmitter {
       falsePositives: 0,
       averageChangeScore: 0,
       lastAnalysis: null,
-      processingTime: 0,
-      // Phase 2.4 additions
-      scheduledMonitors: 0,
-      alertsSent: 0,
-      alertsThrottled: 0,
-      semanticAnalyses: 0,
-      visualRegression: 0,
-      trendPatternsDetected: 0,
-      averageAlertResponseTime: 0,
-      monitoringUptime: 0
+      processingTime: 0
     };
     
     // Semantic analysis tools (if enabled)
@@ -133,322 +102,11 @@ export class ChangeTracker extends EventEmitter {
       await this.initializeSemanticAnalyzer();
     }
     
-    // Initialize Phase 2.4 components
-    await this.initializeEnhancedFeatures();
-    
     this.emit('initialized');
   }
   
   /**
-   * Initialize Enhanced Features for Phase 2.4
-   */
-  async initializeEnhancedFeatures() {
-    try {
-      // Initialize monitoring dashboard
-      this.monitoringDashboard.status = 'initializing';
-      
-      // Load existing monitoring templates
-      await this.loadMonitoringTemplates();
-      
-      // Initialize alert system
-      await this.initializeAlertSystem();
-      
-      // Set up historical analysis
-      await this.initializeHistoricalAnalysis();
-      
-      // Initialize semantic diff engine if enabled
-      if (this.options.enableSemanticAnalysis) {
-        await this.initializeSemanticDiffEngine();
-      }
-      
-      this.monitoringDashboard.status = 'active';
-      this.emit('enhancedFeaturesInitialized');
-      
-    } catch (error) {
-      this.monitoringDashboard.status = 'error';
-      this.emit('error', { operation: 'initializeEnhancedFeatures', error: error.message });
-      throw error;
-    }
-  }
-  
-  /**
-   * Load monitoring templates from storage
-   */
-  async loadMonitoringTemplates() {
-    const defaultTemplates = {
-      'news-site': {
-        name: 'News Site Monitoring',
-        frequency: '*/15 * * * *', // Every 15 minutes
-        options: {
-          granularity: 'section',
-          trackText: true,
-          trackStructure: false,
-          significanceThresholds: { minor: 0.05, moderate: 0.2, major: 0.5 }
-        },
-        alertRules: {
-          threshold: 'minor',
-          methods: ['webhook', 'email'],
-          throttle: 300000 // 5 minutes
-        }
-      },
-      'e-commerce': {
-        name: 'E-commerce Site Monitoring',
-        frequency: '0 */2 * * *', // Every 2 hours
-        options: {
-          granularity: 'element',
-          trackText: true,
-          trackStructure: true,
-          trackImages: true,
-          customSelectors: ['.price', '.stock-status', '.product-title']
-        },
-        alertRules: {
-          threshold: 'moderate',
-          methods: ['webhook', 'slack'],
-          throttle: 600000 // 10 minutes
-        }
-      },
-      'documentation': {
-        name: 'Documentation Monitoring',
-        frequency: '0 9 * * *', // Daily at 9 AM
-        options: {
-          granularity: 'section',
-          trackText: true,
-          trackStructure: true,
-          excludeSelectors: ['.last-updated', '.edit-link']
-        },
-        alertRules: {
-          threshold: 'major',
-          methods: ['email'],
-          throttle: 3600000 // 1 hour
-        }
-      }
-    };
-    
-    for (const [id, template] of Object.entries(defaultTemplates)) {
-      this.monitoringTemplates.set(id, template);
-    }
-  }
-  
-  /**
-   * Initialize alert system with default rules
-   */
-  async initializeAlertSystem() {
-    // Default alert rules
-    const defaultAlertRules = {
-      'critical-changes': {
-        condition: (changeResult) => changeResult.significance === 'critical',
-        actions: ['webhook', 'email', 'slack'],
-        throttle: 0, // No throttling for critical changes
-        priority: 'high'
-      },
-      'frequent-changes': {
-        condition: (url, history) => {
-          const recent = history.filter(h => Date.now() - h.timestamp < 3600000); // Last hour
-          return recent.length > 5;
-        },
-        actions: ['webhook'],
-        throttle: 1800000, // 30 minutes
-        priority: 'medium'
-      },
-      'structural-changes': {
-        condition: (changeResult) => changeResult.changeType === 'structural',
-        actions: ['webhook', 'email'],
-        throttle: 600000, // 10 minutes
-        priority: 'medium'
-      }
-    };
-    
-    for (const [id, rule] of Object.entries(defaultAlertRules)) {
-      this.alertRules.set(id, rule);
-    }
-  }
-  
-  /**
-   * Initialize historical analysis capabilities
-   */
-  async initializeHistoricalAnalysis() {
-    // Initialize trend analysis patterns
-    this.trendAnalysis.set('patterns', {
-      dailyChangePatterns: new Map(),
-      weeklyTrends: new Map(),
-      contentVelocity: new Map(),
-      changeFrequency: new Map()
-    });
-  }
-  
-  /**
-   * Initialize semantic diff engine
-   */
-  async initializeSemanticDiffEngine() {
-    // Initialize semantic analysis components
-    this.semanticDiffCache.set('initialized', true);
-    this.semanticDiffCache.set('algorithms', {
-      textSimilarity: this.calculateTextSimilarity.bind(this),
-      structuralSimilarity: this.calculateStructuralSimilarity.bind(this),
-      semanticSimilarity: this.calculateSemanticSimilarity.bind(this)
-    });
-  }
-  
-  /**
-   * Create scheduled monitoring with cron-like scheduling
-   * @param {string} url - URL to monitor
-   * @param {string} schedule - Cron expression
-   * @param {Object} options - Monitoring options
-   * @returns {Object} - Monitor configuration
-   */
-  async createScheduledMonitor(url, schedule, options = {}) {
-    try {
-      // Validate cron expression
-      if (!cron.validate(schedule)) {
-        throw new Error(`Invalid cron expression: ${schedule}`);
-      }
-      
-      const monitorId = `scheduled_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-      
-      const monitorConfig = {
-        id: monitorId,
-        url,
-        schedule,
-        options: {
-          granularity: 'section',
-          trackText: true,
-          trackStructure: true,
-          alertRules: {
-            threshold: 'moderate',
-            methods: ['webhook'],
-            throttle: 600000
-          },
-          ...options
-        },
-        stats: {
-          created: Date.now(),
-          executions: 0,
-          lastExecution: null,
-          changesDetected: 0,
-          errors: 0,
-          averageExecutionTime: 0
-        },
-        status: 'active'
-      };
-      
-      // Create cron job
-      const cronJob = cron.schedule(schedule, async () => {
-        await this.executeScheduledMonitor(monitorId);
-      }, {
-        scheduled: true,
-        timezone: 'UTC'
-      });
-      
-      monitorConfig.cronJob = cronJob;
-      
-      // Store monitor
-      this.scheduledMonitors.set(monitorId, monitorConfig);
-      this.monitoringDashboard.monitors.set(monitorId, {
-        url,
-        schedule,
-        status: 'active',
-        nextExecution: cronJob.nextDates().toString()
-      });
-      
-      this.stats.scheduledMonitors++;
-      
-      this.emit('scheduledMonitorCreated', {
-        monitorId,
-        url,
-        schedule,
-        nextExecution: cronJob.nextDates().toString()
-      });
-      
-      return {
-        success: true,
-        monitorId,
-        url,
-        schedule,
-        nextExecution: cronJob.nextDates().toString(),
-        options: monitorConfig.options
-      };
-      
-    } catch (error) {
-      this.emit('error', { operation: 'createScheduledMonitor', url, error: error.message });
-      throw new Error(`Failed to create scheduled monitor: ${error.message}`);
-    }
-  }
-  
-  /**
-   * Execute scheduled monitor check
-   * @param {string} monitorId - Monitor ID
-   */
-  async executeScheduledMonitor(monitorId) {
-    const startTime = Date.now();
-    
-    try {
-      const monitor = this.scheduledMonitors.get(monitorId);
-      if (!monitor || monitor.status !== 'active') {
-        return;
-      }
-      
-      monitor.stats.executions++;
-      monitor.stats.lastExecution = Date.now();
-      
-      // Fetch current content
-      const response = await fetch(monitor.url, {
-        headers: {
-          'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0-Enhanced',
-          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-        },
-        timeout: 30000
-      });
-      
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-      }
-      
-      const currentContent = await response.text();
-      
-      // Perform enhanced comparison
-      const comparisonResult = await this.performEnhancedComparison(
-        monitor.url,
-        currentContent,
-        monitor.options
-      );
-      
-      // Update execution time stats
-      const executionTime = Date.now() - startTime;
-      monitor.stats.averageExecutionTime = 
-        (monitor.stats.averageExecutionTime * (monitor.stats.executions - 1) + executionTime) / 
-        monitor.stats.executions;
-      
-      // Process change result
-      if (comparisonResult.hasChanges) {
-        monitor.stats.changesDetected++;
-        
-        // Update trend analysis
-        await this.updateTrendAnalysis(monitor.url, comparisonResult);
-        
-        // Check alert rules and send notifications
-        await this.processAlertRules(monitor.url, comparisonResult, monitor.options.alertRules);
-      }
-      
-      this.emit('scheduledMonitorExecuted', {
-        monitorId,
-        url: monitor.url,
-        hasChanges: comparisonResult.hasChanges,
-        significance: comparisonResult.significance,
-        executionTime
-      });
-      
-    } catch (error) {
-      const monitor = this.scheduledMonitors.get(monitorId);
-      if (monitor) {
-        monitor.stats.errors++;
-      }
-      
-      this.emit('scheduledMonitorError', {
-        monitorId,
-        error: error.message,
-        timestamp: Date.now()
-      });
-    }\n  }\n  \n  /**\n   * Perform enhanced comparison with semantic analysis\n   * @param {string} url - URL being compared\n   * @param {string} currentContent - Current content\n   * @param {Object} options - Comparison options\n   * @returns {Object} - Enhanced comparison results\n   */\n  async performEnhancedComparison(url, currentContent, options = {}) {\n    try {\n      // Get standard comparison\n      const standardComparison = await this.compareWithBaseline(url, currentContent, options);\n      \n      if (!standardComparison.hasChanges) {\n        return standardComparison;\n      }\n      \n      // Enhance with semantic analysis\n      const semanticAnalysis = await this.performSemanticAnalysis(\n        url, \n        currentContent, \n        standardComparison\n      );\n      \n      // Enhance with visual regression detection\n      const visualAnalysis = await this.performVisualRegressionAnalysis(\n        url,\n        currentContent,\n        options\n      );\n      \n      // Enhance with structured data analysis\n      const structuredAnalysis = await this.performStructuredDataAnalysis(\n        url,\n        currentContent,\n        standardComparison\n      );\n      \n      // Calculate enhanced significance score\n      const enhancedSignificance = await this.calculateEnhancedSignificance(\n        standardComparison,\n        semanticAnalysis,\n        visualAnalysis,\n        structuredAnalysis\n      );\n      \n      return {\n        ...standardComparison,\n        enhancedFeatures: {\n          semanticAnalysis,\n          visualAnalysis,\n          structuredAnalysis,\n          enhancedSignificance\n        },\n        significance: enhancedSignificance,\n        analysisType: 'enhanced'\n      };\n      \n    } catch (error) {\n      this.emit('error', { operation: 'performEnhancedComparison', url, error: error.message });\n      // Fall back to standard comparison\n      return await this.compareWithBaseline(url, currentContent, options);\n    }\n  }\n  \n  /**\n   * Perform semantic analysis of changes\n   * @param {string} url - URL\n   * @param {string} currentContent - Current content\n   * @param {Object} standardComparison - Standard comparison results\n   * @returns {Object} - Semantic analysis results\n   */\n  async performSemanticAnalysis(url, currentContent, standardComparison) {\n    const analysis = {\n      textualSimilarity: 0,\n      conceptualChanges: [],\n      sentimentChanges: [],\n      topicShifts: [],\n      keywordChanges: [],\n      confidenceScore: 0\n    };\n    \n    try {\n      // Get baseline content\n      const baseline = this.getLatestBaseline(url);\n      if (!baseline) {\n        return analysis;\n      }\n      \n      // Extract text content from both versions\n      const $ = load(currentContent);\n      const currentText = $.text().replace(/\\s+/g, ' ').trim();\n      \n      const $baseline = load(baseline.analysis.originalContent);\n      const baselineText = $baseline.text().replace(/\\s+/g, ' ').trim();\n      \n      // Calculate textual similarity using advanced algorithms\n      analysis.textualSimilarity = this.calculateTextSimilarity(baselineText, currentText);\n      \n      // Detect keyword changes\n      analysis.keywordChanges = this.detectKeywordChanges(baselineText, currentText);\n      \n      // Simple topic shift detection\n      analysis.topicShifts = this.detectTopicShifts(baselineText, currentText);\n      \n      // Calculate confidence score\n      analysis.confidenceScore = this.calculateSemanticConfidence(analysis);\n      \n      this.stats.semanticAnalyses++;\n      \n      return analysis;\n      \n    } catch (error) {\n      this.emit('error', { operation: 'performSemanticAnalysis', url, error: error.message });\n      return analysis;\n    }\n  }\n  \n  /**\n   * Perform visual regression analysis\n   * @param {string} url - URL\n   * @param {string} currentContent - Current content\n   * @param {Object} options - Analysis options\n   * @returns {Object} - Visual analysis results\n   */\n  async performVisualRegressionAnalysis(url, currentContent, options = {}) {\n    const analysis = {\n      layoutChanges: [],\n      cssChanges: [],\n      imageChanges: [],\n      fontChanges: [],\n      colorChanges: [],\n      hasVisualChanges: false\n    };\n    \n    try {\n      const $ = load(currentContent);\n      const baseline = this.getLatestBaseline(url);\n      \n      if (!baseline) {\n        return analysis;\n      }\n      \n      const $baseline = load(baseline.analysis.originalContent);\n      \n      // Detect layout changes\n      analysis.layoutChanges = this.detectLayoutChanges($baseline, $);\n      \n      // Detect CSS changes\n      analysis.cssChanges = this.detectCSSChanges($baseline, $);\n      \n      // Detect image changes\n      analysis.imageChanges = this.detectImageChanges($baseline, $);\n      \n      // Determine if there are visual changes\n      analysis.hasVisualChanges = \n        analysis.layoutChanges.length > 0 ||\n        analysis.cssChanges.length > 0 ||\n        analysis.imageChanges.length > 0;\n      \n      if (analysis.hasVisualChanges) {\n        this.stats.visualRegression++;\n      }\n      \n      return analysis;\n      \n    } catch (error) {\n      this.emit('error', { operation: 'performVisualRegressionAnalysis', url, error: error.message });\n      return analysis;\n    }\n  }\n  \n  /**\n   * Perform structured data analysis\n   * @param {string} url - URL\n   * @param {string} currentContent - Current content\n   * @param {Object} standardComparison - Standard comparison results\n   * @returns {Object} - Structured data analysis\n   */\n  async performStructuredDataAnalysis(url, currentContent, standardComparison) {\n    const analysis = {\n      schemaChanges: [],\n      dataFieldChanges: [],\n      validationChanges: [],\n      metadataChanges: [],\n      hasStructuredChanges: false\n    };\n    \n    try {\n      const $ = load(currentContent);\n      const baseline = this.getLatestBaseline(url);\n      \n      if (!baseline) {\n        return analysis;\n      }\n      \n      // Extract structured data (JSON-LD, microdata, etc.)\n      const currentStructuredData = this.extractStructuredData($);\n      const baselineStructuredData = this.extractStructuredData(load(baseline.analysis.originalContent));\n      \n      // Compare structured data\n      analysis.schemaChanges = this.compareStructuredData(baselineStructuredData, currentStructuredData);\n      \n      // Detect metadata changes\n      analysis.metadataChanges = this.compareMetadata(\n        baseline.analysis.metadata,\n        standardComparison.details.current?.metadata || {}\n      );\n      \n      analysis.hasStructuredChanges = \n        analysis.schemaChanges.length > 0 ||\n        analysis.metadataChanges.length > 0;\n      \n      return analysis;\n      \n    } catch (error) {\n      this.emit('error', { operation: 'performStructuredDataAnalysis', url, error: error.message });\n      return analysis;\n    }\n  }\n  \n  /**\n   * Update trend analysis with new change data\n   * @param {string} url - URL\n   * @param {Object} changeResult - Change analysis results\n   */\n  async updateTrendAnalysis(url, changeResult) {\n    try {\n      const patterns = this.trendAnalysis.get('patterns');\n      const now = new Date();\n      const dayKey = now.toISOString().slice(0, 10); // YYYY-MM-DD\n      const hourKey = now.toISOString().slice(0, 13); // YYYY-MM-DDTHH\n      \n      // Update daily patterns\n      if (!patterns.dailyChangePatterns.has(url)) {\n        patterns.dailyChangePatterns.set(url, new Map());\n      }\n      \n      const urlDailyPatterns = patterns.dailyChangePatterns.get(url);\n      if (!urlDailyPatterns.has(dayKey)) {\n        urlDailyPatterns.set(dayKey, {\n          changes: 0,\n          significance: [],\n          types: []\n        });\n      }\n      \n      const dayData = urlDailyPatterns.get(dayKey);\n      dayData.changes++;\n      dayData.significance.push(changeResult.significance);\n      dayData.types.push(changeResult.changeType);\n      \n      // Update change frequency\n      if (!patterns.changeFrequency.has(url)) {\n        patterns.changeFrequency.set(url, []);\n      }\n      \n      patterns.changeFrequency.get(url).push({\n        timestamp: Date.now(),\n        significance: changeResult.significance,\n        type: changeResult.changeType\n      });\n      \n      // Keep only last 1000 entries per URL\n      const frequency = patterns.changeFrequency.get(url);\n      if (frequency.length > 1000) {\n        frequency.splice(0, frequency.length - 1000);\n      }\n      \n      // Detect patterns\n      await this.detectChangePatterns(url, patterns);\n      \n    } catch (error) {\n      this.emit('error', { operation: 'updateTrendAnalysis', url, error: error.message });\n    }\n  }\n  \n  /**\n   * Process alert rules and send notifications\n   * @param {string} url - URL\n   * @param {Object} changeResult - Change results\n   * @param {Object} alertRules - Alert configuration\n   */\n  async processAlertRules(url, changeResult, alertRules = {}) {\n    try {\n      const alertsToSend = [];\n      \n      // Check each alert rule\n      for (const [ruleId, rule] of this.alertRules.entries()) {\n        let shouldTrigger = false;\n        \n        if (typeof rule.condition === 'function') {\n          try {\n            const history = this.getChangeHistory(url, 100);\n            shouldTrigger = rule.condition(changeResult, history);\n          } catch (error) {\n            this.emit('error', { \n              operation: 'evaluateAlertRule', \n              ruleId, \n              url, \n              error: error.message \n            });\n            continue;\n          }\n        }\n        \n        if (shouldTrigger) {\n          // Check throttling\n          const throttleKey = `${url}_${ruleId}`;\n          const lastAlert = this.alertThrottling.get(throttleKey);\n          \n          if (lastAlert && Date.now() - lastAlert < rule.throttle) {\n            this.stats.alertsThrottled++;\n            continue;\n          }\n          \n          alertsToSend.push({\n            ruleId,\n            rule,\n            url,\n            changeResult,\n            timestamp: Date.now()\n          });\n          \n          // Update throttling\n          this.alertThrottling.set(throttleKey, Date.now());\n        }\n      }\n      \n      // Send alerts\n      for (const alert of alertsToSend) {\n        await this.sendAlert(alert);\n      }\n      \n    } catch (error) {\n      this.emit('error', { operation: 'processAlertRules', url, error: error.message });\n    }\n  }\n  \n  /**\n   * Send alert notification\n   * @param {Object} alert - Alert configuration\n   */\n  async sendAlert(alert) {\n    const startTime = Date.now();\n    \n    try {\n      const alertData = {\n        id: `alert_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,\n        ruleId: alert.ruleId,\n        url: alert.url,\n        timestamp: alert.timestamp,\n        priority: alert.rule.priority,\n        changeResult: {\n          significance: alert.changeResult.significance,\n          changeType: alert.changeResult.changeType,\n          summary: alert.changeResult.summary\n        }\n      };\n      \n      // Send to each configured method\n      const promises = alert.rule.actions.map(async (action) => {\n        try {\n          await this.sendNotificationByMethod(action, alertData);\n          this.emit('alertSent', { action, alertId: alertData.id, url: alert.url });\n        } catch (error) {\n          this.emit('alertError', { \n            action, \n            alertId: alertData.id, \n            url: alert.url, \n            error: error.message \n          });\n        }\n      });\n      \n      await Promise.allSettled(promises);\n      \n      // Store alert in history\n      if (!this.alertHistory.has(alert.url)) {\n        this.alertHistory.set(alert.url, []);\n      }\n      \n      this.alertHistory.get(alert.url).unshift(alertData);\n      \n      // Keep only last 100 alerts per URL\n      const history = this.alertHistory.get(alert.url);\n      if (history.length > 100) {\n        history.splice(100);\n      }\n      \n      // Update stats\n      this.stats.alertsSent++;\n      const responseTime = Date.now() - startTime;\n      this.stats.averageAlertResponseTime = \n        (this.stats.averageAlertResponseTime * (this.stats.alertsSent - 1) + responseTime) / \n        this.stats.alertsSent;\n      \n      this.emit('alertProcessed', {\n        alertId: alertData.id,\n        url: alert.url,\n        responseTime\n      });\n      \n    } catch (error) {\n      this.emit('error', { operation: 'sendAlert', url: alert.url, error: error.message });\n    }\n  }\n  \n  /**\n   * Send notification by specific method\n   * @param {string} method - Notification method\n   * @param {Object} alertData - Alert data\n   */\n  async sendNotificationByMethod(method, alertData) {\n    switch (method) {\n      case 'webhook':\n        await this.sendWebhookAlert(alertData);\n        break;\n      case 'email':\n        await this.sendEmailAlert(alertData);\n        break;\n      case 'slack':\n        await this.sendSlackAlert(alertData);\n        break;\n      default:\n        throw new Error(`Unknown notification method: ${method}`);\n    }\n  }\n  \n  /**\n   * Generate trend analysis report\n   * @param {string} url - URL (optional, for specific URL analysis)\n   * @returns {Object} - Trend analysis report\n   */\n  async generateTrendAnalysisReport(url = null) {\n    try {\n      const report = {\n        timestamp: Date.now(),\n        scope: url ? 'url-specific' : 'global',\n        url,\n        patterns: {},\n        insights: [],\n        recommendations: []\n      };\n      \n      const patterns = this.trendAnalysis.get('patterns');\n      \n      if (url) {\n        // URL-specific analysis\n        report.patterns = await this.analyzeUrlPatterns(url, patterns);\n      } else {\n        // Global analysis\n        report.patterns = await this.analyzeGlobalPatterns(patterns);\n      }\n      \n      // Generate insights\n      report.insights = this.generateTrendInsights(report.patterns);\n      \n      // Generate recommendations\n      report.recommendations = this.generateTrendRecommendations(report.patterns, report.insights);\n      \n      return report;\n      \n    } catch (error) {\n      this.emit('error', { operation: 'generateTrendAnalysisReport', url, error: error.message });\n      throw error;\n    }\n  }\n  \n  /**\n   * Export historical data\n   * @param {Object} options - Export options\n   * @returns {Object} - Exported data\n   */\n  async exportHistoricalData(options = {}) {\n    const {\n      format = 'json',\n      url = null,\n      startTime = null,\n      endTime = null,\n      includeContent = false,\n      includeSnapshots = false\n    } = options;\n    \n    try {\n      const exportData = {\n        metadata: {\n          exportTime: Date.now(),\n          format,\n          scope: url ? 'url-specific' : 'global',\n          url,\n          timeRange: { startTime, endTime },\n          options\n        },\n        changeHistory: {},\n        snapshots: {},\n        alertHistory: {},\n        trendAnalysis: {},\n        statistics: this.getEnhancedStats()\n      };\n      \n      // Export change history\n      const urls = url ? [url] : Array.from(this.changeHistory.keys());\n      \n      for (const targetUrl of urls) {\n        let history = this.getChangeHistory(targetUrl, 10000);\n        \n        // Apply time filters\n        if (startTime || endTime) {\n          history = history.filter(entry => {\n            if (startTime && entry.timestamp < startTime) return false;\n            if (endTime && entry.timestamp > endTime) return false;\n            return true;\n          });\n        }\n        \n        // Remove content if not requested\n        if (!includeContent) {\n          history = history.map(entry => {\n            const { details, ...rest } = entry;\n            return {\n              ...rest,\n              details: details ? {\n                similarity: details.similarity,\n                significance: details.significance\n              } : undefined\n            };\n          });\n        }\n        \n        exportData.changeHistory[targetUrl] = history;\n        \n        // Export alert history\n        if (this.alertHistory.has(targetUrl)) {\n          exportData.alertHistory[targetUrl] = this.alertHistory.get(targetUrl);\n        }\n      }\n      \n      // Export trend analysis\n      const patterns = this.trendAnalysis.get('patterns');\n      if (patterns) {\n        exportData.trendAnalysis = {\n          dailyPatterns: Object.fromEntries(patterns.dailyChangePatterns),\n          changeFrequency: Object.fromEntries(patterns.changeFrequency)\n        };\n      }\n      \n      // Format output\n      if (format === 'csv') {\n        return this.convertToCSV(exportData);\n      }\n      \n      return exportData;\n      \n    } catch (error) {\n      this.emit('error', { operation: 'exportHistoricalData', error: error.message });\n      throw error;\n    }\n  }\n  \n  /**\n   * Get monitoring dashboard status\n   * @returns {Object} - Dashboard data\n   */\n  getMonitoringDashboard() {\n    return {\n      status: this.monitoringDashboard.status,\n      monitors: Array.from(this.monitoringDashboard.monitors.entries()).map(([id, config]) => ({\n        id,\n        ...config\n      })),\n      recentAlerts: this.monitoringDashboard.alerts.slice(-10),\n      trends: this.monitoringDashboard.trends,\n      statistics: this.getEnhancedStats(),\n      timestamp: Date.now()\n    };\n  }\n  \n  /**\n   * Get enhanced statistics\n   * @returns {Object} - Enhanced statistics\n   */\n  getEnhancedStats() {\n    return {\n      ...this.stats,\n      activeScheduledMonitors: this.scheduledMonitors.size,\n      alertRules: this.alertRules.size,\n      monitoringTemplates: this.monitoringTemplates.size,\n      throttledAlerts: this.alertThrottling.size,\n      trendPatterns: this.trendAnalysis.has('patterns') ? \n        this.trendAnalysis.get('patterns').dailyChangePatterns.size : 0\n    };\n  }\n  \n  /**\n   * Create baseline snapshot for change tracking
+   * Create baseline snapshot for change tracking
    * @param {string} url - URL to track
    * @param {string} content - Content to establish as baseline
    * @param {Object} options - Tracking options
@@ -1677,628 +1335,12 @@ export class ChangeTracker extends EventEmitter {
     // Placeholder for semantic analysis initialization
   }
 
-  // Enhanced Feature Helper Methods
-
-  /**
-   * Get latest baseline for a URL
-   * @param {string} url - URL
-   * @returns {Object} - Latest baseline
-   */
-  getLatestBaseline(url) {
-    const snapshots = this.snapshots.get(url);
-    return snapshots && snapshots.length > 0 ? snapshots[snapshots.length - 1] : null;
-  }
-
-  /**
-   * Calculate text similarity using advanced algorithms
-   * @param {string} text1 - First text
-   * @param {string} text2 - Second text
-   * @returns {number} - Similarity score (0-1)
-   */
-  calculateTextSimilarity(text1, text2) {
-    if (!text1 || !text2) return 0;
-    
-    // Simple Jaccard similarity for keywords
-    const words1 = new Set(text1.toLowerCase().split(/\W+/).filter(w => w.length > 3));
-    const words2 = new Set(text2.toLowerCase().split(/\W+/).filter(w => w.length > 3));
-    
-    const intersection = new Set([...words1].filter(x => words2.has(x)));
-    const union = new Set([...words1, ...words2]);
-    
-    return union.size > 0 ? intersection.size / union.size : 0;
-  }
-
-  /**
-   * Calculate semantic similarity
-   * @param {string} text1 - First text
-   * @param {string} text2 - Second text
-   * @returns {number} - Semantic similarity score
-   */
-  calculateSemanticSimilarity(text1, text2) {
-    // Placeholder for advanced semantic analysis
-    // Could integrate with NLP services or local models
-    return this.calculateTextSimilarity(text1, text2);
-  }
-
-  /**
-   * Detect keyword changes between texts
-   * @param {string} baselineText - Baseline text
-   * @param {string} currentText - Current text
-   * @returns {Array} - Keyword changes
-   */
-  detectKeywordChanges(baselineText, currentText) {
-    const changes = [];
-    
-    try {
-      const baselineWords = baselineText.toLowerCase().split(/\W+/).filter(w => w.length > 3);
-      const currentWords = currentText.toLowerCase().split(/\W+/).filter(w => w.length > 3);
-      
-      const baselineFreq = this.calculateWordFrequency(baselineWords);
-      const currentFreq = this.calculateWordFrequency(currentWords);
-      
-      // Find significant frequency changes
-      const allWords = new Set([...Object.keys(baselineFreq), ...Object.keys(currentFreq)]);
-      
-      for (const word of allWords) {
-        const baseFreq = baselineFreq[word] || 0;
-        const currFreq = currentFreq[word] || 0;
-        const change = Math.abs(currFreq - baseFreq);
-        
-        if (change > 2) { // Significant frequency change
-          changes.push({
-            word,
-            baselineFrequency: baseFreq,
-            currentFrequency: currFreq,
-            change: currFreq - baseFreq,
-            type: currFreq > baseFreq ? 'increased' : 'decreased'
-          });
-        }
-      }
-    } catch (error) {
-      this.emit('error', { operation: 'detectKeywordChanges', error: error.message });
-    }
-    
-    return changes.slice(0, 20); // Top 20 changes
-  }
-
-  /**
-   * Detect topic shifts between texts
-   * @param {string} baselineText - Baseline text
-   * @param {string} currentText - Current text
-   * @returns {Array} - Topic shifts
-   */
-  detectTopicShifts(baselineText, currentText) {
-    const shifts = [];
-    
-    try {
-      // Simple topic detection based on key phrases
-      const topicKeywords = {
-        technology: ['software', 'computer', 'digital', 'tech', 'system', 'data'],
-        business: ['company', 'market', 'business', 'sales', 'revenue', 'profit'],
-        health: ['health', 'medical', 'doctor', 'treatment', 'disease', 'patient'],
-        politics: ['government', 'policy', 'political', 'election', 'vote', 'congress'],
-        sports: ['game', 'team', 'player', 'score', 'match', 'championship']
-      };
-      
-      const baselineTopics = this.detectTopics(baselineText, topicKeywords);
-      const currentTopics = this.detectTopics(currentText, topicKeywords);
-      
-      // Compare topic presence
-      for (const topic of Object.keys(topicKeywords)) {
-        const baselineScore = baselineTopics[topic] || 0;
-        const currentScore = currentTopics[topic] || 0;
-        const change = currentScore - baselineScore;
-        
-        if (Math.abs(change) > 0.1) {
-          shifts.push({
-            topic,
-            baselineScore,
-            currentScore,
-            change,
-            type: change > 0 ? 'emerged' : 'diminished'
-          });
-        }
-      }
-    } catch (error) {
-      this.emit('error', { operation: 'detectTopicShifts', error: error.message });
-    }
-    
-    return shifts;
-  }
-
-  /**
-   * Calculate semantic confidence score
-   * @param {Object} analysis - Semantic analysis
-   * @returns {number} - Confidence score
-   */
-  calculateSemanticConfidence(analysis) {
-    let confidence = 0;
-    
-    // Base confidence on available data
-    if (analysis.textualSimilarity > 0) confidence += 0.3;
-    if (analysis.keywordChanges.length > 0) confidence += 0.3;
-    if (analysis.topicShifts.length > 0) confidence += 0.2;
-    
-    // Adjust based on data quality
-    const dataQuality = Math.min(
-      analysis.keywordChanges.length / 10, // Max 10 keyword changes for full score
-      1
-    );
-    
-    return Math.min(confidence * dataQuality, 1);
-  }
-
-  /**
-   * Detect layout changes between DOM structures
-   * @param {Object} baseline - Baseline DOM
-   * @param {Object} current - Current DOM
-   * @returns {Array} - Layout changes
-   */
-  detectLayoutChanges(baseline, current) {
-    const changes = [];
-    
-    try {
-      // Compare element counts by type
-      const baselineElements = this.countElements(baseline);
-      const currentElements = this.countElements(current);
-      
-      for (const [tag, baseCount] of Object.entries(baselineElements)) {
-        const currCount = currentElements[tag] || 0;
-        if (Math.abs(currCount - baseCount) > 0) {
-          changes.push({
-            type: 'element_count_change',
-            tag,
-            baseline: baseCount,
-            current: currCount,
-            change: currCount - baseCount
-          });
-        }
-      }
-      
-      // Check for new element types
-      for (const [tag, currCount] of Object.entries(currentElements)) {
-        if (!baselineElements[tag]) {
-          changes.push({
-            type: 'new_element_type',
-            tag,
-            count: currCount
-          });
-        }
-      }
-    } catch (error) {
-      this.emit('error', { operation: 'detectLayoutChanges', error: error.message });
-    }
-    
-    return changes;
-  }
-
-  /**
-   * Detect CSS changes
-   * @param {Object} baseline - Baseline DOM
-   * @param {Object} current - Current DOM
-   * @returns {Array} - CSS changes
-   */
-  detectCSSChanges(baseline, current) {
-    const changes = [];
-    
-    try {
-      // Extract style information
-      const baselineStyles = this.extractStyles(baseline);
-      const currentStyles = this.extractStyles(current);
-      
-      // Compare inline styles
-      const styleDiff = this.compareStyles(baselineStyles, currentStyles);
-      changes.push(...styleDiff);
-      
-    } catch (error) {
-      this.emit('error', { operation: 'detectCSSChanges', error: error.message });
-    }
-    
-    return changes;
-  }
-
-  /**
-   * Extract structured data from DOM
-   * @param {Object} $ - Cheerio DOM
-   * @returns {Object} - Structured data
-   */
-  extractStructuredData($) {
-    const structuredData = {
-      jsonLd: [],
-      microdata: [],
-      rdfa: [],
-      openGraph: {},
-      twitterCard: {},
-      schema: []
-    };
-    
-    try {
-      // Extract JSON-LD
-      $('script[type="application/ld+json"]').each((index, element) => {
-        try {
-          const data = JSON.parse($(element).html());
-          structuredData.jsonLd.push(data);
-        } catch (e) {
-          // Invalid JSON, skip
-        }
-      });
-      
-      // Extract Open Graph
-      $('meta[property^="og:"]').each((index, element) => {
-        const property = $(element).attr('property');
-        const content = $(element).attr('content');
-        if (property && content) {
-          structuredData.openGraph[property] = content;
-        }
-      });
-      
-      // Extract Twitter Card
-      $('meta[name^="twitter:"]').each((index, element) => {
-        const name = $(element).attr('name');
-        const content = $(element).attr('content');
-        if (name && content) {
-          structuredData.twitterCard[name] = content;
-        }
-      });
-      
-    } catch (error) {
-      this.emit('error', { operation: 'extractStructuredData', error: error.message });
-    }
-    
-    return structuredData;
-  }
-
-  /**
-   * Compare structured data
-   * @param {Object} baseline - Baseline structured data
-   * @param {Object} current - Current structured data
-   * @returns {Array} - Schema changes
-   */
-  compareStructuredData(baseline, current) {
-    const changes = [];
-    
-    try {
-      // Compare JSON-LD
-      const jsonLdChanges = this.compareArrayData(baseline.jsonLd, current.jsonLd, 'json-ld');
-      changes.push(...jsonLdChanges);
-      
-      // Compare Open Graph
-      const ogChanges = this.compareObjectData(baseline.openGraph, current.openGraph, 'open-graph');
-      changes.push(...ogChanges);
-      
-      // Compare Twitter Card
-      const twitterChanges = this.compareObjectData(baseline.twitterCard, current.twitterCard, 'twitter-card');
-      changes.push(...twitterChanges);
-      
-    } catch (error) {
-      this.emit('error', { operation: 'compareStructuredData', error: error.message });
-    }
-    
-    return changes;
-  }
-
-  /**
-   * Compare metadata objects
-   * @param {Object} baseline - Baseline metadata
-   * @param {Object} current - Current metadata
-   * @returns {Array} - Metadata changes
-   */
-  compareMetadata(baseline, current) {
-    const changes = [];
-    
-    try {
-      const baselineKeys = Object.keys(baseline || {});
-      const currentKeys = Object.keys(current || {});
-      const allKeys = new Set([...baselineKeys, ...currentKeys]);
-      
-      for (const key of allKeys) {
-        const baseValue = baseline?.[key];
-        const currValue = current?.[key];
-        
-        if (JSON.stringify(baseValue) !== JSON.stringify(currValue)) {
-          changes.push({
-            type: 'metadata_change',
-            field: key,
-            baseline: baseValue,
-            current: currValue,
-            changeType: !baseValue ? 'added' : !currValue ? 'removed' : 'modified'
-          });
-        }
-      }
-    } catch (error) {
-      this.emit('error', { operation: 'compareMetadata', error: error.message });
-    }
-    
-    return changes;
-  }
-
-  /**
-   * Calculate enhanced significance score
-   * @param {Object} standardComparison - Standard comparison
-   * @param {Object} semanticAnalysis - Semantic analysis
-   * @param {Object} visualAnalysis - Visual analysis
-   * @param {Object} structuredAnalysis - Structured analysis
-   * @returns {string} - Enhanced significance level
-   */
-  async calculateEnhancedSignificance(standardComparison, semanticAnalysis, visualAnalysis, structuredAnalysis) {
-    try {
-      let enhancedScore = 0;
-      const weights = {
-        standard: 0.4,
-        semantic: 0.2,
-        visual: 0.2,
-        structured: 0.2
-      };
-      
-      // Standard comparison score
-      const standardScore = this.getSignificanceScore(standardComparison.significance);
-      enhancedScore += standardScore * weights.standard;
-      
-      // Semantic analysis score
-      const semanticScore = semanticAnalysis.confidenceScore * 
-        (1 - semanticAnalysis.textualSimilarity);
-      enhancedScore += semanticScore * weights.semantic;
-      
-      // Visual analysis score
-      const visualScore = visualAnalysis.hasVisualChanges ? 0.7 : 0;
-      enhancedScore += visualScore * weights.visual;
-      
-      // Structured data score
-      const structuredScore = structuredAnalysis.hasStructuredChanges ? 0.8 : 0;
-      enhancedScore += structuredScore * weights.structured;
-      
-      // Convert to significance level
-      return this.scoreToSignificance(enhancedScore);
-      
-    } catch (error) {
-      this.emit('error', { operation: 'calculateEnhancedSignificance', error: error.message });
-      return standardComparison.significance;
-    }
-  }
-
-  /**
-   * Detect change patterns in historical data
-   * @param {string} url - URL
-   * @param {Object} patterns - Pattern data
-   */
-  async detectChangePatterns(url, patterns) {
-    try {
-      const frequency = patterns.changeFrequency.get(url);
-      if (!frequency || frequency.length < 10) return;
-      
-      // Detect recurring patterns
-      const recurringPatterns = this.detectRecurringPatterns(frequency);
-      
-      // Detect time-based patterns
-      const timePatterns = this.detectTimePatterns(frequency);
-      
-      // Update trend analysis
-      if (recurringPatterns.length > 0 || timePatterns.length > 0) {
-        this.stats.trendPatternsDetected++;
-        
-        this.emit('patternsDetected', {
-          url,
-          recurringPatterns,
-          timePatterns,
-          timestamp: Date.now()
-        });
-      }
-      
-    } catch (error) {
-      this.emit('error', { operation: 'detectChangePatterns', url, error: error.message });
-    }
-  }
-
-  /**
-   * Send webhook alert
-   * @param {Object} alertData - Alert data
-   */
-  async sendWebhookAlert(alertData) {
-    // Placeholder for webhook implementation
-    this.emit('webhookAlert', alertData);
-  }
-
-  /**
-   * Send email alert
-   * @param {Object} alertData - Alert data
-   */
-  async sendEmailAlert(alertData) {
-    // Placeholder for email implementation
-    this.emit('emailAlert', alertData);
-  }
-
-  /**
-   * Send Slack alert
-   * @param {Object} alertData - Alert data
-   */
-  async sendSlackAlert(alertData) {
-    // Placeholder for Slack implementation
-    this.emit('slackAlert', alertData);
-  }
-
-  // Utility helper methods
-
-  calculateWordFrequency(words) {
-    const frequency = {};
-    words.forEach(word => {
-      frequency[word] = (frequency[word] || 0) + 1;
-    });
-    return frequency;
-  }
-
-  detectTopics(text, topicKeywords) {
-    const topics = {};
-    const words = text.toLowerCase().split(/\W+/);
-    
-    for (const [topic, keywords] of Object.entries(topicKeywords)) {
-      let score = 0;
-      keywords.forEach(keyword => {
-        score += words.filter(word => word.includes(keyword)).length;
-      });
-      topics[topic] = score / words.length;
-    }
-    
-    return topics;
-  }
-
-  countElements($) {
-    const counts = {};
-    $('*').each((index, element) => {
-      const tag = element.name;
-      counts[tag] = (counts[tag] || 0) + 1;
-    });
-    return counts;
-  }
-
-  extractStyles($) {
-    const styles = {};
-    $('[style]').each((index, element) => {
-      const style = $(element).attr('style');
-      if (style) {
-        styles[`element_${index}`] = style;
-      }
-    });
-    return styles;
-  }
-
-  compareStyles(baseline, current) {
-    const changes = [];
-    const allKeys = new Set([...Object.keys(baseline), ...Object.keys(current)]);
-    
-    for (const key of allKeys) {
-      if (baseline[key] !== current[key]) {
-        changes.push({
-          type: 'style_change',
-          element: key,
-          baseline: baseline[key],
-          current: current[key]
-        });
-      }
-    }
-    
-    return changes;
-  }
-
-  compareArrayData(baseline, current, type) {
-    const changes = [];
-    
-    if (baseline.length !== current.length) {
-      changes.push({
-        type: `${type}_count_change`,
-        baseline: baseline.length,
-        current: current.length
-      });
-    }
-    
-    return changes;
-  }
-
-  compareObjectData(baseline, current, type) {
-    const changes = [];
-    const allKeys = new Set([...Object.keys(baseline), ...Object.keys(current)]);
-    
-    for (const key of allKeys) {
-      if (baseline[key] !== current[key]) {
-        changes.push({
-          type: `${type}_change`,
-          field: key,
-          baseline: baseline[key],
-          current: current[key]
-        });
-      }
-    }
-    
-    return changes;
-  }
-
-  getSignificanceScore(significance) {
-    const scores = {
-      'none': 0,
-      'minor': 0.2,
-      'moderate': 0.5,
-      'major': 0.8,
-      'critical': 1.0
-    };
-    return scores[significance] || 0;
-  }
-
-  scoreToSignificance(score) {
-    if (score >= 0.9) return 'critical';
-    if (score >= 0.7) return 'major';
-    if (score >= 0.4) return 'moderate';
-    if (score >= 0.1) return 'minor';
-    return 'none';
-  }
-
-  analyzeUrlPatterns(url, patterns) {
-    // Placeholder for URL-specific pattern analysis
-    return {
-      dailyAverage: 0,
-      peakTimes: [],
-      commonTypes: []
-    };
-  }
-
-  analyzeGlobalPatterns(patterns) {
-    // Placeholder for global pattern analysis
-    return {
-      totalUrls: patterns.dailyChangePatterns.size,
-      mostActiveUrls: [],
-      commonPatterns: []
-    };
-  }
-
-  generateTrendInsights(patterns) {
-    return [
-      'Pattern analysis requires more data',
-      'Monitoring is active and collecting data'
-    ];
-  }
-
-  generateTrendRecommendations(patterns, insights) {
-    return [
-      'Continue monitoring to build pattern database',
-      'Consider adjusting monitoring frequency based on change patterns'
-    ];
-  }
-
-  detectRecurringPatterns(frequency) {
-    // Placeholder for recurring pattern detection
-    return [];
-  }
-
-  detectTimePatterns(frequency) {
-    // Placeholder for time-based pattern detection
-    return [];
-  }
-
-  convertToCSV(data) {
-    // Placeholder for CSV conversion
-    return JSON.stringify(data, null, 2);
-  }
-
   cleanup() {
-    // Stop all scheduled monitors
-    for (const [id, monitor] of this.scheduledMonitors.entries()) {
-      if (monitor.cronJob) {
-        monitor.cronJob.destroy();
-      }
-    }
-    
-    // Clear all data
     this.contentHistory.clear();
     this.baselineContent.clear();
     this.activeMonitors.clear();
     this.changeNotifications.clear();
     this.snapshotManager.clear();
-    this.scheduledMonitors.clear();
-    this.monitoringTemplates.clear();
-    this.alertRules.clear();
-    this.alertHistory.clear();
-    this.trendAnalysis.clear();
-    this.visualRegression.clear();
-    this.alertThrottling.clear();
-    this.semanticDiffCache.clear();
   }
 
 }