crawlforge-mcp-server 3.0.0

package/src/utils/MemoryMonitor.js

@@ -0,0 +1,173 @@
+/**
+ * Memory Monitoring and Leak Detection Utility
+ * Tracks memory usage patterns and detects potential leaks
+ */
+
+export class MemoryMonitor {
+  constructor(options = {}) {
+    this.options = {
+      sampleInterval: options.sampleInterval || 30000, // 30 seconds
+      maxSamples: options.maxSamples || 100,
+      leakThreshold: options.leakThreshold || 50 * 1024 * 1024, // 50MB
+      enableLogging: options.enableLogging !== false,
+      alertCallback: options.alertCallback || null,
+      ...options
+    };
+    
+    this.samples = [];
+    this.isMonitoring = false;
+    this.intervalId = null;
+    this.leakWarnings = 0;
+  }
+  
+  /**
+   * Start memory monitoring
+   */
+  start() {
+    if (this.isMonitoring) {
+      return;
+    }
+    
+    this.isMonitoring = true;
+    this.log('Starting memory monitoring...');
+    
+    this.intervalId = setInterval(() => {
+      this.takeSample();
+      this.analyzeMemoryTrend();
+    }, this.options.sampleInterval);
+    
+    // Take initial sample
+    this.takeSample();
+  }
+  
+  /**
+   * Stop memory monitoring
+   */
+  stop() {
+    if (!this.isMonitoring) {
+      return;
+    }
+    
+    this.isMonitoring = false;
+    
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+    
+    this.log('Memory monitoring stopped');
+  }
+  
+  /**
+   * Take a memory usage sample
+   */
+  takeSample() {
+    const usage = process.memoryUsage();
+    const timestamp = Date.now();
+    
+    const sample = {
+      timestamp,
+      heapUsed: usage.heapUsed,
+      heapTotal: usage.heapTotal,
+      external: usage.external,
+      rss: usage.rss,
+      heapUsedMB: Math.round(usage.heapUsed / 1024 / 1024 * 100) / 100,
+      heapTotalMB: Math.round(usage.heapTotal / 1024 / 1024 * 100) / 100
+    };
+    
+    this.samples.push(sample);
+    
+    // Keep only the last N samples
+    if (this.samples.length > this.options.maxSamples) {
+      this.samples.shift();
+    }
+    
+    return sample;
+  }
+  
+  /**
+   * Get current memory statistics
+   */
+  getStats() {
+    if (this.samples.length === 0) {
+      return null;
+    }
+    
+    const latest = this.samples[this.samples.length - 1];
+    const peak = Math.max(...this.samples.map(s => s.heapUsed));
+    const average = this.samples.reduce((sum, s) => sum + s.heapUsed, 0) / this.samples.length;
+    
+    return {
+      current: {
+        heapUsedMB: latest.heapUsedMB,
+        heapTotalMB: latest.heapTotalMB,
+        timestamp: latest.timestamp
+      },
+      peak: {
+        heapUsedMB: Math.round(peak / 1024 / 1024 * 100) / 100
+      },
+      average: {
+        heapUsedMB: Math.round(average / 1024 / 1024 * 100) / 100
+      },
+      samples: this.samples.length,
+      leakWarnings: this.leakWarnings,
+      isMonitoring: this.isMonitoring
+    };
+  }
+  
+  /**
+   * Analyze memory trend for potential leaks
+   */
+  analyzeMemoryTrend() {
+    if (this.samples.length < 5) {
+      return; // Need at least 5 samples for trend analysis
+    }
+    
+    // Check if memory is consistently increasing
+    const recentSamples = this.samples.slice(-10); // Last 10 samples
+    let increasingCount = 0;
+    
+    for (let i = 1; i < recentSamples.length; i++) {
+      if (recentSamples[i].heapUsed > recentSamples[i - 1].heapUsed) {
+        increasingCount++;
+      }
+    }
+    
+    // If memory increased in 80% of recent samples, potential leak
+    if (increasingCount > recentSamples.length * 0.8) {
+      const memoryGrowth = recentSamples[recentSamples.length - 1].heapUsed - recentSamples[0].heapUsed;
+      
+      if (memoryGrowth > this.options.leakThreshold) {
+        this.leakWarnings++;
+        const growthMB = Math.round(memoryGrowth / 1024 / 1024 * 100) / 100;
+        
+        this.log(`Warning: Potential memory leak detected. Memory grew by ${growthMB}MB over ${recentSamples.length} samples`);
+        
+        if (this.options.alertCallback) {
+          this.options.alertCallback({
+            type: 'memory_leak',
+            growthMB,
+            samples: recentSamples.length,
+            warnings: this.leakWarnings
+          });
+        }
+      }
+    }
+  }
+  
+  /**
+   * Log messages if logging is enabled
+   */
+  log(message) {
+    if (this.options.enableLogging) {
+      console.error(`[MemoryMonitor] ${message}`);
+    }
+  }
+}
+
+// Export singleton instance for global use
+export const memoryMonitor = new MemoryMonitor({
+  enableLogging: process.env.NODE_ENV === 'development',
+  sampleInterval: 30000, // 30 seconds
+  leakThreshold: 100 * 1024 * 1024 // 100MB
+});

package/src/utils/RetryManager.js

@@ -0,0 +1,386 @@
+/**
+ * RetryManager - Comprehensive retry management with multiple strategies
+ * Handles exponential backoff, circuit breaking integration, and retry policies
+ */
+
+export class RetryManager {
+  constructor(options = {}) {
+    const {
+      maxRetries = 3,
+      baseDelay = 1000,
+      maxDelay = 30000,
+      strategy = 'exponential',
+      jitter = true,
+      retryableErrors = ['ECONNRESET', 'ENOTFOUND', 'ECONNREFUSED', 'ETIMEDOUT'],
+      retryableStatusCodes = [408, 429, 500, 502, 503, 504],
+      onRetry = null,
+      onFailure = null
+    } = options;
+
+    this.maxRetries = maxRetries;
+    this.baseDelay = baseDelay;
+    this.maxDelay = maxDelay;
+    this.strategy = strategy;
+    this.jitter = jitter;
+    this.retryableErrors = new Set(retryableErrors);
+    this.retryableStatusCodes = new Set(retryableStatusCodes);
+    this.onRetry = onRetry;
+    this.onFailure = onFailure;
+
+    // Strategy implementations
+    this.strategies = {
+      linear: this.linearBackoff.bind(this),
+      exponential: this.exponentialBackoff.bind(this),
+      fibonacci: this.fibonacciBackoff.bind(this),
+      fixed: this.fixedBackoff.bind(this)
+    };
+
+    // Statistics tracking
+    this.stats = {
+      totalAttempts: 0,
+      totalRetries: 0,
+      successfulRetries: 0,
+      failedOperations: 0,
+      averageRetryDelay: 0
+    };
+  }
+
+  /**
+   * Execute an operation with retry logic
+   * @param {Function} operation - Async function to execute
+   * @param {Object} context - Context information for logging/callbacks
+   * @returns {Promise} Result of successful operation
+   */
+  async execute(operation, context = {}) {
+    let lastError;
+    let totalDelay = 0;
+    const startTime = Date.now();
+
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      this.stats.totalAttempts++;
+
+      try {
+        const result = await operation();
+        
+        if (attempt > 0) {
+          this.stats.successfulRetries++;
+          this.updateAverageDelay(totalDelay);
+        }
+
+        return result;
+      } catch (error) {
+        lastError = error;
+
+        // Check if error is retryable
+        if (!this.isRetryableError(error)) {
+          this.stats.failedOperations++;
+          if (this.onFailure) {
+            await this.onFailure(error, attempt, context);
+          }
+          throw error;
+        }
+
+        // Don't retry on last attempt
+        if (attempt === this.maxRetries) {
+          this.stats.failedOperations++;
+          if (this.onFailure) {
+            await this.onFailure(error, attempt, context);
+          }
+          throw new RetryExhaustedError(
+            `Operation failed after ${this.maxRetries} retries: ${error.message}`,
+            error,
+            attempt
+          );
+        }
+
+        // Calculate delay for next attempt
+        const delay = this.calculateDelay(attempt);
+        totalDelay += delay;
+
+        this.stats.totalRetries++;
+
+        // Call retry callback if provided
+        if (this.onRetry) {
+          await this.onRetry(error, attempt, delay, context);
+        }
+
+        // Wait before retrying
+        await this.delay(delay);
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Execute multiple operations with retry, stopping on first success
+   * @param {Array<Function>} operations - Array of async functions
+   * @param {Object} context - Context information
+   * @returns {Promise} Result of first successful operation
+   */
+  async executeAny(operations, context = {}) {
+    let lastError;
+
+    for (const operation of operations) {
+      try {
+        return await this.execute(operation, context);
+      } catch (error) {
+        lastError = error;
+      }
+    }
+
+    throw new Error(`All ${operations.length} operations failed. Last error: ${lastError.message}`);
+  }
+
+  /**
+   * Execute operation with circuit breaker integration
+   * @param {Function} operation - Async function to execute
+   * @param {CircuitBreaker} circuitBreaker - Circuit breaker instance
+   * @param {string} domain - Domain for circuit breaker
+   * @param {Object} context - Context information
+   * @returns {Promise} Result of successful operation
+   */
+  async executeWithCircuitBreaker(operation, circuitBreaker, domain, context = {}) {
+    return this.execute(async () => {
+      return circuitBreaker.execute(domain, operation);
+    }, { ...context, domain });
+  }
+
+  /**
+   * Check if an error is retryable based on configuration
+   * @param {Error} error - Error to check
+   * @returns {boolean} Whether the error is retryable
+   */
+  isRetryableError(error) {
+    // Check error codes
+    if (error.code && this.retryableErrors.has(error.code)) {
+      return true;
+    }
+
+    // Check HTTP status codes
+    if (error.response && error.response.status) {
+      return this.retryableStatusCodes.has(error.response.status);
+    }
+
+    // Check error types
+    if (error instanceof TypeError && error.message.includes('fetch')) {
+      return true;
+    }
+
+    // Check for timeout errors
+    if (error.message && (
+      error.message.includes('timeout') ||
+      error.message.includes('ETIMEDOUT') ||
+      error.message.includes('socket hang up')
+    )) {
+      return true;
+    }
+
+    // Circuit breaker errors are retryable
+    if (error.message && error.message.includes('Circuit breaker is OPEN')) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Calculate delay based on configured strategy
+   * @param {number} attempt - Current attempt number (0-based)
+   * @returns {number} Delay in milliseconds
+   */
+  calculateDelay(attempt) {
+    const strategy = this.strategies[this.strategy] || this.strategies.exponential;
+    let delay = strategy(attempt);
+
+    // Apply maximum delay cap
+    delay = Math.min(delay, this.maxDelay);
+
+    // Apply jitter to prevent thundering herd
+    if (this.jitter) {
+      delay = this.addJitter(delay);
+    }
+
+    return Math.max(0, delay);
+  }
+
+  /**
+   * Linear backoff strategy
+   * @param {number} attempt - Attempt number
+   * @returns {number} Delay in milliseconds
+   */
+  linearBackoff(attempt) {
+    return this.baseDelay * (attempt + 1);
+  }
+
+  /**
+   * Exponential backoff strategy
+   * @param {number} attempt - Attempt number
+   * @returns {number} Delay in milliseconds
+   */
+  exponentialBackoff(attempt) {
+    return this.baseDelay * Math.pow(2, attempt);
+  }
+
+  /**
+   * Fibonacci backoff strategy
+   * @param {number} attempt - Attempt number
+   * @returns {number} Delay in milliseconds
+   */
+  fibonacciBackoff(attempt) {
+    if (attempt <= 1) return this.baseDelay;
+    
+    let a = 1, b = 1;
+    for (let i = 2; i <= attempt; i++) {
+      [a, b] = [b, a + b];
+    }
+    
+    return this.baseDelay * b;
+  }
+
+  /**
+   * Fixed delay strategy
+   * @param {number} attempt - Attempt number
+   * @returns {number} Delay in milliseconds
+   */
+  fixedBackoff(attempt) {
+    return this.baseDelay;
+  }
+
+  /**
+   * Add jitter to delay to prevent thundering herd
+   * @param {number} delay - Base delay
+   * @returns {number} Jittered delay
+   */
+  addJitter(delay) {
+    // Use full jitter: random value between 0 and delay
+    return Math.random() * delay;
+  }
+
+  /**
+   * Promise-based delay utility
+   * @param {number} ms - Milliseconds to delay
+   * @returns {Promise} Promise that resolves after delay
+   */
+  delay(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Update average retry delay statistic
+   * @param {number} totalDelay - Total delay for this operation
+   */
+  updateAverageDelay(totalDelay) {
+    const currentAverage = this.stats.averageRetryDelay;
+    const count = this.stats.successfulRetries;
+    this.stats.averageRetryDelay = ((currentAverage * (count - 1)) + totalDelay) / count;
+  }
+
+  /**
+   * Get retry statistics
+   * @returns {Object} Statistics object
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      successRate: this.stats.totalAttempts > 0 
+        ? ((this.stats.totalAttempts - this.stats.failedOperations) / this.stats.totalAttempts) * 100 
+        : 0,
+      retryRate: this.stats.totalAttempts > 0 
+        ? (this.stats.totalRetries / this.stats.totalAttempts) * 100 
+        : 0
+    };
+  }
+
+  /**
+   * Reset statistics
+   */
+  resetStats() {
+    this.stats = {
+      totalAttempts: 0,
+      totalRetries: 0,
+      successfulRetries: 0,
+      failedOperations: 0,
+      averageRetryDelay: 0
+    };
+  }
+
+  /**
+   * Create a configured retry manager for specific use cases
+   * @param {string} preset - Preset name ('aggressive', 'conservative', 'network', 'api')
+   * @returns {RetryManager} Configured retry manager
+   */
+  static createPreset(preset) {
+    const presets = {
+      aggressive: {
+        maxRetries: 5,
+        baseDelay: 500,
+        maxDelay: 10000,
+        strategy: 'exponential',
+        jitter: true
+      },
+      conservative: {
+        maxRetries: 2,
+        baseDelay: 2000,
+        maxDelay: 60000,
+        strategy: 'linear',
+        jitter: false
+      },
+      network: {
+        maxRetries: 3,
+        baseDelay: 1000,
+        maxDelay: 30000,
+        strategy: 'exponential',
+        jitter: true,
+        retryableErrors: ['ECONNRESET', 'ENOTFOUND', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE']
+      },
+      api: {
+        maxRetries: 4,
+        baseDelay: 1000,
+        maxDelay: 16000,
+        strategy: 'exponential',
+        jitter: true,
+        retryableStatusCodes: [408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524]
+      }
+    };
+
+    const config = presets[preset];
+    if (!config) {
+      throw new Error(`Unknown preset: ${preset}. Available presets: ${Object.keys(presets).join(', ')}`);
+    }
+
+    return new RetryManager(config);
+  }
+}
+
+/**
+ * Custom error class for retry exhausted scenarios
+ */
+export class RetryExhaustedError extends Error {
+  constructor(message, originalError, attempts) {
+    super(message);
+    this.name = 'RetryExhaustedError';
+    this.originalError = originalError;
+    this.attempts = attempts;
+  }
+}
+
+/**
+ * Decorator function to add retry logic to any async function
+ * @param {RetryManager} retryManager - Retry manager instance
+ * @param {Object} context - Context for the operation
+ * @returns {Function} Decorator function
+ */
+export function withRetry(retryManager, context = {}) {
+  return function(target, propertyKey, descriptor) {
+    const originalMethod = descriptor.value;
+    
+    descriptor.value = async function(...args) {
+      return retryManager.execute(() => originalMethod.apply(this, args), context);
+    };
+    
+    return descriptor;
+  };
+}
+
+export default RetryManager;