aetherframework-cluster 1.0.0

package/src/core/HealthMonitor.js

@@ -0,0 +1,571 @@
+// packages/cluster/src/core/HealthMonitor.js
+import { EventEmitter } from 'events';
+
+/**
+ * Health Monitor - Monitors the health of worker processes and the cluster
+ * Provides health check endpoints and monitoring capabilities
+ */
+class HealthMonitor extends EventEmitter {
+  constructor(clusterManager, options = {}) {
+    super();
+    
+    this.clusterManager = clusterManager;
+    this.options = {
+      checkInterval: options.checkInterval || 30000,
+      timeout: options.timeout || 5000,
+      memoryThreshold: options.memoryThreshold || 0.8,
+      cpuThreshold: options.cpuThreshold || 0.7,
+      maxErrorRate: options.maxErrorRate || 10,
+      maxRestarts: options.maxRestarts || 5,
+      ...options
+    };
+    
+    this.healthChecks = new Map();
+    this.healthHistory = [];
+    this.maxHistorySize = 100;
+    this.checkInterval = null;
+    this.lastCheckTime = null;
+    
+    // Register default health checks
+    this.registerDefaultChecks();
+  }
+  
+  /**
+   * Register default health checks
+   */
+  registerDefaultChecks() {
+    // Worker availability check
+    this.registerCheck('worker_availability', async () => {
+      const stats = this.clusterManager.getStats();
+      const activeWorkers = stats.workers.filter(w => w.state === 'online').length;
+      
+      if (activeWorkers === 0) {
+        throw new Error('No active workers available');
+      }
+      
+      if (activeWorkers < stats.total * 0.5) {
+        throw new Error(`Less than 50% of workers active: ${activeWorkers}/${stats.total}`);
+      }
+      
+      return {
+        status: 'healthy',
+        activeWorkers,
+        totalWorkers: stats.total,
+        percentage: (activeWorkers / stats.total) * 100
+      };
+    });
+    
+    // Memory usage check
+    this.registerCheck('memory_usage', async () => {
+      const memoryUsage = process.memoryUsage();
+      const memoryRatio = memoryUsage.heapUsed / memoryUsage.heapTotal;
+      
+      if (memoryRatio > this.options.memoryThreshold) {
+        throw new Error(`High memory usage: ${(memoryRatio * 100).toFixed(1)}% (threshold: ${this.options.memoryThreshold * 100}%)`);
+      }
+      
+      return {
+        status: 'healthy',
+        heapUsed: memoryUsage.heapUsed,
+        heapTotal: memoryUsage.heapTotal,
+        ratio: memoryRatio,
+        threshold: this.options.memoryThreshold
+      };
+    });
+    
+    // Error rate check
+    this.registerCheck('error_rate', async () => {
+      const stats = this.clusterManager.getStats();
+      const errorRate = stats.totalRequests > 0 ? (stats.errors / stats.totalRequests) * 100 : 0;
+      
+      if (errorRate > this.options.maxErrorRate) {
+        throw new Error(`High error rate: ${errorRate.toFixed(2)}% (threshold: ${this.options.maxErrorRate}%)`);
+      }
+      
+      return {
+        status: 'healthy',
+        totalRequests: stats.totalRequests,
+        totalErrors: stats.errors,
+        errorRate,
+        threshold: this.options.maxErrorRate
+      };
+    });
+    
+    // Uptime check
+    this.registerCheck('uptime', async () => {
+      const uptime = process.uptime();
+      const stats = this.clusterManager.getStats();
+      
+      if (uptime < 60) {
+        return {
+          status: 'degraded',
+          uptime,
+          message: 'System recently started'
+        };
+      }
+      
+      return {
+        status: 'healthy',
+        uptime,
+        clusterUptime: stats.uptime
+      };
+    });
+    
+    // Worker health check
+    this.registerCheck('worker_health', async () => {
+      const stats = this.clusterManager.getStats();
+      const unhealthyWorkers = [];
+      
+      for (const worker of stats.workers) {
+        // Check if worker has been active recently
+        const idleTime = worker.lastRequestTime ? Date.now() - worker.lastRequestTime : Infinity;
+        
+        if (idleTime > 300000) { // 5 minutes
+          unhealthyWorkers.push({
+            pid: worker.pid,
+            issue: 'Worker idle for too long',
+            idleTime: Math.floor(idleTime / 1000)
+          });
+        }
+        
+        // Check error rate for individual workers
+        const workerErrorRate = worker.requests > 0 ? (worker.errors / worker.requests) * 100 : 0;
+        if (workerErrorRate > 20) { // 20% error rate threshold
+          unhealthyWorkers.push({
+            pid: worker.pid,
+            issue: 'High error rate',
+            errorRate: workerErrorRate.toFixed(2)
+          });
+        }
+      }
+      
+      if (unhealthyWorkers.length > 0) {
+        return {
+          status: 'degraded',
+          unhealthyWorkers,
+          totalWorkers: stats.workers.length,
+          message: `${unhealthyWorkers.length} workers have issues`
+        };
+      }
+      
+      return {
+        status: 'healthy',
+        totalWorkers: stats.workers.length,
+        message: 'All workers healthy'
+      };
+    });
+  }
+  
+  /**
+   * Register a custom health check
+   * @param {string} name - Check name
+   * @param {Function} checkFn - Check function returning Promise
+   * @param {Object} options - Check options
+   */
+  registerCheck(name, checkFn, options = {}) {
+    this.healthChecks.set(name, {
+      fn: checkFn,
+      options: {
+        timeout: this.options.timeout,
+        ...options
+      },
+      lastRun: null,
+      lastResult: null,
+      history: []
+    });
+    
+    console.log(`✅ Registered health check: ${name}`);
+  }
+  
+  /**
+   * Unregister a health check
+   * @param {string} name - Check name
+   * @returns {boolean} Success status
+   */
+  unregisterCheck(name) {
+    return this.healthChecks.delete(name);
+  }
+  
+  /**
+   * Run a specific health check
+   * @param {string} name - Check name
+   * @returns {Promise<Object>} Check result
+   */
+  async runCheck(name) {
+    if (!this.healthChecks.has(name)) {
+      throw new Error(`Health check "${name}" not found`);
+    }
+    
+    const check = this.healthChecks.get(name);
+    const startTime = Date.now();
+    
+    try {
+      // Run check with timeout
+      const result = await Promise.race([
+        check.fn(),
+        new Promise((_, reject) => 
+          setTimeout(() => reject(new Error(`Health check "${name}" timeout`)), check.options.timeout)
+        )
+      ]);
+      
+      const endTime = Date.now();
+      const duration = endTime - startTime;
+      
+      const checkResult = {
+        name,
+        status: 'healthy',
+        duration,
+        timestamp: new Date().toISOString(),
+        data: result
+      };
+      
+      // Update check history
+      check.lastRun = endTime;
+      check.lastResult = checkResult;
+      check.history.push(checkResult);
+      
+      // Limit history size
+      if (check.history.length > this.maxHistorySize) {
+        check.history = check.history.slice(-this.maxHistorySize);
+      }
+      
+      this.emit('check:completed', checkResult);
+      return checkResult;
+      
+    } catch (error) {
+      const endTime = Date.now();
+      const duration = endTime - startTime;
+      
+      const checkResult = {
+        name,
+        status: 'unhealthy',
+        duration,
+        timestamp: new Date().toISOString(),
+        error: error.message,
+        stack: error.stack
+      };
+      
+      // Update check history
+      check.lastRun = endTime;
+      check.lastResult = checkResult;
+      check.history.push(checkResult);
+      
+      // Limit history size
+      if (check.history.length > this.maxHistorySize) {
+        check.history = check.history.slice(-this.maxHistorySize);
+      }
+      
+      this.emit('check:failed', checkResult);
+      return checkResult;
+    }
+  }
+  
+  /**
+   * Run all health checks
+   * @returns {Promise<Object>} Overall health status
+   */
+  async runAllChecks() {
+    const startTime = Date.now();
+    this.lastCheckTime = startTime;
+    
+    const checkPromises = [];
+    const checkNames = Array.from(this.healthChecks.keys());
+    
+    // Run all checks in parallel
+    for (const name of checkNames) {
+      checkPromises.push(this.runCheck(name));
+    }
+    
+    const results = await Promise.allSettled(checkPromises);
+    const endTime = Date.now();
+    const duration = endTime - startTime;
+    
+    // Process results
+    const checkResults = [];
+    let healthyChecks = 0;
+    let unhealthyChecks = 0;
+    let warnings = [];
+    let errors = [];
+    
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i];
+      const name = checkNames[i];
+      
+      if (result.status === 'fulfilled') {
+        checkResults.push(result.value);
+        
+        if (result.value.status === 'healthy') {
+          healthyChecks++;
+        } else {
+          unhealthyChecks++;
+          warnings.push(`${name}: ${result.value.error}`);
+        }
+      } else {
+        // Check failed to run
+        unhealthyChecks++;
+        errors.push(`${name}: ${result.reason.message}`);
+        
+        checkResults.push({
+          name,
+          status: 'unhealthy',
+          duration: 0,
+          timestamp: new Date().toISOString(),
+          error: result.reason.message
+        });
+      }
+    }
+    
+    // Determine overall status
+    let overallStatus = 'healthy';
+    if (unhealthyChecks > 0) {
+      overallStatus = 'degraded';
+    }
+    if (unhealthyChecks > checkNames.length * 0.5) {
+      overallStatus = 'unhealthy';
+    }
+    
+    const healthReport = {
+      status: overallStatus,
+      timestamp: new Date().toISOString(),
+      duration,
+      checks: {
+        total: checkNames.length,
+        healthy: healthyChecks,
+        unhealthy: unhealthyChecks,
+        results: checkResults
+      },
+      warnings: warnings.length > 0 ? warnings : null,
+      errors: errors.length > 0 ? errors : null,
+      recommendations: this.getRecommendations(checkResults)
+    };
+    
+    // Add to history
+    this.healthHistory.push({
+      timestamp: healthReport.timestamp,
+      status: healthReport.status,
+      healthyChecks,
+      unhealthyChecks,
+      duration
+    });
+    
+    // Limit history size
+    if (this.healthHistory.length > this.maxHistorySize) {
+      this.healthHistory = this.healthHistory.slice(-this.maxHistorySize);
+    }
+    
+    this.emit('health:report', healthReport);
+    return healthReport;
+  }
+  
+  /**
+   * Get health check recommendations
+   * @param {Array} checkResults - Array of check results
+   * @returns {Array} Recommendations
+   */
+  getRecommendations(checkResults) {
+    const recommendations = [];
+    
+    for (const result of checkResults) {
+      if (result.status !== 'healthy') {
+        switch (result.name) {
+          case 'worker_availability':
+            recommendations.push('Consider restarting inactive workers or increasing worker count');
+            break;
+          case 'memory_usage':
+            recommendations.push('Monitor memory usage, consider optimizing memory or scaling horizontally');
+            break;
+          case 'error_rate':
+            recommendations.push('Investigate high error rate, check application logs and error handling');
+            break;
+          case 'uptime':
+            if (result.data && result.data.status === 'degraded') {
+              recommendations.push('System recently started, monitor for stability');
+            }
+            break;
+          case 'worker_health':
+            if (result.data && result.data.unhealthyWorkers) {
+              recommendations.push(`Restart ${result.data.unhealthyWorkers.length} unhealthy workers`);
+            }
+            break;
+        }
+      }
+    }
+    
+    return recommendations.length > 0 ? recommendations : ['All systems operational'];
+  }
+  
+  /**
+   * Start periodic health checks
+   */
+  start() {
+    if (this.checkInterval) {
+      console.warn('Health monitor already started');
+      return;
+    }
+    
+    console.log('🚀 Starting health monitor...');
+    
+    // Run initial check
+    this.runAllChecks().catch(error => {
+      console.error('Initial health check failed:', error);
+    });
+    
+    // Start periodic checks
+    this.checkInterval = setInterval(() => {
+      this.runAllChecks().catch(error => {
+        console.error('Periodic health check failed:', error);
+      });
+    }, this.options.checkInterval);
+    
+    console.log(`✅ Health monitor started with ${this.options.checkInterval}ms interval`);
+  }
+  
+  /**
+   * Stop periodic health checks
+   */
+  stop() {
+    if (this.checkInterval) {
+      clearInterval(this.checkInterval);
+      this.checkInterval = null;
+      console.log('🛑 Health monitor stopped');
+    }
+  }
+  
+  /**
+   * Get health check handler for HTTP endpoint
+   * @returns {Function} HTTP handler function
+   */
+  getHandler() {
+    return async (ctx) => {
+      try {
+        const health = await this.runAllChecks();
+        
+        // Set response status based on health
+        ctx.status = health.status === 'healthy' ? 200 : 
+                     health.status === 'degraded' ? 200 : 503;
+        
+        // Add health headers
+        ctx.set('X-Health-Status', health.status);
+        ctx.set('X-Health-Checks', `${health.checks.healthy}/${health.checks.total}`);
+        ctx.set('X-Health-Timestamp', health.timestamp);
+        
+        // Return health report
+        ctx.body = {
+          ...health,
+          // Add system information
+          system: {
+            pid: process.pid,
+            uptime: process.uptime(),
+            memory: process.memoryUsage(),
+            cpu: process.cpuUsage(),
+            version: process.version,
+            platform: process.platform,
+            arch: process.arch
+          },
+          // Add cluster information if available
+          cluster: this.clusterManager ? this.clusterManager.getStats() : null
+        };
+        
+      } catch (error) {
+        console.error('Health check handler error:', error);
+        
+        ctx.status = 503;
+        ctx.body = {
+          status: 'error',
+          timestamp: new Date().toISOString(),
+          message: 'Health check failed',
+          error: error.message,
+          stack: process.env.NODE_ENV === 'development' ? error.stack : undefined
+        };
+      }
+    };
+  }
+  
+  /**
+   * Get detailed health information
+   * @returns {Object} Detailed health information
+   */
+  getDetailedHealth() {
+    const checks = [];
+    
+    for (const [name, check] of this.healthChecks.entries()) {
+      checks.push({
+        name,
+        lastRun: check.lastRun ? new Date(check.lastRun).toISOString() : null,
+        lastResult: check.lastResult,
+        historySize: check.history.length,
+        options: check.options
+      });
+    }
+    
+    return {
+      checks,
+      history: {
+        records: this.healthHistory.length,
+        latest: this.healthHistory[this.healthHistory.length - 1] || null,
+        summary: {
+          totalChecks: this.healthHistory.length,
+          healthyChecks: this.healthHistory.filter(h => h.status === 'healthy').length,
+          degradedChecks: this.healthHistory.filter(h => h.status === 'degraded').length,
+          unhealthyChecks: this.healthHistory.filter(h => h.status === 'unhealthy').length
+        }
+      },
+      config: this.options,
+      lastCheck: this.lastCheckTime ? new Date(this.lastCheckTime).toISOString() : null
+    };
+  }
+  
+  /**
+   * Get health check status
+   * @param {string} name - Check name
+   * @returns {Object|null} Check status
+   */
+  getCheckStatus(name) {
+    if (!this.healthChecks.has(name)) {
+      return null;
+    }
+    
+    const check = this.healthChecks.get(name);
+    return {
+      name,
+      lastRun: check.lastRun,
+      lastResult: check.lastResult,
+      history: check.history.slice(-10), // Last 10 results
+      options: check.options
+    };
+  }
+  
+  /**
+   * Get all check statuses
+   * @returns {Array} Array of check statuses
+   */
+  getAllCheckStatuses() {
+    const statuses = [];
+    
+    for (const [name] of this.healthChecks.entries()) {
+      const status = this.getCheckStatus(name);
+      if (status) {
+        statuses.push(status);
+      }
+    }
+    
+    return statuses;
+  }
+  
+  /**
+   * Reset health check history
+   */
+  resetHistory() {
+    this.healthHistory = [];
+    
+    for (const [name, check] of this.healthChecks.entries()) {
+      check.history = [];
+      check.lastRun = null;
+      check.lastResult = null;
+    }
+    
+    console.log('🔄 Health check history reset');
+  }
+}
+
+export default HealthMonitor;