aetherframework-cluster 1.0.0

package/src/middleware/cluster-health.js

@@ -0,0 +1,330 @@
+// packages/cluster/src/middleware/cluster-health.js
+/**
+ * Cluster Health Middleware - Provides health check endpoints for cluster monitoring
+ * This middleware adds health check routes to monitor cluster status
+ */
+export default function createClusterHealthMiddleware(clusterManager, options = {}) {
+  const config = {
+    path: options.path || '/cluster/health',
+    detailed: options.detailed !== false,
+    auth: options.auth || null,
+    ...options
+  };
+  
+  /**
+   * Health check middleware function
+   * @param {Object} ctx - Context object
+   * @param {Function} next - Next middleware function
+   */
+  return async function clusterHealthMiddleware(ctx, next) {
+    // Only handle health check route
+    if (ctx.path !== config.path) {
+      return next();
+    }
+    
+    // Check authentication if required
+    if (config.auth && !checkAuth(ctx, config.auth)) {
+      ctx.status = 401;
+      ctx.body = {
+        status: 'error',
+        message: 'Unauthorized',
+        timestamp: new Date().toISOString()
+      };
+      return;
+    }
+    
+    try {
+      // Get cluster health status
+      const health = await getClusterHealth(clusterManager, config.detailed);
+      
+      // Set response status based on health
+      ctx.status = health.status === 'healthy' ? 200 : 
+                   health.status === 'degraded' ? 200 : 503;
+      
+      // Add health headers
+      ctx.set('X-Cluster-Health', health.status);
+      ctx.set('X-Cluster-Workers', `${health.workers.healthy}/${health.workers.total}`);
+      ctx.set('X-Cluster-Uptime', `${health.uptime}s`);
+      ctx.set('X-Cluster-Timestamp', health.timestamp);
+      
+      // Return health response
+      ctx.body = health;
+      
+    } catch (error) {
+      console.error('Health check error:', error);
+      
+      ctx.status = 503;
+      ctx.body = {
+        status: 'error',
+        timestamp: new Date().toISOString(),
+        message: 'Health check failed',
+        error: error.message,
+        stack: process.env.NODE_ENV === 'development' ? error.stack : undefined
+      };
+    }
+  };
+}
+
+/**
+ * Get cluster health status
+ * @param {Object} clusterManager - Cluster manager instance
+ * @param {boolean} detailed - Whether to include detailed information
+ * @returns {Promise<Object>} Health status object
+ */
+async function getClusterHealth(clusterManager, detailed = false) {
+  if (!clusterManager) {
+    return {
+      status: 'unknown',
+      message: 'Cluster manager not available',
+      timestamp: new Date().toISOString()
+    };
+  }
+  
+  const stats = clusterManager.getStats();
+  const now = Date.now();
+  
+  // Calculate health metrics
+  const activeWorkers = stats.workers.filter(w => w.state === 'online').length;
+  const totalWorkers = stats.workers.length;
+  const healthPercentage = totalWorkers > 0 ? (activeWorkers / totalWorkers) * 100 : 0;
+  
+  let status = 'healthy';
+  let issues = [];
+  let warnings = [];
+  
+  // Critical issues
+  if (activeWorkers === 0) {
+    status = 'unhealthy';
+    issues.push('No active workers available');
+  }
+  
+  if (totalWorkers === 0) {
+    status = 'unhealthy';
+    issues.push('No workers registered in cluster');
+  }
+  
+  // Warnings
+  if (activeWorkers < totalWorkers * 0.5) {
+    status = 'degraded';
+    warnings.push(`Less than 50% of workers active: ${activeWorkers}/${totalWorkers}`);
+  }
+  
+  // Check error rate
+  const errorRate = stats.totalRequests > 0 ? (stats.errors / stats.totalRequests) * 100 : 0;
+  if (errorRate > 10) {
+    status = 'degraded';
+    warnings.push(`High error rate: ${errorRate.toFixed(2)}%`);
+  }
+  
+  // Check for idle workers
+  const idleWorkers = stats.workers.filter(w => {
+    const idleTime = w.lastRequestTime ? now - w.lastRequestTime : Infinity;
+    return idleTime > 300000; // 5 minutes
+  }).length;
+  
+  if (idleWorkers > totalWorkers * 0.7) {
+    warnings.push(`High idle worker ratio: ${idleWorkers}/${totalWorkers}`);
+  }
+  
+  // Build health report
+  const healthReport = {
+    status,
+    timestamp: new Date().toISOString(),
+    uptime: Math.floor((now - stats.startTime) / 1000),
+    workers: {
+      total: totalWorkers,
+      active: activeWorkers,
+      idle: idleWorkers,
+      dead: totalWorkers - activeWorkers,
+      healthPercentage: healthPercentage.toFixed(2)
+    },
+    performance: {
+      totalRequests: stats.totalRequests,
+      totalErrors: stats.errors,
+      errorRate: errorRate.toFixed(2),
+      workerRestarts: stats.workerRestarts
+    },
+    issues: issues.length > 0 ? issues : null,
+    warnings: warnings.length > 0 ? warnings : null,
+    recommendations: getHealthRecommendations(stats, activeWorkers, totalWorkers, errorRate)
+  };
+  
+  // Add detailed information if requested
+  if (detailed) {
+    healthReport.detailed = {
+      workers: stats.workers.map(w => ({
+        pid: w.pid,
+        id: w.id,
+        state: w.state,
+        uptime: Math.floor((now - w.startTime) / 1000),
+        requests: w.requests,
+        errors: w.errors,
+        errorRate: w.requests > 0 ? (w.errors / w.requests) * 100 : 0,
+        lastRequestTime: w.lastRequestTime
+      })),
+      system: {
+        memory: process.memoryUsage(),
+        cpu: process.cpuUsage(),
+        platform: process.platform,
+        arch: process.arch,
+        version: process.version
+      }
+    };
+  }
+  
+  return healthReport;
+}
+
+/**
+ * Get health recommendations based on cluster status
+ * @param {Object} stats - Cluster statistics
+ * @param {number} activeWorkers - Number of active workers
+ * @param {number} totalWorkers - Total number of workers
+ * @param {number} errorRate - Error rate percentage
+ * @returns {Array} Array of recommendations
+ */
+function getHealthRecommendations(stats, activeWorkers, totalWorkers, errorRate) {
+  const recommendations = [];
+  
+  // Check idle worker ratio
+  const idleRatio = (totalWorkers - activeWorkers) / totalWorkers;
+  if (idleRatio > 0.7) {
+    recommendations.push('Consider reducing the number of workers to save resources');
+  }
+  
+  // Check if all workers are active and fewer than CPU cores
+  const cpuCount = require('os').cpus().length;
+  if (activeWorkers === totalWorkers && totalWorkers < cpuCount) {
+    recommendations.push(`Consider increasing the number of workers to ${cpuCount} (CPU cores)`);
+  }
+  
+  // Check error rate
+  if (errorRate > 10) {
+    recommendations.push('Investigate high error rate, check application logs and error handling');
+  }
+  
+  // Check request distribution
+  const averageRequests = stats.totalRequests / Math.max(totalWorkers, 1);
+  if (averageRequests > 1000) {
+    recommendations.push('High request load, consider scaling horizontally');
+  }
+  
+  // Check worker restarts
+  if (stats.workerRestarts > 0) {
+    recommendations.push(`Monitor worker stability (${stats.workerRestarts} restarts detected)`);
+  }
+  
+  return recommendations.length > 0 ? recommendations : ['All systems operational'];
+}
+
+/**
+ * Check authentication for health endpoint
+ * @param {Object} ctx - Context object
+ * @param {Object|Function} auth - Authentication configuration or function
+ * @returns {boolean} Authentication status
+ */
+function checkAuth(ctx, auth) {
+  if (typeof auth === 'function') {
+    return auth(ctx);
+  }
+  
+  if (typeof auth === 'object') {
+    // Check API key
+    if (auth.apiKey) {
+      const apiKey = ctx.headers['x-api-key'] || ctx.query.apiKey;
+      return apiKey === auth.apiKey;
+    }
+    
+    // Check basic auth
+    if (auth.username && auth.password) {
+      const authHeader = ctx.headers.authorization;
+      if (!authHeader || !authHeader.startsWith('Basic ')) {
+        return false;
+      }
+      
+      const credentials = Buffer.from(authHeader.slice(6), 'base64').toString();
+      const [username, password] = credentials.split(':');
+      return username === auth.username && password === auth.password;
+    }
+  }
+  
+  return true; // No auth required
+}
+
+/**
+ * Create detailed health check endpoint
+ * @param {Object} clusterManager - Cluster manager instance
+ * @param {Object} options - Middleware options
+ * @returns {Function} Middleware function
+ */
+export function createDetailedHealthMiddleware(clusterManager, options = {}) {
+  return async function detailedHealthMiddleware(ctx, next) {
+    if (ctx.path !== '/cluster/health/detailed') {
+      return next();
+    }
+    
+    try {
+      const health = await getClusterHealth(clusterManager, true);
+      
+      // Add load balancer stats if available
+      if (clusterManager.loadBalancer) {
+        health.loadBalancer = clusterManager.loadBalancer.getStats();
+      }
+      
+      // Add worker manager stats if available
+      if (clusterManager.workerManager) {
+        health.workerManager = clusterManager.workerManager.getClusterStats();
+      }
+      
+      ctx.status = health.status === 'healthy' ? 200 : 
+                   health.status === 'degraded' ? 200 : 503;
+      ctx.body = health;
+      
+    } catch (error) {
+      ctx.status = 503;
+      ctx.body = {
+        status: 'error',
+        timestamp: new Date().toISOString(),
+        message: 'Detailed health check failed',
+        error: error.message
+      };
+    }
+  };
+}
+
+/**
+ * Create health check endpoint with custom path
+ * @param {Object} clusterManager - Cluster manager instance
+ * @param {string} path - Custom path for health endpoint
+ * @param {Object} options - Additional options
+ * @returns {Function} Middleware function
+ */
+export function createCustomHealthMiddleware(clusterManager, path, options = {}) {
+  return async function customHealthMiddleware(ctx, next) {
+    if (ctx.path !== path) {
+      return next();
+    }
+    
+    try {
+      const health = await getClusterHealth(clusterManager, options.detailed || false);
+      
+      // Add custom metrics if provided
+      if (options.metrics) {
+        health.customMetrics = await options.metrics(ctx);
+      }
+      
+      ctx.status = health.status === 'healthy' ? 200 : 
+                   health.status === 'degraded' ? 200 : 503;
+      ctx.body = health;
+      
+    } catch (error) {
+      ctx.status = 503;
+      ctx.body = {
+        status: 'error',
+        timestamp: new Date().toISOString(),
+        message: 'Health check failed',
+        error: error.message
+      };
+    }
+  };
+}