aetherframework-cluster 1.0.0

package/src/core/WorkerManager.js

@@ -0,0 +1,619 @@
+// packages/cluster/src/core/WorkerManager.js
+import { EventEmitter } from 'events';
+
+/**
+ * Worker Manager - Manages worker processes and their lifecycle
+ * Tracks worker statistics, load balancing, and health status
+ */
+class WorkerManager extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    
+    this.options = {
+      maxWorkers: options.maxWorkers || 10,
+      minWorkers: options.minWorkers || 1,
+      workerTimeout: options.workerTimeout || 30000,
+      maxRestarts: options.maxRestarts || 5,
+      restartDelay: options.restartDelay || 1000,
+      ...options
+    };
+    
+    this.workers = new Map();
+    this.workerStats = new Map();
+    this.workerLoad = new Map();
+    this.restartCounts = new Map();
+    this.workerStartTime = new Map();
+    
+    // Performance metrics
+    this.metrics = {
+      totalRequests: 0,
+      totalErrors: 0,
+      totalRestarts: 0,
+      startTime: Date.now()
+    };
+  }
+  
+  /**
+   * Register a new worker
+   * @param {number} pid - Process ID
+   * @param {Object} worker - Worker object
+   * @param {Object} metadata - Worker metadata
+   * @returns {number} Worker ID
+   */
+  registerWorker(pid, worker, metadata = {}) {
+    const workerId = this.generateWorkerId();
+    
+    this.workers.set(pid, {
+      ...worker,
+      id: workerId,
+      pid,
+      metadata,
+      state: 'starting',
+      startTime: Date.now(),
+      lastHeartbeat: Date.now()
+    });
+    
+    this.workerStats.set(pid, {
+      pid,
+      workerId,
+      requests: 0,
+      errors: 0,
+      memoryUsage: null,
+      cpuUsage: null,
+      uptime: 0,
+      lastRequestTime: null,
+      lastError: null,
+      state: 'starting'
+    });
+    
+    this.workerLoad.set(pid, 0);
+    this.restartCounts.set(pid, 0);
+    this.workerStartTime.set(pid, Date.now());
+    
+    console.log(`✅ Worker ${pid} (ID: ${workerId}) registered`);
+    this.emit('worker:registered', { pid, workerId, metadata });
+    
+    return workerId;
+  }
+  
+  /**
+   * Unregister a worker
+   * @param {number} pid - Process ID
+   * @returns {boolean} Success status
+   */
+  unregisterWorker(pid) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const worker = this.workers.get(pid);
+    const stats = this.workerStats.get(pid);
+    
+    this.workers.delete(pid);
+    this.workerStats.delete(pid);
+    this.workerLoad.delete(pid);
+    this.restartCounts.delete(pid);
+    this.workerStartTime.delete(pid);
+    
+    console.log(`🗑️ Worker ${pid} (ID: ${worker.id}) unregistered`);
+    this.emit('worker:unregistered', { pid, workerId: worker.id, stats });
+    
+    return true;
+  }
+  
+  /**
+   * Update worker state
+   * @param {number} pid - Process ID
+   * @param {string} state - New state
+   * @param {Object} data - Additional data
+   */
+  updateWorkerState(pid, state, data = {}) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const worker = this.workers.get(pid);
+    const oldState = worker.state;
+    
+    worker.state = state;
+    worker.lastUpdate = Date.now();
+    
+    if (state === 'ready') {
+      worker.readyTime = Date.now();
+    }
+    
+    // Update stats
+    if (this.workerStats.has(pid)) {
+      const stats = this.workerStats.get(pid);
+      stats.state = state;
+      stats.lastUpdate = Date.now();
+      
+      if (data.memoryUsage) {
+        stats.memoryUsage = data.memoryUsage;
+      }
+      
+      if (data.cpuUsage) {
+        stats.cpuUsage = data.cpuUsage;
+      }
+      
+      if (data.uptime) {
+        stats.uptime = data.uptime;
+      }
+    }
+    
+    console.log(`🔄 Worker ${pid} state changed: ${oldState} -> ${state}`);
+    this.emit('worker:stateChanged', { pid, oldState, newState: state, data });
+    
+    return true;
+  }
+  
+  /**
+   * Record a request for a worker
+   * @param {number} pid - Process ID
+   * @param {Object} requestInfo - Request information
+   */
+  recordRequest(pid, requestInfo = {}) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const worker = this.workers.get(pid);
+    const stats = this.workerStats.get(pid);
+    
+    if (stats) {
+      stats.requests++;
+      stats.lastRequestTime = Date.now();
+      stats.state = 'active';
+      
+      // Update load
+      const currentLoad = this.workerLoad.get(pid) || 0;
+      this.workerLoad.set(pid, currentLoad + 1);
+      
+      // Update global metrics
+      this.metrics.totalRequests++;
+      
+      // Auto-decrease load after timeout
+      setTimeout(() => {
+        const load = this.workerLoad.get(pid);
+        if (load > 0) {
+          this.workerLoad.set(pid, load - 1);
+        }
+      }, this.options.workerTimeout);
+    }
+    
+    this.emit('worker:request', { pid, requestInfo, stats });
+    return true;
+  }
+  
+  /**
+   * Record an error for a worker
+   * @param {number} pid - Process ID
+   * @param {Error} error - Error object
+   * @param {Object} context - Error context
+   */
+  recordError(pid, error, context = {}) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const stats = this.workerStats.get(pid);
+    
+    if (stats) {
+      stats.errors++;
+      stats.lastError = {
+        message: error.message,
+        stack: error.stack,
+        timestamp: Date.now(),
+        context
+      };
+      
+      // Update global metrics
+      this.metrics.totalErrors++;
+    }
+    
+    console.error(`❌ Worker ${pid} error:`, error.message);
+    this.emit('worker:error', { pid, error, context, stats });
+    
+    return true;
+  }
+  
+  /**
+   * Update worker heartbeat
+   * @param {number} pid - Process ID
+   * @returns {boolean} Success status
+   */
+  updateHeartbeat(pid) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const worker = this.workers.get(pid);
+    worker.lastHeartbeat = Date.now();
+    
+    return true;
+  }
+  
+  /**
+   * Check if a worker is alive
+   * @param {number} pid - Process ID
+   * @returns {boolean} Alive status
+   */
+  isWorkerAlive(pid) {
+    if (!this.workers.has(pid)) {
+      return false;
+    }
+    
+    const worker = this.workers.get(pid);
+    const now = Date.now();
+    const lastHeartbeat = worker.lastHeartbeat || 0;
+    
+    return (now - lastHeartbeat) < this.options.workerTimeout;
+  }
+  
+  /**
+   * Get the least loaded worker
+   * @returns {number|null} PID of least loaded worker
+   */
+  getLeastLoadedWorker() {
+    let minLoad = Infinity;
+    let leastLoadedWorker = null;
+    
+    for (const [pid, load] of this.workerLoad.entries()) {
+      const worker = this.workers.get(pid);
+      const stats = this.workerStats.get(pid);
+      
+      // Only consider ready and alive workers
+      if (worker && worker.state === 'ready' && this.isWorkerAlive(pid)) {
+        if (load < minLoad) {
+          minLoad = load;
+          leastLoadedWorker = pid;
+        }
+      }
+    }
+    
+    return leastLoadedWorker;
+  }
+  
+  /**
+   * Get worker statistics
+   * @param {number} pid - Process ID
+   * @returns {Object|null} Worker statistics
+   */
+  getWorkerStats(pid) {
+    if (!this.workers.has(pid)) {
+      return null;
+    }
+    
+    const worker = this.workers.get(pid);
+    const stats = this.workerStats.get(pid);
+    const load = this.workerLoad.get(pid) || 0;
+    const restartCount = this.restartCounts.get(pid) || 0;
+    const startTime = this.workerStartTime.get(pid) || Date.now();
+    
+    const uptime = Date.now() - startTime;
+    const errorRate = stats.requests > 0 ? (stats.errors / stats.requests) * 100 : 0;
+    
+    return {
+      pid,
+      workerId: worker.id,
+      state: worker.state,
+      uptime: Math.floor(uptime / 1000), // seconds
+      requests: stats.requests,
+      errors: stats.errors,
+      errorRate: errorRate.toFixed(2),
+      load,
+      memoryUsage: stats.memoryUsage,
+      cpuUsage: stats.cpuUsage,
+      lastRequestTime: stats.lastRequestTime,
+      lastHeartbeat: worker.lastHeartbeat,
+      isAlive: this.isWorkerAlive(pid),
+      restartCount,
+      metadata: worker.metadata,
+      lastError: stats.lastError
+    };
+  }
+  
+  /**
+   * Get all worker statistics
+   * @returns {Array} Array of worker statistics
+   */
+  getAllWorkerStats() {
+    const allStats = [];
+    
+    for (const pid of this.workers.keys()) {
+      const stats = this.getWorkerStats(pid);
+      if (stats) {
+        allStats.push(stats);
+      }
+    }
+    
+    return allStats;
+  }
+  
+  /**
+   * Get cluster statistics
+   * @returns {Object} Cluster statistics
+   */
+  getClusterStats() {
+    const allStats = this.getAllWorkerStats();
+    const now = Date.now();
+    
+    const totalWorkers = allStats.length;
+    const activeWorkers = allStats.filter(w => w.state === 'ready' || w.state === 'active').length;
+    const idleWorkers = allStats.filter(w => w.state === 'idle').length;
+    const deadWorkers = allStats.filter(w => !w.isAlive).length;
+    
+    const totalRequests = allStats.reduce((sum, w) => sum + w.requests, 0);
+    const totalErrors = allStats.reduce((sum, w) => sum + w.errors, 0);
+    const totalLoad = allStats.reduce((sum, w) => sum + w.load, 0);
+    
+    const averageRequestsPerWorker = totalWorkers > 0 ? totalRequests / totalWorkers : 0;
+    const averageLoadPerWorker = totalWorkers > 0 ? totalLoad / totalWorkers : 0;
+    const clusterErrorRate = totalRequests > 0 ? (totalErrors / totalRequests) * 100 : 0;
+    
+    const leastLoadedWorker = this.getLeastLoadedWorker();
+    
+    return {
+      timestamp: new Date().toISOString(),
+      uptime: Math.floor((now - this.metrics.startTime) / 1000),
+      workers: {
+        total: totalWorkers,
+        active: activeWorkers,
+        idle: idleWorkers,
+        dead: deadWorkers,
+        byState: this.groupWorkersByState(allStats)
+      },
+      performance: {
+        totalRequests,
+        totalErrors,
+        totalRestarts: this.metrics.totalRestarts,
+        errorRate: clusterErrorRate.toFixed(2),
+        averageRequestsPerWorker: averageRequestsPerWorker.toFixed(2),
+        averageLoadPerWorker: averageLoadPerWorker.toFixed(2)
+      },
+      loadBalancing: {
+        leastLoadedWorker,
+        totalLoad,
+        averageLoad: averageLoadPerWorker.toFixed(2)
+      },
+      health: this.getClusterHealth(allStats)
+    };
+  }
+  
+  /**
+   * Group workers by state
+   * @param {Array} workers - Array of worker stats
+   * @returns {Object} Workers grouped by state
+   */
+  groupWorkersByState(workers) {
+    const groups = {
+      starting: [],
+      ready: [],
+      active: [],
+      idle: [],
+      stopping: [],
+      dead: []
+    };
+    
+    workers.forEach(worker => {
+      if (groups[worker.state]) {
+        groups[worker.state].push(worker.pid);
+      }
+    });
+    
+    return groups;
+  }
+  
+  /**
+   * Get cluster health status
+   * @param {Array} workers - Array of worker stats
+   * @returns {Object} Health status
+   */
+  getClusterHealth(workers) {
+    const totalWorkers = workers.length;
+    const healthyWorkers = workers.filter(w => w.isAlive && w.errorRate < 10).length;
+    const unhealthyWorkers = workers.filter(w => !w.isAlive || w.errorRate >= 10).length;
+    
+    const healthPercentage = totalWorkers > 0 ? (healthyWorkers / totalWorkers) * 100 : 0;
+    
+    let status = 'healthy';
+    let issues = [];
+    
+    if (healthPercentage < 50) {
+      status = 'critical';
+      issues.push('Less than 50% of workers are healthy');
+    } else if (healthPercentage < 80) {
+      status = 'degraded';
+      issues.push('Less than 80% of workers are healthy');
+    }
+    
+    // Check for high error rates
+    const highErrorWorkers = workers.filter(w => w.errorRate >= 10);
+    if (highErrorWorkers.length > 0) {
+      issues.push(`${highErrorWorkers.length} workers have error rate >= 10%`);
+    }
+    
+    // Check for dead workers
+    const deadWorkers = workers.filter(w => !w.isAlive);
+    if (deadWorkers.length > 0) {
+      issues.push(`${deadWorkers.length} workers are not responding`);
+    }
+    
+    return {
+      status,
+      healthPercentage: healthPercentage.toFixed(2),
+      healthyWorkers,
+      unhealthyWorkers,
+      totalWorkers,
+      issues: issues.length > 0 ? issues : ['All systems operational'],
+      timestamp: new Date().toISOString()
+    };
+  }
+  
+  /**
+   * Increment restart count for a worker
+   * @param {number} pid - Process ID
+   * @returns {number} New restart count
+   */
+  incrementRestartCount(pid) {
+    const currentCount = this.restartCounts.get(pid) || 0;
+    const newCount = currentCount + 1;
+    
+    this.restartCounts.set(pid, newCount);
+    this.metrics.totalRestarts++;
+    
+    return newCount;
+  }
+  
+  /**
+   * Check if a worker should be restarted
+   * @param {number} pid - Process ID
+   * @returns {boolean} Whether to restart
+   */
+  shouldRestartWorker(pid) {
+    const restartCount = this.restartCounts.get(pid) || 0;
+    return restartCount < this.options.maxRestarts;
+  }
+  
+  /**
+   * Get restart count for a worker
+   * @param {number} pid - Process ID
+   * @returns {number} Restart count
+   */
+  getRestartCount(pid) {
+    return this.restartCounts.get(pid) || 0;
+  }
+  
+  /**
+   * Reset restart count for a worker
+   * @param {number} pid - Process ID
+   */
+  resetRestartCount(pid) {
+    this.restartCounts.set(pid, 0);
+  }
+  
+  /**
+   * Generate a unique worker ID
+   * @returns {string} Worker ID
+   */
+  generateWorkerId() {
+    return `worker_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+  
+  /**
+   * Clean up dead workers
+   * @returns {Array} Array of cleaned up worker PIDs
+   */
+  cleanupDeadWorkers() {
+    const deadWorkers = [];
+    
+    for (const [pid, worker] of this.workers.entries()) {
+      if (!this.isWorkerAlive(pid)) {
+        deadWorkers.push(pid);
+        this.unregisterWorker(pid);
+      }
+    }
+    
+    return deadWorkers;
+  }
+  
+  /**
+   * Get worker load
+   * @param {number} pid - Process ID
+   * @returns {number} Current load
+   */
+  getWorkerLoad(pid) {
+    return this.workerLoad.get(pid) || 0;
+  }
+  
+  /**
+   * Set worker load
+   * @param {number} pid - Process ID
+   * @param {number} load - New load value
+   */
+  setWorkerLoad(pid, load) {
+    this.workerLoad.set(pid, Math.max(0, load));
+  }
+  
+  /**
+   * Get all worker loads
+   * @returns {Map} Map of worker loads
+   */
+  getAllWorkerLoads() {
+    return new Map(this.workerLoad);
+  }
+  
+  /**
+   * Get worker by ID
+   * @param {string} workerId - Worker ID
+   * @returns {Object|null} Worker object
+   */
+  getWorkerById(workerId) {
+    for (const [pid, worker] of this.workers.entries()) {
+      if (worker.id === workerId) {
+        return { pid, ...worker };
+      }
+    }
+    return null;
+  }
+  
+  /**
+   * Get worker by PID
+   * @param {number} pid - Process ID
+   * @returns {Object|null} Worker object
+   */
+  getWorkerByPid(pid) {
+    const worker = this.workers.get(pid);
+    if (!worker) return null;
+    
+    return { pid, ...worker };
+  }
+  
+  /**
+   * Get all workers
+   * @returns {Array} Array of worker objects
+   */
+  getAllWorkers() {
+    const workers = [];
+    
+    for (const [pid, worker] of this.workers.entries()) {
+      workers.push({ pid, ...worker });
+    }
+    
+    return workers;
+  }
+  
+  /**
+   * Get worker count by state
+   * @param {string} state - Worker state
+   * @returns {number} Count of workers in state
+   */
+  getWorkerCountByState(state) {
+    let count = 0;
+    
+    for (const worker of this.workers.values()) {
+      if (worker.state === state) {
+        count++;
+      }
+    }
+    
+    return count;
+  }
+  
+  /**
+   * Get metrics
+   * @returns {Object} Metrics object
+   */
+  getMetrics() {
+    return {
+      ...this.metrics,
+      uptime: Date.now() - this.metrics.startTime,
+      workerCount: this.workers.size,
+      activeWorkerCount: this.getWorkerCountByState('active'),
+      readyWorkerCount: this.getWorkerCountByState('ready'),
+      totalLoad: Array.from(this.workerLoad.values()).reduce((sum, load) => sum + load, 0)
+    };
+  }
+}
+
+export default WorkerManager;