aetherframework-cluster 1.0.0

package/src/middleware/graceful-shutdown.js

@@ -0,0 +1,443 @@
+// packages/cluster/src/middleware/graceful-shutdown.js
+import { EventEmitter } from 'events';
+
+/**
+ * Graceful Shutdown Middleware - Handles graceful shutdown of the application
+ * Provides endpoints to trigger graceful shutdown and monitors shutdown process
+ */
+class GracefulShutdownMiddleware extends EventEmitter {
+  constructor(clusterManager, options = {}) {
+    super();
+    
+    this.clusterManager = clusterManager;
+    this.options = {
+      path: options.path || '/cluster/shutdown',
+      auth: options.auth || null,
+      timeout: options.timeout || 10000, // 10 seconds
+      forceTimeout: options.forceTimeout || 30000, // 30 seconds
+      ...options
+    };
+    
+    this.isShuttingDown = false;
+    this.shutdownStartTime = null;
+    this.shutdownTimeout = null;
+    this.forceShutdownTimeout = null;
+    this.connections = new Set();
+  }
+  
+  /**
+   * Get middleware function for HTTP framework integration
+   * @returns {Function} Middleware function
+   */
+  middleware() {
+    return async (ctx, next) => {
+      // Handle shutdown endpoint
+      if (ctx.path === this.options.path && ctx.method === 'POST') {
+        await this.handleShutdownRequest(ctx);
+        return;
+      }
+      
+      // Handle status endpoint
+      if (ctx.path === `${this.options.path}/status` && ctx.method === 'GET') {
+        await this.handleShutdownStatus(ctx);
+        return;
+      }
+      
+      // Track active connections during shutdown
+      if (this.isShuttingDown) {
+        ctx.status = 503;
+        ctx.set('Retry-After', '60');
+        ctx.body = {
+          status: 'error',
+          message: 'Service is shutting down',
+          timestamp: new Date().toISOString()
+        };
+        return;
+      }
+      
+      // Track connection
+      const connection = { ctx, startTime: Date.now() };
+      this.connections.add(connection);
+      
+      // Remove connection when response is finished
+      ctx.res.on('finish', () => {
+        this.connections.delete(connection);
+      });
+      
+      try {
+        await next();
+      } finally {
+        this.connections.delete(connection);
+      }
+    };
+  }
+  
+  /**
+   * Handle shutdown request
+   * @param {Object} ctx - Context object
+   */
+  async handleShutdownRequest(ctx) {
+    // Check authentication if required
+    if (this.options.auth && !this.checkAuth(ctx)) {
+      ctx.status = 401;
+      ctx.body = {
+        status: 'error',
+        message: 'Unauthorized',
+        timestamp: new Date().toISOString()
+      };
+      return;
+    }
+    
+    // Check if already shutting down
+    if (this.isShuttingDown) {
+      ctx.status = 409;
+      ctx.body = {
+        status: 'error',
+        message: 'Shutdown already in progress',
+        timestamp: new Date().toISOString(),
+        shutdownStartTime: this.shutdownStartTime
+      };
+      return;
+    }
+    
+    // Parse shutdown options
+    const { force = false, timeout = this.options.timeout, message = 'Shutdown requested' } = ctx.request.body || {};
+    
+    try {
+      // Start graceful shutdown
+      await this.startGracefulShutdown({
+        force,
+        timeout: parseInt(timeout, 10),
+        message
+      });
+      
+      ctx.status = 202;
+      ctx.body = {
+        status: 'accepted',
+        message: 'Graceful shutdown initiated',
+        timestamp: new Date().toISOString(),
+        shutdownStartTime: this.shutdownStartTime,
+        timeout: this.options.timeout,
+        forceTimeout: this.options.forceTimeout,
+        activeConnections: this.connections.size
+      };
+      
+    } catch (error) {
+      ctx.status = 500;
+      ctx.body = {
+        status: 'error',
+        message: 'Failed to initiate shutdown',
+        timestamp: new Date().toISOString(),
+        error: error.message
+      };
+    }
+  }
+  
+  /**
+   * Handle shutdown status request
+   * @param {Object} ctx - Context object
+   */
+  async handleShutdownStatus(ctx) {
+    ctx.status = 200;
+    ctx.body = {
+      status: 'success',
+      timestamp: new Date().toISOString(),
+      shutdown: {
+        isShuttingDown: this.isShuttingDown,
+        startTime: this.shutdownStartTime,
+        elapsedTime: this.shutdownStartTime ? Date.now() - this.shutdownStartTime : 0,
+        activeConnections: this.connections.size,
+        timeout: this.options.timeout,
+        forceTimeout: this.options.forceTimeout
+      },
+      cluster: this.clusterManager ? {
+        totalWorkers: this.clusterManager.workers ? this.clusterManager.workers.size : 0,
+        activeWorkers: this.clusterManager.workers ? 
+          Array.from(this.clusterManager.workers.values()).filter(w => w.state === 'online').length : 0
+      } : null
+    };
+  }
+  
+  /**
+   * Start graceful shutdown process
+   * @param {Object} options - Shutdown options
+   */
+  async startGracefulShutdown(options = {}) {
+    if (this.isShuttingDown) {
+      throw new Error('Shutdown already in progress');
+    }
+    
+    this.isShuttingDown = true;
+    this.shutdownStartTime = Date.now();
+    
+    console.log(`🚨 Starting graceful shutdown: ${options.message}`);
+    this.emit('shutdown:started', {
+      timestamp: this.shutdownStartTime,
+      options
+    });
+    
+    // Set shutdown timeout
+    this.shutdownTimeout = setTimeout(() => {
+      this.handleShutdownTimeout();
+    }, options.timeout || this.options.timeout);
+    
+    // Set force shutdown timeout
+    this.forceShutdownTimeout = setTimeout(() => {
+      this.handleForceShutdown();
+    }, options.force ? 0 : this.options.forceTimeout);
+    
+    try {
+      // Notify cluster manager
+      if (this.clusterManager) {
+        await this.notifyClusterManager(options);
+      }
+      
+      // Wait for active connections to complete
+      await this.waitForConnections();
+      
+      // Perform cleanup
+      await this.performCleanup();
+      
+      // Complete shutdown
+      await this.completeShutdown();
+      
+    } catch (error) {
+      console.error('Graceful shutdown error:', error);
+      this.emit('shutdown:error', { error });
+      
+      // Force shutdown on error
+      this.handleForceShutdown();
+    }
+  }
+  
+  /**
+   * Notify cluster manager about shutdown
+   * @param {Object} options - Shutdown options
+   */
+  async notifyClusterManager(options) {
+    if (!this.clusterManager) {
+      return;
+    }
+    
+    console.log('📤 Notifying cluster manager about shutdown...');
+    
+    // Send shutdown signal to all workers
+    this.clusterManager.workers.forEach((info, pid) => {
+      try {
+        info.worker.send({ type: 'SHUTDOWN', reason: options.message });
+        console.log(`📤 Sent shutdown signal to worker ${pid}`);
+      } catch (error) {
+        console.error(`Failed to send shutdown signal to worker ${pid}:`, error);
+      }
+    });
+    
+    this.emit('shutdown:notified', { workers: this.clusterManager.workers.size });
+  }
+  
+  /**
+   * Wait for active connections to complete
+   * @returns {Promise<void>}
+   */
+  async waitForConnections() {
+    if (this.connections.size === 0) {
+      console.log('✅ No active connections to wait for');
+      return;
+    }
+    
+    console.log(`⏳ Waiting for ${this.connections.size} active connections to complete...`);
+    this.emit('shutdown:waiting', { connections: this.connections.size });
+    
+    // Wait for all connections to complete or timeout
+    const startTime = Date.now();
+    const maxWaitTime = this.options.timeout;
+    
+    while (this.connections.size > 0 && Date.now() - startTime < maxWaitTime) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    
+    if (this.connections.size > 0) {
+      console.log(`⚠️ ${this.connections.size} connections still active after timeout`);
+      this.emit('shutdown:connectionsTimeout', { remaining: this.connections.size });
+    } else {
+      console.log('✅ All connections completed');
+      this.emit('shutdown:connectionsComplete');
+    }
+  }
+  
+  /**
+   * Perform cleanup tasks
+   * @returns {Promise<void>}
+   */
+  async performCleanup() {
+    console.log('🧹 Performing cleanup tasks...');
+    this.emit('shutdown:cleanupStarted');
+    
+    // Close database connections, file handles, etc.
+    // This is where you would add your application-specific cleanup logic
+    
+    // Example: Close database connections
+    // if (this.clusterManager.db) {
+    //   await this.clusterManager.db.close();
+    // }
+    
+    // Example: Close file handles
+    // if (this.clusterManager.fileHandles) {
+    //   this.clusterManager.fileHandles.forEach(handle => handle.close());
+    // }
+    
+    console.log('✅ Cleanup completed');
+    this.emit('shutdown:cleanupCompleted');
+  }
+  
+  /**
+   * Complete shutdown process
+   * @returns {Promise<void>}
+   */
+  async completeShutdown() {
+    console.log('✅ Graceful shutdown completed');
+    this.emit('shutdown:completed');
+    
+    // Clear timeouts
+    if (this.shutdownTimeout) {
+      clearTimeout(this.shutdownTimeout);
+      this.shutdownTimeout = null;
+    }
+    
+    if (this.forceShutdownTimeout) {
+      clearTimeout(this.forceShutdownTimeout);
+      this.forceShutdownTimeout = null;
+    }
+    
+    // Exit process
+    setTimeout(() => {
+      console.log('👋 Process exiting');
+      process.exit(0);
+    }, 1000);
+  }
+  
+  /**
+   * Handle shutdown timeout
+   */
+  handleShutdownTimeout() {
+    console.log('⏰ Shutdown timeout reached, forcing shutdown');
+    this.emit('shutdown:timeout');
+    
+    this.handleForceShutdown();
+  }
+  
+  /**
+   * Handle force shutdown
+   */
+  handleForceShutdown() {
+    console.log('💥 Force shutdown initiated');
+    this.emit('shutdown:force');
+    
+    // Clear all timeouts
+    if (this.shutdownTimeout) {
+      clearTimeout(this.shutdownTimeout);
+      this.shutdownTimeout = null;
+    }
+    
+    if (this.forceShutdownTimeout) {
+      clearTimeout(this.forceShutdownTimeout);
+      this.forceShutdownTimeout = null;
+    }
+    
+    // Force exit
+    setTimeout(() => {
+      console.log('💥 Force exiting process');
+      process.exit(1);
+    }, 1000);
+  }
+  
+  /**
+   * Check authentication
+   * @param {Object} ctx - Context object
+   * @returns {boolean} Authentication status
+   */
+  checkAuth(ctx) {
+    if (typeof this.options.auth === 'function') {
+      return this.options.auth(ctx);
+    }
+    
+    if (typeof this.options.auth === 'object') {
+      // Check API key
+      if (this.options.auth.apiKey) {
+        const apiKey = ctx.headers['x-api-key'] || ctx.query.apiKey;
+        return apiKey === this.options.auth.apiKey;
+      }
+      
+      // Check basic auth
+      if (this.options.auth.username && this.options.auth.password) {
+        const authHeader = ctx.headers.authorization;
+        if (!authHeader || !authHeader.startsWith('Basic ')) {
+          return false;
+        }
+        
+        const credentials = Buffer.from(authHeader.slice(6), 'base64').toString();
+        const [username, password] = credentials.split(':');
+        return username === this.options.auth.username && password === this.options.auth.password;
+      }
+    }
+    
+    return true; // No auth required
+  }
+  
+  /**
+   * Get shutdown status
+   * @returns {Object} Shutdown status
+   */
+  getStatus() {
+    return {
+      isShuttingDown: this.isShuttingDown,
+      startTime: this.shutdownStartTime,
+      elapsedTime: this.shutdownStartTime ? Date.now() - this.shutdownStartTime : 0,
+      activeConnections: this.connections.size,
+      timeout: this.options.timeout,
+      forceTimeout: this.options.forceTimeout
+    };
+  }
+  
+  /**
+   * Cancel shutdown
+   * @returns {boolean} Success status
+   */
+  cancelShutdown() {
+    if (!this.isShuttingDown) {
+      return false;
+    }
+    
+    console.log('🔄 Cancelling shutdown');
+    
+    this.isShuttingDown = false;
+    this.shutdownStartTime = null;
+    
+    // Clear timeouts
+    if (this.shutdownTimeout) {
+      clearTimeout(this.shutdownTimeout);
+      this.shutdownTimeout = null;
+    }
+    
+    if (this.forceShutdownTimeout) {
+      clearTimeout(this.forceShutdownTimeout);
+      this.forceShutdownTimeout = null;
+    }
+    
+    this.emit('shutdown:cancelled');
+    return true;
+  }
+  
+  /**
+   * Get middleware configuration
+   * @returns {Object} Middleware configuration
+   */
+  getConfig() {
+    return {
+      ...this.options,
+      isShuttingDown: this.isShuttingDown,
+      activeConnections: this.connections.size
+    };
+  }
+}
+
+export default GracefulShutdownMiddleware;