@cocreate/server 1.5.1 → 1.6.0

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+# [1.6.0](https://github.com/CoCreate-app/CoCreate-server/compare/v1.5.1...v1.6.0) (2026-07-18)
+
+
+### Features
+
+* add ingress state management with cordon, shedding, and draining events ([b52a198](https://github.com/CoCreate-app/CoCreate-server/commit/b52a1986c566fe9a73c8e694542be69d808ae3eb))
+* implement graceful shutdown process with selective shedding and draining for improved resource management ([5c367e0](https://github.com/CoCreate-app/CoCreate-server/commit/5c367e0284f587965bd5b4248b1f1f97a4b51b2f))
+
 ## [1.5.1](https://github.com/CoCreate-app/CoCreate-server/compare/v1.5.0...v1.5.1) (2026-07-17)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocreate/server",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "description": "A dynamic SSL certificate management and generation tool for proxies like NGINX, with a fallback to Node.js SSL termination. It seamlessly integrates HTTP, HTTPS, and ACME protocols to ensure secure, encrypted connections.",
   "keywords": [
     "ssl-certificate-management",

package/src/index.js

@@ -50,6 +50,11 @@ const sharedMap = new Map();
 const pendingPromises = new Map();
 const activeWorkers = new Map();
 
+// Track overall system state during shutdowns to prevent concurrent signal re-evaluation
+let isPrimaryShuttingDown = false;
+let isMeshClosed = false; // Dual-lock flag to guarantee DB prunes finish before process exits
+let primaryShutdownTimeout = null;
+
 export class CoCreateServer extends EventEmitter {
     constructor() {
         super();
@@ -71,15 +76,76 @@ export class CoCreateServer extends EventEmitter {
         this.ip = null;
         this.infrastructure = null;
 
-        // Create HTTP server for edge proxy ACME challenges and secure HTTPS redirects
+        // Ingress state management flags
+        this.isCordon = false;
+        this.isShedding = false;
+        this.isDraining = false;
+
+        this.on('cordon', (payload) => {
+            const active = payload?.active !== false;
+            
+            // Core Logic: Telemetry or callers cannot uncordon the node if shedding or draining is active
+            if (!active && (this.isShedding || this.isDraining)) {
+                console.warn(`[@cocreate/server] Blocked un-cordon request. Overriding active states: isShedding=${this.isShedding}, isDraining=${this.isDraining}`);
+                return;
+            }
+            
+            this.isCordon = active;
+            const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
+            console.log(`[@cocreate/server] ${processLabel} isCordon updated to: ${this.isCordon}`);
+        });
+
+        this.on('shed', (payload) => {
+            this.isShedding = payload?.active !== false;
+            
+            // Shedding state activation automatically triggers a network cordon
+            if (this.isShedding) {
+                this.isCordon = true;
+                
+                // Selective Shedding Integration: Target specified organizations inside the socket server
+                if (payload?.organizations && this.wsManager && typeof this.wsManager.destroyOrganizations === 'function') {
+                    const orgList = Array.isArray(payload.organizations) ? payload.organizations : [payload.organizations];
+                    console.log(`[@cocreate/server] Shedding event triggered. Selective eviction of org IDs:`, orgList);
+                    this.wsManager.destroyOrganizations(orgList);
+                }
+            }
+            const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
+            console.log(`[@cocreate/server] ${processLabel} isShedding updated to: ${this.isShedding} (isCordon: ${this.isCordon})`);
+        });
+
+        this.on('drain', (payload) => {
+            this.isDraining = payload?.active !== false;
+            
+            // Draining state activation automatically triggers a network cordon
+            if (this.isDraining) {
+                this.isCordon = true;
+                
+                // Complete Worker Drain Integration: Instantly purge all sockets and loaded maps
+                if (this.wsManager && typeof this.wsManager.destroyAll === 'function') {
+                    console.log(`[@cocreate/server] Worker-level drain event triggered. Purging all tenant configurations.`);
+                    this.wsManager.destroyAll();
+                }
+            }
+            const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
+            console.log(`[@cocreate/server] ${processLabel} isDraining updated to: ${this.isDraining} (isCordon: ${this.isCordon})`);
+        });
+
         this.http = Http.createServer((req, res) => {
             const requestUrl = req.url || '/';
             // 1. Allow ACME challenges to pass through over unencrypted HTTP
             if (requestUrl.startsWith('/.well-known/acme-challenge/')) {
                 return this.handleEdgeRequest(req, res);
             }
-            // 2. Force a 301 redirect to HTTPS for all other traffic
+            
             const host = req.headers.host ? req.headers.host.split(':')[0] : 'localhost';
+
+            // 2. Intercept with a 307 Temporary Redirect if cordon is active
+            if (this.isCordon) {
+                res.writeHead(307, { "Location": `https://${host}${requestUrl}` });
+                return res.end();
+            }
+
+            // 3. Force a 301 redirect to HTTPS for all other traffic
             res.writeHead(301, { "Location": `https://${host}${requestUrl}` });
             res.end();
         });
@@ -110,7 +176,6 @@ export class CoCreateServer extends EventEmitter {
             }
 
             // 2. Compile target worker list dynamically using a flat ternary condition.
-            // This eliminates duplicate loops, nested blocks, and repeated try/catch statements.
             const targetIds = payload.broadcast === true
                 ? Array.from(this.workers.keys())
                 : (Array.isArray(payload.workers) ? payload.workers : []);
@@ -118,7 +183,7 @@ export class CoCreateServer extends EventEmitter {
             // 3. Dispatch to all compiled destination targets in a single unified loop.
             for (const wId of targetIds) {
                 const worker = this.workers.get(wId);
-                if (worker?.process && typeof worker.process.send === 'function') {
+                if (worker?.process && typeof worker.process.send === 'function' && worker.process.isConnected()) {
                     try {
                         worker.process.send({ ipcPayload: true, payload });
                     } catch (err) {
@@ -172,22 +237,17 @@ export class CoCreateServer extends EventEmitter {
         if (configuredWorkers !== false && configuredWorkers !== 'false') {
             const parsedWorkers = parseInt(configuredWorkers, 10);
             if (parsedWorkers === -1) {
-                // Use total CPU cores minus 1 (minimum of 1) to reserve one core for the master process
                 this.totalWorkers = Math.max(1, os.cpus().length - 1);
             } else {
                 this.totalWorkers = parsedWorkers || os.cpus().length;
             }
         }
 
-        // Explicitly set workerId = 1 on the single Primary process if workers configuration specifies no clustering (<= 1)
         if (this.isPrimary && this.totalWorkers <= 1) {
             this.workerId = 1;
         }
 
-        // DB interface is booted first so that ServerMesh can cleanly read nodes and write startup heartbeats
         this.crud = await CrudServer.init(this);
-
-        // Assign the ServerMesh globally. This guarantees both Workers and Primary can safely call "this.mesh.send"
         this.mesh = ServerMesh;
         this.autoscaler = ServerAutoscaler;
 
@@ -199,6 +259,9 @@ export class CoCreateServer extends EventEmitter {
             await this.mesh.init(this);
             await this.autoscaler.init(this);
 
+            // Setup Primary process Graceful Signal Listeners
+            setupPrimaryShutdownHooks(this);
+
             if (this.totalWorkers > 1) {
                 console.log(`[@cocreate/server] [PRIMARY] Orchestrator running with PID: ${process.pid}`);
                 for (let i = 0; i < this.totalWorkers; i++) {
@@ -209,6 +272,12 @@ export class CoCreateServer extends EventEmitter {
                         SERVER_IP: this.ip,
                         SERVER_INFRASTRUCTURE: JSON.stringify(this.infrastructure)
                     });
+
+                    // Protect against unhandled 'error' events on the cluster worker wrapper
+                    worker.on('error', (err) => {
+                        console.error(`[@cocreate/server] [PRIMARY] Error on Worker ${currentWorkerId}:`, err.message);
+                    });
+
                     activeWorkers.set(currentWorkerId, {
                         id: currentWorkerId,
                         pid: worker.process.pid,
@@ -219,6 +288,7 @@ export class CoCreateServer extends EventEmitter {
                 // Supervise and automatically reboot dead or crashed worker processes
                 cluster.on('exit', (worker, code, signal) => {
                     console.log(`[@cocreate/server] [PRIMARY] Worker ${worker.process.pid} exited (Code: ${code}, Signal: ${signal})`);
+                    
                     let deadWorkerId = null;
                     for (let [id, w] of activeWorkers.entries()) {
                         if (w.pid === worker.process.pid) {
@@ -227,6 +297,22 @@ export class CoCreateServer extends EventEmitter {
                             break;
                         }
                     }
+
+                    // SAFETY: If Primary is shutting down or draining, DO NOT fork a replacement worker process!
+                    if (isPrimaryShuttingDown || this.isDraining) {
+                        console.log(`[@cocreate/server] [PRIMARY] Worker ${worker.process.pid} removed. Draining active workers: ${activeWorkers.size} remaining.`);
+                        
+                        // Dual-Lock Guard: Only exit Master if workers are dead AND mesh has completed the MongoDB prune
+                        if (activeWorkers.size === 0 && isMeshClosed) {
+                            console.log("[@cocreate/server] [PRIMARY] All workers successfully drained and Mesh closed. Terminating master process cleanly.");
+                            if (primaryShutdownTimeout) {
+                                clearTimeout(primaryShutdownTimeout);
+                            }
+                            process.exit(0);
+                        }
+                        return;
+                    }
+
                     const targetId = deadWorkerId || (activeWorkers.size + 1);
                     
                     const newWorker = cluster.fork({ 
@@ -235,6 +321,11 @@ export class CoCreateServer extends EventEmitter {
                         SERVER_IP: this.ip,
                         SERVER_INFRASTRUCTURE: JSON.stringify(this.infrastructure)
                     });
+
+                    newWorker.on('error', (err) => {
+                        console.error(`[@cocreate/server] [PRIMARY] Error on replacement Worker ${targetId}:`, err.message);
+                    });
+
                     activeWorkers.set(targetId, {
                         id: targetId,
                         pid: newWorker.process.pid,
@@ -242,7 +333,6 @@ export class CoCreateServer extends EventEmitter {
                     });
                 });
 
-                // In clustered mode, only the Primary exits early. The workers handle the core traffic.
                 if (this.totalWorkers > 1) {
                     return this;
                 }
@@ -260,23 +350,19 @@ export class CoCreateServer extends EventEmitter {
             this.infrastructure = await resolveInfrastructure(this.id);
         }
 
-        // Initialize HTTPS & Ephemeral certificate via ACME
         this.certificate = Acme;
         this.https = Https.createServer({
             SNICallback: (domain, cb) => this.certificate.sniCallback(domain, cb)
         }, (req, res) => this.handleEdgeRequest(req, res));
         await this.certificate.init(this);
 
-        // Standard core services
         this.wsManager = new SocketServer(this);
         this.api = await ApiServer.init(this);
         this.files = FileServer.init(this);
         this.loader = await LazyLoader.init(this);
 
-        // Initialize performance telemetry
         this.telemetry = await ServerTelemetry.init(this);
 
-        // Bind directly to standard web ports (Requires setcap or root privileges)
         const HTTP_PORT = process.env.HTTP_PORT || 80;
         const HTTPS_PORT = process.env.HTTPS_PORT || 443;
         const BIND_IP = process.env.BIND_IP || '0.0.0.0';
@@ -300,6 +386,14 @@ export class CoCreateServer extends EventEmitter {
     async handleEdgeRequest(req, res) {
         try {
             const requestUrl = req.url || '/';
+
+            // Check if cordon is active (exclude critical ACME challenges from loop-breaking)
+            if (this.isCordon && !requestUrl.startsWith('/.well-known/acme-challenge/')) {
+                const host = req.headers.host ? req.headers.host.split(':')[0] : 'localhost';
+                res.writeHead(307, { "Location": `https://${host}${requestUrl}` });
+                return res.end();
+            }
+
             if (requestUrl.startsWith("/webhooks/")) {
                 if (this.loader) return this.loader.request(req, res);
             }
@@ -341,10 +435,82 @@ function workerStorageManager(serverCtx, method, key, value) {
     });
 }
 
+function setupPrimaryShutdownHooks(serverCtx) {
+    const handleSignal = async (signal) => {
+        if (isPrimaryShuttingDown) return;
+        isPrimaryShuttingDown = true;
+        serverCtx.isCordon = true;
+        serverCtx.isDraining = true;
+
+        console.log(`\n[@cocreate/server] [PRIMARY] Intercepted process termination signal: ${signal}. Commencing cluster-wide graceful shutdown...`);
+
+        // STEP 1: Close Mesh FIRST. Wipes the server record profile from the database while CRUD connection pools are completely active.
+        if (serverCtx.mesh && typeof serverCtx.mesh.close === 'function') {
+            try {
+                console.log("[@cocreate/server] [PRIMARY] Step 1: Initiating Server Mesh close & server record database removal...");
+                await serverCtx.mesh.close();
+                console.log("[@cocreate/server] [PRIMARY] Success: Server Mesh closed and server record cleanly wiped from database.");
+            } catch (err) {
+                console.error("[@cocreate/server] [PRIMARY] Failed to close Server Mesh or wipe server profile cleanly:", err);
+            }
+        }
+        isMeshClosed = true;
+
+        // Dual-Lock Resolution: If all workers finished exiting while the database prune was running,
+        // terminate the Master process cleanly right now instead of waiting for the safety timeout.
+        if (activeWorkers.size === 0) {
+            console.log("[@cocreate/server] [PRIMARY] All workers successfully drained and Mesh closed. Terminating master process cleanly.");
+            if (primaryShutdownTimeout) {
+                clearTimeout(primaryShutdownTimeout);
+            }
+            process.exit(0);
+        }
+
+        // STEP 2: Now that the database profile is clean, tell all worker processes over IPC to begin local graceful drains.
+        console.log("[@cocreate/server] [PRIMARY] Step 2: Informing worker processes to start local connection drains...");
+        for (const [wId, worker] of activeWorkers.entries()) {
+            const workerInstance = worker.process;
+            if (workerInstance && typeof workerInstance.send === 'function' && workerInstance.isConnected()) {
+                console.log(`[@cocreate/server] [PRIMARY] Sending 'shutdown' instruction to Worker ${wId} (PID: ${worker.pid})`);
+                try {
+                    workerInstance.send({ type: 'shutdown' });
+                } catch (err) {
+                    console.error(`[@cocreate/server] Failed sending shutdown IPC instruction to worker ${wId}:`, err);
+                }
+            } else {
+                console.log(`[@cocreate/server] [PRIMARY] Skipping disconnected Worker ${wId} (PID: ${worker.pid})`);
+            }
+        }
+
+        // STEP 3: Set Cooperative Grace Fallback Timer (5 Seconds)
+        // If workers finish early, the exit event triggers process exit immediately. If they hang, we force-kill.
+        primaryShutdownTimeout = setTimeout(() => {
+            console.warn("[@cocreate/server] [PRIMARY] Safety grace window expired! Force-terminating remaining processes...");
+            
+            for (const [wId, worker] of activeWorkers.entries()) {
+                if (worker?.process) {
+                    try {
+                        console.log(`[@cocreate/server] [PRIMARY] Force-killing hanging Worker ${wId} (PID: ${worker.pid}).`);
+                        worker.process.kill('SIGKILL');
+                    } catch (err) {
+                        console.error(`[@cocreate/server] Error force-killing Worker ${wId}:`, err);
+                    }
+                }
+            }
+            
+            console.log("[@cocreate/server] [PRIMARY] All worker processes forcefully reaped. Terminating Master process.");
+            process.exit(0);
+        }, 5000);
+    };
+
+    process.on('SIGTERM', () => handleSignal('SIGTERM'));
+    process.on('SIGINT', () => handleSignal('SIGINT'));
+    process.on('SIGBREAK', () => handleSignal('SIGBREAK'));
+}
+
 function setupIPC(serverCtx) {
     if (serverCtx.isPrimary) {
         cluster.on('message', (worker, message) => {
-            // 1. Check for standard storage operations
             if (message && message.id === serverCtx.id) {
                 try {
                     message.value = executeLocalWorkerStorageOperation(message.method, message.key, message.value);
@@ -354,11 +520,9 @@ function setupIPC(serverCtx) {
                     worker.send(message);
                 }
             }
-            // 2. Routing logic for CoCreate Unified IPC Payloads
             else if (message && message.ipcPayload && message.payload) {
                 const { payload } = message;
                 
-                // Track sending worker PID to worker mapping
                 let senderWorkerId = null;
                 for (const [wId, w] of serverCtx.workers.entries()) {
                     if (w.pid === worker.process.pid) {
@@ -367,17 +531,25 @@ function setupIPC(serverCtx) {
                     }
                 }
                 
-                // Expose sender metadata to routing logic
                 payload.senderWorkerId = senderWorkerId;
-                
-                // Pass it off to the primary orchestrator's central send method
                 serverCtx.send(payload);
             }
         });
     } else {
         process.on('message', (message) => {
-            // 1. Check for standard storage operations
-            if (message && message.id === serverCtx.id) {
+            // Intercept Master-directed shutdown instruction over the worker's IPC channel
+            if (message && message.type === 'shutdown') {
+                console.log(`[@cocreate/server] [Worker ${process.pid}] Received master coordinated shutdown signal. Commencing local drain...`);
+                
+                // Immediately cordon worker traffic and trigger the internal SocketServer destroyAll logic
+                serverCtx.isCordon = true;
+                serverCtx.isDraining = true;
+                
+                if (serverCtx.wsManager && typeof serverCtx.wsManager.destroyAll === 'function') {
+                    serverCtx.wsManager.destroyAll();
+                }
+            }
+            else if (message && message.id === serverCtx.id) {
                 const pending = pendingPromises.get(message.messageId);
                 if (pending) {
                     pendingPromises.delete(message.messageId);
@@ -388,7 +560,6 @@ function setupIPC(serverCtx) {
                     }
                 }
             }
-            // 2. Incoming Unified IPC payload from Master -> Worker Process
             else if (message && message.ipcPayload && message.payload) {
                 serverCtx.emit(message.payload.method, message.payload);
             }