cloudcms-server 3.3.1-beta.8 → 4.0.0-beta.1

package/launchpad/launchers/cluster.js

@@ -1,121 +1,114 @@
-var cluster = require("cluster");
-var async = require("async");
-
-var clusterlock = require("../../temp/clusterlock");
-
-var workers = [];
-
-module.exports = function(options)
-{
-    // always take up the max # of CPU's
-    var cpuCount = require("os").cpus().length;
-
-    var factoryCallback = options.factory;
-    var reportCallback = options.report;
-    if (!reportCallback) {
-        reportCallback = function() { };
-    }
-    var completionCallback = options.complete;
-    if (!completionCallback) {
-        completionCallback = function() { };
-    }
-
-    if (cluster.isMaster)
+const cluster = require("cluster");
+const { Server } = require("socket.io");
+const { setupMaster, setupWorker } = require("@socket.io/sticky");
+const { createAdapter, setupPrimary } = require("@socket.io/cluster-adapter");
+
+module.exports = function(config) {
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
     {
-        // master
-
-        var fns = [];
-        for (var i = 0; i < cpuCount; i++)
-        {
-            var fn = function (i, workers) {
-                return function (done) {
-
-                    var spawn = function (i, afterSpawnFn) {
-
-                        if (!afterSpawnFn) {
-                            afterSpawnFn = function () { };
-                        }
-
-                        workers[i] = cluster.fork();
-
-                        workers[i].on('exit', function (worker, workers, i) {
-                            return function() {
-                                console.error("launchpad: worker " + i + " exited");
-                                worker.exited = true;
-
-                                // are all workers exited?
-                                var all = true;
-                                for (var z = 0; z < workers.length; z++)
-                                {
-                                    if (!workers[z].exited) {
-                                        all = false;
-                                    }
-                                }
-
-                                if (all)
-                                {
-                                    console.log("launchpad: all workers exited, terminating process");
-                                    return process.exit(-1);
-                                }
+        // setup sticky sessions
+        setupMaster(httpServer, {
+            //loadBalancingMethod: "least-connection"
+            loadBalancingMethod: "round-robin"
+        });
 
-                                // set a timeout to otherwise restart worker in 5 seconds
-                                setTimeout(function() {
-                                    console.log("launchpad: restarting worker: " + i);
-                                    spawn(i);
-                                    worker.exited = false;
-                                }, 5000);
-                            }
-                        }(workers[i], workers, i));
+        // setup connections between the workers
+        setupPrimary();
 
-                        workers[i].on('message', function (msg, c) {
+        // needed for packets containing buffers
+        // cluster.setupPrimary({
+        //     serialization: "advanced"
+        // });
 
-                            if (msg === "server-startup")
-                            {
-                                afterSpawnFn();
-                            }
-                        });
-                    };
-                    spawn(i, function () {
-                        done();
-                    });
-                };
-            }(i, workers);
-            fns.push(fn);
+        return callback();
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
+    {
+        var startupCount = 0;
+    
+        var numCPUs = require("os").cpus().length;
+        if (process.env.FORCE_SINGLE_CPU) {
+            numCPUs = 1;
         }
-
-        async.parallel(fns, function (err) {
-
-            // start up cluster locks
-            clusterlock.setup();
-
-            // tell the first worker to report
-            if (workers.length > 0)
-            {
-                workers[0].send("server-report");
+    
+        var workers = [];
+        for (let i = 0; i < numCPUs; i++)
+        {
+            var workerEnv = {};
+            if (i === 0) {
+                workerEnv["CLUSTER_REPORT"] = true;
             }
-
-            setTimeout(function() {
-                completionCallback();
-            }, 250);
+            
+            var worker = cluster.fork(workerEnv);
+            
+            worker.on('message', function (msg, c) {
+                //console.log("Worker message: " + msg + ", c: " + c);
+                if (msg === "worker-startup") {
+                    startupCount++;
+                }
+            });
+            
+            worker.on('error', function(err) {
+                console.log("Worker.on(error) - " + JSON.stringify(err));
+            });
+            
+            workers.push(worker);
+        }
+        
+        cluster.on("error", function(err) {
+            console.log("Cluster.on(error) - " + JSON.stringify(err));
         });
-    }
-    else
-    {
-        // slave
-
-        factoryCallback(function(server) {
-
-            server.listen(server._listenPort);
-
-            // listen for the "server-report" message and fire the callback
-            process.on('message', function (msg, msgData) {
-
-                if (msg === "server-report")
+        
+        cluster.on("exit", (worker) => {
+            console.log(`Worker ${worker.process.pid} died`);
+            //cluster.fork();
+        });
+        
+        // wait for workers to start
+        var waitFn = function() {
+            setTimeout(function() {
+                if (startupCount >= numCPUs) {
+                    return callback(null, workers);
+                }
+                else
                 {
-                    reportCallback();
+                    waitFn();
                 }
-
+            }, 25);
+        };
+        waitFn();
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        // worker
+    
+        const io = new Server(httpServer);
+        httpServer.io = io;
+        
+        // use the cluster adapter
+        io.adapter(createAdapter());
+    
+        // setup connection with the primary process
+        setupWorker(io);
+        
+        // on connect
+        io.on("connection", (socket) => {
+            // TODO
+    
+            // always catch err
+            socket.on("error", function(err) {
+                console.log("Caught socket error");
+                console.log(err.stack);
             });
         });
-    }
-};
+
+        return callback();
+    };
+    
+    return r;
+}

package/launchpad/launchers/redis.js

@@ -0,0 +1,70 @@
+const { Server } = require("socket.io");
+const { setupWorker } = require("@socket.io/sticky");
+const { createAdapter } = require("@socket.io/redis-adapter");
+const redisHelper = require("../../util/redis");
+const IORedis = require("ioredis");
+
+const clusterLauncherFactory = require("./cluster");
+
+module.exports = function(config) {
+    
+    var clusterLauncher = clusterLauncherFactory(config);
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        clusterLauncher.startCluster(httpServer, callback);
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
+    {
+        clusterLauncher.afterStartCluster(httpServer, callback);
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        var redisOptions = redisHelper.redisOptions({}, "CLOUDCMS_CLUSTER");
+        var pubClient = new IORedis(redisOptions.url);
+        var subClient = pubClient.duplicate();
+
+        const io = new Server(httpServer);
+        httpServer.io = io;
+
+        io.engine.on("connection_error", function(err) {
+            // console.log("CONNECTION ERROR");
+            // console.log("REQUEST: ", err.req);      // the request object
+            // console.log("CODE: " + err.code);     // the error code, for example 1
+            // console.log("MESSAGE: ", err.message);  // the error message, for example "Session ID unknown"
+            // console.log("CONTEXT: ", err.context);  // some additional error context
+        });
+
+        // use the redis adapter
+        io.adapter(createAdapter(pubClient, subClient, {
+            //publishOnSpecificResponseChannel: true
+        }));
+
+        // setup connection with the primary process
+        setupWorker(io);
+
+        // on connect
+        io.on("connection", (socket) => {
+            //console.log("Redis Launcher on('connection') - socket id:" + socket.id);
+            socket.on('message', function(m) {
+                console.log("Socket Connection message: " + m);
+            });
+
+            // always catch err
+            socket.on("error", function(err) {
+                console.log("Caught socket error");
+                console.log(err.stack);
+            });
+
+            // TODO
+        });
+
+        return callback();
+    };
+    
+    return r;
+}

package/launchpad/launchers/single.js

@@ -1,23 +1,37 @@
-module.exports = function(options) {
+const { Server } = require("socket.io");
 
-    var factoryCallback = options.factory;
-    var reportCallback = options.report;
-    if (!reportCallback) {
-        reportCallback = function () {
-        };
-    }
-    var completionCallback = options.complete;
-    if (!completionCallback) {
-        completionCallback = function () {
-        };
-    }
-
-    factoryCallback(function (server) {
-
-        reportCallback();
-
-        server.listen(server._listenPort);
-
-        completionCallback();
-    });
-};
+module.exports = function(config) {
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        callback();
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
+    {
+        callback();
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        var io = new Server(httpServer);
+    
+        io.on("connection", (socket) => {
+            // TODO
+            
+            // always catch err
+            socket.on("error", function(err) {
+                console.log("Caught socket error");
+                console.log(err.stack);
+            });
+        });
+        
+        httpServer.io = io;
+        
+        callback();
+    };
+    
+    return r;
+}

package/locks/locks.js

@@ -25,6 +25,8 @@ var async = require("async");
  */
 exports = module.exports = function()
 {
+    var lockTimeoutMs = 120000; // two minutes
+    
     var provider = null;
 
     var r = {};
@@ -34,10 +36,13 @@ exports = module.exports = function()
         var self = this;
 
         // set up defaults
-        if (!process.env.CLOUDCMS_LOCKS_TYPE) {
+        if (!process.env.CLOUDCMS_LOCKS_TYPE)
+        {
             process.env.CLOUDCMS_LOCKS_TYPE = "memory";
-
-            if (process.configuration.setup !== "single") {
+    
+            // auto-configure
+            if (process.env.CLOUDCMS_LAUNCHPAD_SETUP === "redis")
+            {
                 process.env.CLOUDCMS_LOCKS_TYPE = "redis";
             }
         }
@@ -71,12 +76,61 @@ exports = module.exports = function()
      */
     var lock = r.lock = function(key, fn)
     {
-        provider.lock(key, function(releaseFn) {
-            fn(function(afterReleaseCallback) {
-
-                releaseFn();
-
-                if (afterReleaseCallback)
+        var __log = function(key, text) {
+            // var skip = false;
+            // if (key === "channels") { skip = true; }
+            // if (!skip) {
+            //     console.log("[LOCK: " + key + "] " + text);
+            // }
+        };
+    
+        __log(key, "request");
+        provider.lock(key, function(err, _releaseFn) {
+            
+            if (err) {
+                console.log("[LOCK: " + key + "] err: ", err);
+                try { _releaseFn(); } catch (e) { }
+                return fn(err);
+            }
+            
+            // wrap the releaseFn with a wrapper that can only fire once
+            var releaseFn = function(_releaseFn)
+            {
+                var triggered = false;
+                return function() {
+                    if (!triggered) {
+                        triggered = true;
+                        _releaseFn();
+                        return true;
+                    }
+                    
+                    return false;
+                }
+            }(_releaseFn);
+            
+            // after 120 seconds, we force release lock (if it hasn't already been released)
+            (function(key, releaseFn) {
+                setTimeout(function() {
+                    var released = releaseFn();
+                    if (released) {
+                        __log(key, "timed out, released");
+                    }
+                }, lockTimeoutMs);
+            })(key, releaseFn);
+            
+            __log(key, "taken");
+            
+            fn(err, function(afterReleaseCallback) {
+                __log(key, "pre-release");
+
+                var released = releaseFn();
+                if (released) {
+                    __log(key, "released");
+                } else {
+                    __log(key, "not released, was previously released on timeout");
+                }
+    
+                if (released && afterReleaseCallback)
                 {
                     afterReleaseCallback();
                 }

package/locks/providers/cluster.js

@@ -13,13 +13,15 @@ exports = module.exports = function(lockConfig)
 
     r.init = function(callback)
     {
+        ClusterLock.setup();
+        
         callback();
     };
 
     r.lock = function(key, fn)
     {
         ClusterLock.lock(key, function(releaseCallbackFn) {
-            fn(releaseCallbackFn);
+            fn(null, releaseCallbackFn);
         });
     };
 

package/locks/providers/memory.js

@@ -1,6 +1,4 @@
-var path = require("path");
-
-var ReadWriteLock = require("rwlock");
+var AsyncLock = require('async-lock');
 
 /**
  * Simple in-memory lock service.
@@ -12,8 +10,8 @@ var ReadWriteLock = require("rwlock");
 exports = module.exports = function(lockConfig)
 {
     var r = {};
-
-    var locker = new ReadWriteLock();
+    
+    var lock = new AsyncLock();
 
     r.init = function(callback)
     {
@@ -22,8 +20,13 @@ exports = module.exports = function(lockConfig)
 
     r.lock = function(key, fn)
     {
-        locker.writeLock(key, function(releaseCallbackFn) {
-            fn(releaseCallbackFn);
+        lock.acquire(key, function(releaseCallbackFn) {
+            fn(null, releaseCallbackFn);
+        }, function(err, ret) {
+            // lock was released
+            if (err) {
+                console.error("Memory Lock heard error: ", err, " return value: ", ret);
+            }
         });
     };
 

package/locks/providers/redis.js

@@ -1,6 +1,7 @@
-var path = require("path");
-var redis = require("redis");
-var logFactory = require("../../util/logger");
+const redisHelper = require("../../util/redis");
+
+const IORedis = require("ioredis");
+var Redlock = require("redlock");
 
 /**
  * Redis lock service.
@@ -9,98 +10,77 @@ var logFactory = require("../../util/logger");
  */
 exports = module.exports = function(locksConfig)
 {
-    var redisLock = require("redislock");
-    redisLock.setDefaults({
-        timeout: 200000,
-        retries: 2000,
-        delay: 50
-    });
-
-    var nrp = null;
+    var redlock = null;
     var client = null;
-
-    var logger = logFactory("REDIS LOCK");
-
-    // allow for global redis default
-    // allow for redis broadcast specific
-    // otherwise default to error
-    if (typeof(process.env.CLOUDCMS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else if (typeof(process.env.CLOUDCMS_LOCKS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_LOCKS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else {
-        logger.setLevel("error");
-    }
-
+    
+    var logger = redisHelper.redisLogger("REDIS_LOCKS", "CLOUDCMS_LOCKS_", "debug")
+    
     var r = {};
 
     r.init = function(callback)
     {
-        var redisPort = locksConfig.port;
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_LOCKS_REDIS_PORT;
-        }
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_REDIS_PORT;
-        }
-
-        var redisEndpoint = locksConfig.endpoint;
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_LOCKS_REDIS_ENDPOINT;
-        }
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
-        }
-
-        var redisOptions = {};
-
-        //redis.debug_mode = true;
-
-        client = redis.createClient(redisPort, redisEndpoint, redisOptions);
+        var redisOptions = redisHelper.redisOptions(locksConfig);
+        client = new IORedis(redisOptions.url);
+        
+        redlock = new Redlock(
+            [client],
+            {
+                // the expected clock drift; for more details
+                // see http://redis.io/topics/distlock
+                driftFactor: 0.01, // multiplied by lock ttl to determine drift time
+        
+                // the max number of times Redlock will attempt
+                // to lock a resource before erroring
+                retryCount:  10,
+        
+                // the time in ms between attempts
+                retryDelay:  200, // time in ms
+        
+                // the max time in ms randomly added to retries
+                // to improve performance under high contention
+                // see https://www.awsarchitectureblog.com/2015/03/backoff.html
+                retryJitter:  200 // time in ms
+            }
+        );
 
-        callback();
+        return callback();
     };
 
     r.lock = function(key, fn)
     {
+        key = key.trim();
+        key = key.toLowerCase();
+        key = key.replace(/[\W_]+/g,"");
+        
         var lockKey = "cloudcms:locks:write:" + key;
-
-        var lock = redisLock.createLock(client);
-
-        var releaseCallbackFn = function(lock, lockKey) {
-            return function() {
-                logger.info("lock.release - " + lockKey);
-                lock.release(function(err) {
-
-                    if (err) {
-                        console.log("Failed to release redis lock: " + lockKey);
-                        console.log("Error: " + err);
-                        return;
-                    }
-
-                    logger.info("lock.released - " + lockKey);
-                });
-            }
-        }(lock, lockKey);
-
-        logger.info("lock.acquire - " + lockKey);
-        lock.acquire(lockKey, function(err) {
-
+    
+        logger.debug("lock.acquire:", lockKey);
+        
+        redlock.lock(lockKey, 2000, function(err, lock) {
+            
             if (err) {
-                console.log("Failed to acquire redis lock: " + lockKey);
-                console.log("Error: " + err);
-                return;
+                logger.error("Failed to acquire redis lock:", lockKey, err);
+                return fn(err);
             }
-
-            logger.info("lock.acquired - " + lockKey);
-
-            fn(releaseCallbackFn);
+    
+            logger.debug("lock.acquired:", lockKey);
+
+            var releaseCallbackFn = function(lock, lockKey) {
+                return function() {
+    
+                    logger.debug("lock.release:", lockKey);
+                    
+                    lock.unlock(function(err) {
+                        if (err) {
+                            logger.error("Failed to release redis lock:", lockKey, err);
+                        }
+                    });
+                }
+            }(lock, lockKey);
+    
+            logger.debug("lock.invokeFn");
+            fn(null, releaseCallbackFn);
+            
         });
     };