cloudcms-server 0.9.257 → 0.9.260

package/README.md

@@ -13,7 +13,7 @@ This looks like:
 
 ````
 {
-    "setup": "single", // single, cluster, sticky-cluster
+    "setup": "single", // single, cluster, redis
     "name": "Cloud CMS Application Server",
     "socketFunctions": [],
     "routeFunctions": [],

package/broadcast/providers/redis.js

@@ -29,7 +29,7 @@ exports = module.exports = function(broadcastConfig)
                 "scope": "broadcast_cache"
             };
     
-            logger.info("using config = " + nrpConfig);
+            //logger.info("using config = " + nrpConfig);
     
             nrp = new NRP(nrpConfig);
             nrp.connect(function(err) {

package/clients/redis.js

@@ -1,4 +1,4 @@
-var redis = require("redis");
+//var redis = require("redis");
 var redisHelper = require("../util/redis");
 
 var clients = {};
@@ -10,18 +10,34 @@ var clients = {};
  */
 exports = module.exports = {};
 
-var create = exports.create = function(config, callback)
+var create = exports.create = function(config, baseOptions, callback)
 {
     if (typeof(config) === "function") {
         callback = config;
         config = {};
+        baseOptions = {};
+    }
+    
+    if (typeof(baseOptions) === "function") {
+        callback = baseOptions;
+        baseOptions = {};
     }
     
     if (!config) {
         config = {};
     }
     
+    if (!baseOptions) {
+        baseOptions = {};
+    }
+    
     var redisOptions = redisHelper.redisOptions(config);
+    if (baseOptions) {
+        for (var k in baseOptions) {
+            redisOptions[k] = baseOptions[k];
+        }
+    }
+    
     var url = redisOptions.url;
     
     // cached client?

package/launchpad/index.js

@@ -1,17 +1,167 @@
-module.exports = function(options)
+const cluster = require("cluster");
+
+module.exports = function(type, config, options)
 {
-    // allow for override
-    if (process.env.CLOUDCMS_LAUNCHPAD_SETUP) {
-        options.setup = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
+    if (!type) {
+        type = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
     }
-
-    // default to single mode
-    if (!options.setup) {
-        options.setup = "single";
+    if (!type) {
+        type = "single";
+    }
+    
+    process.env.CLOUDCMS_LAUNCHPAD_SETUP = type
+    
+    var launcherFactory = require("./launchers/" + type);
+    if (!launcherFactory)
+    {
+        throw new Error("Cannot find launcher factory: " + type);
+    }
+    
+    var launcher = launcherFactory(config);
+    
+    var reportFn = options.report;
+    if (!reportFn) {
+        options.report = reportFn = function () {};
+    }
+    
+    var completionFn = options.complete;
+    if (!completionFn) {
+        options.complete = completionFn = function(err) {
+            throw new Error(err);
+        };
+    }
+    
+    var fork = true;
+    if (type === "single") {
+        fork = false;
     }
+    
+    if (!fork)
+    {
+        return launchWorker(launcher, config, options, function(err, app, httpServer) {
+            
+            if (err) {
+                return completionFn(config, err);
+            }
+    
+            var httpServerPort = -1;
+            // if (app) {
+            //     httpServerPort = app.get("port");
+            // }
+            if (httpServerPort === -1) {
+                httpServerPort = process.env.PORT;
+            }
+            if (httpServerPort === -1) {
+                httpServerPort = 3000;
+            }
+    
+            httpServer.listen(httpServerPort);
+    
+            reportFn(config);
+            completionFn(config);
+        });
+    }
+    else
+    {
+        if (cluster.isMaster)
+        {
+            return launchMaster(launcher, config, options, function(err, workers) {
+    
+                if (err) {
+                    return completionFn(config, err);
+                }
+    
+                //reportFn(config);
+                completionFn(config);
+            });
+        }
+        else
+        {
+            return launchWorker(launcher, config, options, function(err) {
+                completionFn(config, err);
+            });
+        }
+    }
+};
+
+var launchMaster = function(launcher, config, options, done)
+{
+    var createHttpServer = options.createHttpServer;
+    
+    createHttpServer(null, function(err, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+    
+        launcher.startCluster(httpServer, function(err) {
+    
+            if (err) {
+                return done(err);
+            }
+    
+            var httpServerPort = -1;
+            // if (app) {
+            //     httpServerPort = app.get("port");
+            // }
+            if (httpServerPort === -1) {
+                httpServerPort = process.env.PORT;
+            }
+            if (httpServerPort === -1) {
+                httpServerPort = 3000;
+            }
+    
+            httpServer.listen(httpServerPort);
+    
+            launcher.afterStartCluster(httpServer, function(err, workers) {
+                done(err, workers);
+            });
+        });
+    });
+};
 
-    process.env.CLOUDCMS_LAUNCHPAD_SETUP = options.setup;
+var launchWorker = function(launcher, config, options, done)
+{
+    var startServer = options.startServer;
+    
+    var configureServer = options.configureServer;
+    if (!configureServer) {
+        options.configureServer = configureServer = function(config, app, httpServer, done) {
+            done();
+        }
+    }
+    
+    startServer(config, function(err, app, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+        
+        launcher.afterStartServer(app, httpServer, function(err) {
+            
+            if (err) {
+                return done(err);
+            }
+    
+            configureServer(config, app, httpServer, function(err) {
+                
+                if (err) {
+                    return done(err);
+                }
+                
+                // if we are on a worker process, then inform the master that we completed
+                if (process.send) {
+                    process.send("worker-startup");
+                }
+    
+                if (process.env["CLUSTER_REPORT"])
+                {
+                    var reportFn = options.report;
+                    reportFn(config);
+                }
 
-    var launch = require("./launchers/" + options.setup);
-    launch(options);
+                done(null, app, httpServer);
+            });
+        });
+    });
 };

package/launchpad/launchers/cluster.js

@@ -1,121 +1,99 @@
-var cluster = require("cluster");
-var async = require("async");
-
 var clusterlock = require("../../temp/clusterlock");
 
-var workers = [];
+const cluster = require("cluster");
+const { Server } = require("socket.io");
+const numCPUs = require("os").cpus().length;
+const { setupMaster, setupWorker } = require("@socket.io/sticky");
+const { createAdapter, setupPrimary } = require("@socket.io/cluster-adapter");
+
+module.exports = function(config) {
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        // setup sticky sessions
+        setupMaster(httpServer, {
+            loadBalancingMethod: "least-connection",
+        });
 
-module.exports = function(options)
-{
-    // always take up the max # of CPU's
-    var cpuCount = require("os").cpus().length;
+        // setup connections between the workers
+        setupPrimary();
 
-    var factoryCallback = options.factory;
-    var reportCallback = options.report;
-    if (!reportCallback) {
-        reportCallback = function() { };
-    }
-    var completionCallback = options.complete;
-    if (!completionCallback) {
-        completionCallback = function() { };
-    }
+        // needed for packets containing buffers
+        cluster.setupPrimary({
+            serialization: "advanced"
+        });
 
-    if (cluster.isMaster)
+        return callback();
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
     {
-        // master
-
-        var fns = [];
-        for (var i = 0; i < cpuCount; i++)
+        // start up cluster locks
+        clusterlock.setup();
+        
+        var startupCount = 0;
+        
+        var workers = [];
+        for (let i = 0; i < numCPUs; i++)
         {
-            var fn = function (i, workers) {
-                return function (done) {
-
-                    var spawn = function (i, afterSpawnFn) {
-
-                        if (!afterSpawnFn) {
-                            afterSpawnFn = function () { };
-                        }
-
-                        workers[i] = cluster.fork();
-
-                        workers[i].on('exit', function (worker, workers, i) {
-                            return function() {
-                                console.error("launchpad: worker " + i + " exited");
-                                worker.exited = true;
-
-                                // are all workers exited?
-                                var all = true;
-                                for (var z = 0; z < workers.length; z++)
-                                {
-                                    if (!workers[z].exited) {
-                                        all = false;
-                                    }
-                                }
-
-                                if (all)
-                                {
-                                    console.log("launchpad: all workers exited, terminating process");
-                                    return process.exit(-1);
-                                }
-
-                                // set a timeout to otherwise restart worker in 5 seconds
-                                setTimeout(function() {
-                                    console.log("launchpad: restarting worker: " + i);
-                                    spawn(i);
-                                    worker.exited = false;
-                                }, 5000);
-                            }
-                        }(workers[i], workers, i));
-
-                        workers[i].on('message', function (msg, c) {
-
-                            if (msg === "server-startup")
-                            {
-                                afterSpawnFn();
-                            }
-                        });
-                    };
-                    spawn(i, function () {
-                        done();
-                    });
-                };
-            }(i, workers);
-            fns.push(fn);
-        }
-
-        async.parallel(fns, function (err) {
-
-            // start up cluster locks
-            clusterlock.setup();
-
-            // tell the first worker to report
-            if (workers.length > 0)
-            {
-                workers[0].send("server-report");
+            var workerEnv = {};
+            if (i === 0) {
+                workerEnv["CLUSTER_REPORT"] = true;
             }
-
-            setTimeout(function() {
-                completionCallback();
-            }, 250);
+            
+            var worker = cluster.fork(workerEnv);
+            
+            worker.on('message', function (msg, c) {
+                if (msg === "worker-startup") {
+                    startupCount++;
+                }
+            });
+            
+            workers.push(worker);
+        }
+        
+        cluster.on("exit", (worker) => {
+            console.log(`Worker ${worker.process.pid} died`);
+            cluster.fork();
         });
-    }
-    else
-    {
-        // slave
-
-        factoryCallback(function(server) {
-
-            server.listen(server._listenPort);
-
-            // listen for the "server-report" message and fire the callback
-            process.on('message', function (msg, msgData) {
-
-                if (msg === "server-report")
+        
+        // wait for workers to start
+        var waitFn = function() {
+            setTimeout(function() {
+                if (startupCount >= numCPUs) {
+                    return callback(null, workers);
+                }
+                else
                 {
-                    reportCallback();
+                    waitFn();
                 }
-
-            });
+            }, 25);
+        };
+        waitFn();
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        // worker
+    
+        const io = new Server(httpServer);
+    
+        // use the cluster adapter
+        io.adapter(createAdapter());
+    
+        // setup connection with the primary process
+        setupWorker(io);
+    
+        io.on("connection", (socket) => {
+            // TODO
         });
-    }
-};
+
+        httpServer.io = io;
+
+        return callback();
+    };
+    
+    return r;
+}

package/launchpad/launchers/redis.js

@@ -0,0 +1,53 @@
+const { Server } = require("socket.io");
+const { setupWorker } = require("@socket.io/sticky");
+const { createAdapter } = require("@socket.io/cluster-adapter");
+const redisHelper = require("../../util/redis");
+
+const clusterLauncherFactory = require("./cluster");
+
+module.exports = function(config) {
+    
+    var clusterLauncher = clusterLauncherFactory(config);
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        clusterLauncher.startCluster(httpServer, callback);
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
+    {
+        clusterLauncher.afterStartCluster(httpServer, callback);
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        const io = new Server(httpServer);
+    
+        (async function() {
+            var redisOptions = redisHelper.redisOptions({}, "CLOUDCMS_CLUSTER");
+            await redisHelper.createAndConnect(redisOptions, function(err, _client) {
+            
+                const pubClient = _client;
+                const subClient = pubClient.duplicate();
+            
+                io.adapter(createAdapter(pubClient, subClient));
+                //io.listen(httpServerPort);
+            
+                io.on("connection", (socket) => {
+                    // TODO
+                });
+            
+                // setup connection with the primary process
+                setupWorker(io);
+            
+                httpServer.io = io;
+            
+                return callback();
+            });
+        })();
+    };
+    
+    return r;
+}

package/launchpad/launchers/single.js

@@ -1,23 +1,31 @@
-module.exports = function(options) {
+const { Server } = require("socket.io");
 
-    var factoryCallback = options.factory;
-    var reportCallback = options.report;
-    if (!reportCallback) {
-        reportCallback = function () {
-        };
-    }
-    var completionCallback = options.complete;
-    if (!completionCallback) {
-        completionCallback = function () {
-        };
-    }
-
-    factoryCallback(function (server) {
-
-        reportCallback();
-
-        server.listen(server._listenPort);
-
-        completionCallback();
-    });
-};
+module.exports = function(config) {
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        callback();
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
+    {
+        callback();
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        var io = new Server(httpServer);
+    
+        io.on("connection", (socket) => {
+            // TODO
+        });
+        
+        httpServer.io = io;
+        
+        callback();
+    };
+    
+    return r;
+}

package/locks/locks.js

@@ -34,7 +34,8 @@ exports = module.exports = function()
         var self = this;
 
         // set up defaults
-        if (!process.env.CLOUDCMS_LOCKS_TYPE) {
+        if (!process.env.CLOUDCMS_LOCKS_TYPE)
+        {
             process.env.CLOUDCMS_LOCKS_TYPE = "memory";
 
             if (process.configuration.setup !== "single") {
@@ -71,8 +72,8 @@ exports = module.exports = function()
      */
     var lock = r.lock = function(key, fn)
     {
-        provider.lock(key, function(releaseFn) {
-            fn(function(afterReleaseCallback) {
+        provider.lock(key, function(err, releaseFn) {
+            fn(err, function(afterReleaseCallback) {
 
                 releaseFn();
 

package/locks/providers/cluster.js

@@ -19,7 +19,7 @@ exports = module.exports = function(lockConfig)
     r.lock = function(key, fn)
     {
         ClusterLock.lock(key, function(releaseCallbackFn) {
-            fn(releaseCallbackFn);
+            fn(null, releaseCallbackFn);
         });
     };
 

package/locks/providers/memory.js

@@ -23,7 +23,7 @@ exports = module.exports = function(lockConfig)
     r.lock = function(key, fn)
     {
         locker.writeLock(key, function(releaseCallbackFn) {
-            fn(releaseCallbackFn);
+            fn(null, releaseCallbackFn);
         });
     };