cloudcms-server 0.9.255 → 0.9.260

package/README.md

@@ -13,7 +13,7 @@ This looks like:
 
 ````
 {
-    "setup": "single", // single, cluster, sticky-cluster
+    "setup": "single", // single, cluster, redis
     "name": "Cloud CMS Application Server",
     "socketFunctions": [],
     "routeFunctions": [],

package/broadcast/providers/redis.js

@@ -1,8 +1,7 @@
-var path = require("path");
+var redisClientFactory = require("../../clients/redis");
+var redisHelper = require("../../util/redis");
 
-var NRP = require('node-redis-pubsub');
-
-var logFactory = require("../../util/logger");
+var NRP = require("../../clients/nrp");
 
 /**
  * Redis broadcast provider.
@@ -12,55 +11,31 @@ var logFactory = require("../../util/logger");
 exports = module.exports = function(broadcastConfig)
 {
     var nrp = null;
-
-    var logger = logFactory("REDIS BROADCAST");
-    logger.setLevel("error");
-
-    // allow for global redis default
-    // allow for redis broadcast specific
-    // otherwise default to "error"
-    if (typeof(process.env.CLOUDCMS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else if (typeof(process.env.CLOUDCMS_BROADCAST_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_BROADCAST_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-
+    
+    var logger = redisHelper.redisLogger("REDIS_BROADCAST", "CLOUDCMS_BROADCAST_", "error")
+    
     var r = {};
 
     r.start = function(callback)
     {
-        var redisPort = broadcastConfig.port;
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_BROADCAST_REDIS_PORT;
-        }
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_REDIS_PORT;
-        }
-
-        var redisEndpoint = broadcastConfig.endpoint;
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_BROADCAST_REDIS_ENDPOINT;
-        }
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
-        }
-
-        var nrpConfig = {
-            "port": redisPort,
-            "host": redisEndpoint,
-            "scope": "broadcast_cache"
-        };
-
-        logger.info("using config = " + nrpConfig);
-        
-        nrp = new NRP(nrpConfig);
-
-        callback();
+        redisClientFactory.create(broadcastConfig, function(err, client) {
+            
+            if (err) {
+                return callback(err);
+            }
+    
+            var nrpConfig = {
+                "client": client,
+                "scope": "broadcast_cache"
+            };
+    
+            //logger.info("using config = " + nrpConfig);
+    
+            nrp = new NRP(nrpConfig);
+            nrp.connect(function(err) {
+                callback(err);
+            });
+        });
     };
 
     r.publish = function(topic, message, callback)

package/clients/nrp.js

@@ -0,0 +1,117 @@
+// a revision of
+// https://raw.githubusercontent.com/louischatriot/node-redis-pubsub/master/lib/node-redis-pubsub.js
+// that works with Redis 6+
+
+"use strict";
+var redis = require('redis');
+
+/**
+ * Create a new NodeRedisPubsub instance that can subscribe to channels and publish messages
+ * @param {Object} options Options for the client creations:
+ *                 client - a connected Redis client
+ *                 scope - Optional, two NodeRedisPubsubs with different scopes will not share messages
+ */
+function NodeRedisPubsub(options)
+{
+    if (!(this instanceof NodeRedisPubsub)){ return new NodeRedisPubsub(options); }
+    
+    options || (options = {});
+    
+    this.emitter = options.client.duplicate();
+    this.emitter.setMaxListeners(0);
+    this.receiver = options.client.duplicate();
+    this.receiver.setMaxListeners(0);
+    
+    this.prefix = options.scope ? options.scope + ':' : '';
+}
+
+NodeRedisPubsub.prototype.connect = function(callback)
+{
+    var self = this;
+    
+    (async function() {
+        await self.emitter.connect();
+        await self.receiver.connect();
+        callback();
+    })();
+};
+
+/**
+ * Subscribe to a channel
+ * @param {String} channel The channel to subscribe to, can be a pattern e.g. 'user.*'
+ * @param {Function} handler Function to call with the received message.
+ * @param {Function} cb Optional callback to call once the handler is registered.
+ */
+NodeRedisPubsub.prototype.on = NodeRedisPubsub.prototype.subscribe = function(channel, handler, callback)
+{
+    if (!callback)
+    {
+        callback = function(){};
+    }
+    
+    var self = this;
+    
+    if (channel === "error")
+    {
+        self.errorHandler = handler;
+        self.emitter.on("error", handler);
+        self.receiver.on("error", handler);
+        return callback();
+    }
+    
+    var listener = function(self, handler)
+    {
+        return function(message, channel) {
+    
+            var jsonmsg = message;
+            try{
+                jsonmsg = JSON.parse(message);
+            } catch (ex){
+                if(typeof self.errorHandler === 'function'){
+                    return self.errorHandler("Invalid JSON received! Channel: " + self.prefix + channel + " Message: " + message);
+                }
+            }
+            return handler(jsonmsg, channel);
+        }
+    }(self, handler);
+    
+    (async function() {
+        await self.receiver.pSubscribe(self.prefix + channel, listener);
+    })();
+    
+    callback();
+};
+
+/**
+ * Emit an event
+ * @param {String} channel Channel on which to emit the message
+ * @param {Object} message
+ */
+NodeRedisPubsub.prototype.emit = NodeRedisPubsub.prototype.publish = function (channel, message)
+{
+    var self = this;
+    
+    (async function() {
+        return await self.emitter.publish(self.prefix + channel, JSON.stringify(message));
+    })();
+};
+
+/**
+ * Safely close the redis connections 'soon'
+ */
+NodeRedisPubsub.prototype.quit = function()
+{
+    this.emitter.quit();
+    this.receiver.quit();
+};
+
+/**
+ * Dangerously close the redis connections immediately
+ */
+NodeRedisPubsub.prototype.end = function()
+{
+    this.emitter.end(true);
+    this.receiver.end(true);
+};
+
+module.exports = NodeRedisPubsub;

package/clients/redis.js

@@ -0,0 +1,64 @@
+//var redis = require("redis");
+var redisHelper = require("../util/redis");
+
+var clients = {};
+
+/**
+ * Redis client factory.
+ *
+ * @type {*}
+ */
+exports = module.exports = {};
+
+var create = exports.create = function(config, baseOptions, callback)
+{
+    if (typeof(config) === "function") {
+        callback = config;
+        config = {};
+        baseOptions = {};
+    }
+    
+    if (typeof(baseOptions) === "function") {
+        callback = baseOptions;
+        baseOptions = {};
+    }
+    
+    if (!config) {
+        config = {};
+    }
+    
+    if (!baseOptions) {
+        baseOptions = {};
+    }
+    
+    var redisOptions = redisHelper.redisOptions(config);
+    if (baseOptions) {
+        for (var k in baseOptions) {
+            redisOptions[k] = baseOptions[k];
+        }
+    }
+    
+    var url = redisOptions.url;
+    
+    // cached client?
+    var client = clients[url];
+    if (client) {
+        return callback(null, client);
+    }
+    
+    // connect
+    (async function() {
+        await redisHelper.createAndConnect(redisOptions, function(err, client) {
+            
+            if (err) {
+                return callback(err);
+            }
+            
+            // cache it
+            clients[url] = client;
+            
+            // return
+            return callback(null, client);
+        });
+    })();
+};

package/launchpad/index.js

@@ -1,17 +1,167 @@
-module.exports = function(options)
+const cluster = require("cluster");
+
+module.exports = function(type, config, options)
 {
-    // allow for override
-    if (process.env.CLOUDCMS_LAUNCHPAD_SETUP) {
-        options.setup = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
+    if (!type) {
+        type = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
     }
-
-    // default to single mode
-    if (!options.setup) {
-        options.setup = "single";
+    if (!type) {
+        type = "single";
+    }
+    
+    process.env.CLOUDCMS_LAUNCHPAD_SETUP = type
+    
+    var launcherFactory = require("./launchers/" + type);
+    if (!launcherFactory)
+    {
+        throw new Error("Cannot find launcher factory: " + type);
+    }
+    
+    var launcher = launcherFactory(config);
+    
+    var reportFn = options.report;
+    if (!reportFn) {
+        options.report = reportFn = function () {};
+    }
+    
+    var completionFn = options.complete;
+    if (!completionFn) {
+        options.complete = completionFn = function(err) {
+            throw new Error(err);
+        };
+    }
+    
+    var fork = true;
+    if (type === "single") {
+        fork = false;
     }
+    
+    if (!fork)
+    {
+        return launchWorker(launcher, config, options, function(err, app, httpServer) {
+            
+            if (err) {
+                return completionFn(config, err);
+            }
+    
+            var httpServerPort = -1;
+            // if (app) {
+            //     httpServerPort = app.get("port");
+            // }
+            if (httpServerPort === -1) {
+                httpServerPort = process.env.PORT;
+            }
+            if (httpServerPort === -1) {
+                httpServerPort = 3000;
+            }
+    
+            httpServer.listen(httpServerPort);
+    
+            reportFn(config);
+            completionFn(config);
+        });
+    }
+    else
+    {
+        if (cluster.isMaster)
+        {
+            return launchMaster(launcher, config, options, function(err, workers) {
+    
+                if (err) {
+                    return completionFn(config, err);
+                }
+    
+                //reportFn(config);
+                completionFn(config);
+            });
+        }
+        else
+        {
+            return launchWorker(launcher, config, options, function(err) {
+                completionFn(config, err);
+            });
+        }
+    }
+};
+
+var launchMaster = function(launcher, config, options, done)
+{
+    var createHttpServer = options.createHttpServer;
+    
+    createHttpServer(null, function(err, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+    
+        launcher.startCluster(httpServer, function(err) {
+    
+            if (err) {
+                return done(err);
+            }
+    
+            var httpServerPort = -1;
+            // if (app) {
+            //     httpServerPort = app.get("port");
+            // }
+            if (httpServerPort === -1) {
+                httpServerPort = process.env.PORT;
+            }
+            if (httpServerPort === -1) {
+                httpServerPort = 3000;
+            }
+    
+            httpServer.listen(httpServerPort);
+    
+            launcher.afterStartCluster(httpServer, function(err, workers) {
+                done(err, workers);
+            });
+        });
+    });
+};
 
-    process.env.CLOUDCMS_LAUNCHPAD_SETUP = options.setup;
+var launchWorker = function(launcher, config, options, done)
+{
+    var startServer = options.startServer;
+    
+    var configureServer = options.configureServer;
+    if (!configureServer) {
+        options.configureServer = configureServer = function(config, app, httpServer, done) {
+            done();
+        }
+    }
+    
+    startServer(config, function(err, app, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+        
+        launcher.afterStartServer(app, httpServer, function(err) {
+            
+            if (err) {
+                return done(err);
+            }
+    
+            configureServer(config, app, httpServer, function(err) {
+                
+                if (err) {
+                    return done(err);
+                }
+                
+                // if we are on a worker process, then inform the master that we completed
+                if (process.send) {
+                    process.send("worker-startup");
+                }
+    
+                if (process.env["CLUSTER_REPORT"])
+                {
+                    var reportFn = options.report;
+                    reportFn(config);
+                }
 
-    var launch = require("./launchers/" + options.setup);
-    launch(options);
+                done(null, app, httpServer);
+            });
+        });
+    });
 };

package/launchpad/launchers/cluster.js

@@ -1,121 +1,99 @@
-var cluster = require("cluster");
-var async = require("async");
-
 var clusterlock = require("../../temp/clusterlock");
 
-var workers = [];
+const cluster = require("cluster");
+const { Server } = require("socket.io");
+const numCPUs = require("os").cpus().length;
+const { setupMaster, setupWorker } = require("@socket.io/sticky");
+const { createAdapter, setupPrimary } = require("@socket.io/cluster-adapter");
+
+module.exports = function(config) {
+    
+    var r = {};
+    
+    r.startCluster = function(httpServer, callback)
+    {
+        // setup sticky sessions
+        setupMaster(httpServer, {
+            loadBalancingMethod: "least-connection",
+        });
 
-module.exports = function(options)
-{
-    // always take up the max # of CPU's
-    var cpuCount = require("os").cpus().length;
+        // setup connections between the workers
+        setupPrimary();
 
-    var factoryCallback = options.factory;
-    var reportCallback = options.report;
-    if (!reportCallback) {
-        reportCallback = function() { };
-    }
-    var completionCallback = options.complete;
-    if (!completionCallback) {
-        completionCallback = function() { };
-    }
+        // needed for packets containing buffers
+        cluster.setupPrimary({
+            serialization: "advanced"
+        });
 
-    if (cluster.isMaster)
+        return callback();
+    };
+    
+    r.afterStartCluster = function(httpServer, callback)
     {
-        // master
-
-        var fns = [];
-        for (var i = 0; i < cpuCount; i++)
+        // start up cluster locks
+        clusterlock.setup();
+        
+        var startupCount = 0;
+        
+        var workers = [];
+        for (let i = 0; i < numCPUs; i++)
         {
-            var fn = function (i, workers) {
-                return function (done) {
-
-                    var spawn = function (i, afterSpawnFn) {
-
-                        if (!afterSpawnFn) {
-                            afterSpawnFn = function () { };
-                        }
-
-                        workers[i] = cluster.fork();
-
-                        workers[i].on('exit', function (worker, workers, i) {
-                            return function() {
-                                console.error("launchpad: worker " + i + " exited");
-                                worker.exited = true;
-
-                                // are all workers exited?
-                                var all = true;
-                                for (var z = 0; z < workers.length; z++)
-                                {
-                                    if (!workers[z].exited) {
-                                        all = false;
-                                    }
-                                }
-
-                                if (all)
-                                {
-                                    console.log("launchpad: all workers exited, terminating process");
-                                    return process.exit(-1);
-                                }
-
-                                // set a timeout to otherwise restart worker in 5 seconds
-                                setTimeout(function() {
-                                    console.log("launchpad: restarting worker: " + i);
-                                    spawn(i);
-                                    worker.exited = false;
-                                }, 5000);
-                            }
-                        }(workers[i], workers, i));
-
-                        workers[i].on('message', function (msg, c) {
-
-                            if (msg === "server-startup")
-                            {
-                                afterSpawnFn();
-                            }
-                        });
-                    };
-                    spawn(i, function () {
-                        done();
-                    });
-                };
-            }(i, workers);
-            fns.push(fn);
-        }
-
-        async.parallel(fns, function (err) {
-
-            // start up cluster locks
-            clusterlock.setup();
-
-            // tell the first worker to report
-            if (workers.length > 0)
-            {
-                workers[0].send("server-report");
+            var workerEnv = {};
+            if (i === 0) {
+                workerEnv["CLUSTER_REPORT"] = true;
             }
-
-            setTimeout(function() {
-                completionCallback();
-            }, 250);
+            
+            var worker = cluster.fork(workerEnv);
+            
+            worker.on('message', function (msg, c) {
+                if (msg === "worker-startup") {
+                    startupCount++;
+                }
+            });
+            
+            workers.push(worker);
+        }
+        
+        cluster.on("exit", (worker) => {
+            console.log(`Worker ${worker.process.pid} died`);
+            cluster.fork();
         });
-    }
-    else
-    {
-        // slave
-
-        factoryCallback(function(server) {
-
-            server.listen(server._listenPort);
-
-            // listen for the "server-report" message and fire the callback
-            process.on('message', function (msg, msgData) {
-
-                if (msg === "server-report")
+        
+        // wait for workers to start
+        var waitFn = function() {
+            setTimeout(function() {
+                if (startupCount >= numCPUs) {
+                    return callback(null, workers);
+                }
+                else
                 {
-                    reportCallback();
+                    waitFn();
                 }
-
-            });
+            }, 25);
+        };
+        waitFn();
+    };
+    
+    r.afterStartServer = function(app, httpServer, callback)
+    {
+        // worker
+    
+        const io = new Server(httpServer);
+    
+        // use the cluster adapter
+        io.adapter(createAdapter());
+    
+        // setup connection with the primary process
+        setupWorker(io);
+    
+        io.on("connection", (socket) => {
+            // TODO
         });
-    }
-};
+
+        httpServer.io = io;
+
+        return callback();
+    };
+    
+    return r;
+}