cloudcms-server 3.3.1-beta.1 → 3.3.1-beta.10

package/.last_command

@@ -0,0 +1,7 @@
+{
+  "command": {
+    "group": "server",
+    "name": "start"
+  },
+  "arguments": {}
+}

package/broadcast/providers/redis.js

@@ -1,8 +1,7 @@
-var path = require("path");
+var redisClientFactory = require("../../clients/redis");
+var redisHelper = require("../../util/redis");
 
-var NRP = require('node-redis-pubsub');
-
-var logFactory = require("../../util/logger");
+var NRP = require("../../clients/nrp");
 
 /**
  * Redis broadcast provider.
@@ -12,80 +11,56 @@ var logFactory = require("../../util/logger");
 exports = module.exports = function(broadcastConfig)
 {
     var nrp = null;
-
-    var logger = logFactory("REDIS BROADCAST");
-    logger.setLevel("error");
-
-    // allow for global redis default
-    // allow for redis broadcast specific
-    // otherwise default to "error"
-    if (typeof(process.env.CLOUDCMS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else if (typeof(process.env.CLOUDCMS_BROADCAST_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_BROADCAST_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-
+    
+    var logger = redisHelper.redisLogger("REDIS_BROADCAST", "CLOUDCMS_BROADCAST_", "error")
+    
     var r = {};
-
+    
     r.start = function(callback)
     {
-        var redisPort = broadcastConfig.port;
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_BROADCAST_REDIS_PORT;
-        }
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_REDIS_PORT;
-        }
-
-        var redisEndpoint = broadcastConfig.endpoint;
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_BROADCAST_REDIS_ENDPOINT;
-        }
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
-        }
-
-        var nrpConfig = {
-            "port": redisPort,
-            "host": redisEndpoint,
-            "scope": "broadcast_cache"
-        };
-
-        logger.info("using config = " + nrpConfig);
-        
-        nrp = new NRP(nrpConfig);
-
-        callback();
+        redisClientFactory.create(broadcastConfig, function(err, client) {
+            
+            if (err) {
+                return callback(err);
+            }
+            
+            var nrpConfig = {
+                "client": client,
+                "scope": "broadcast_cache"
+            };
+            
+            logger.info("using config = " + nrpConfig);
+            
+            nrp = new NRP(nrpConfig);
+            nrp.connect(function(err) {
+                callback(err);
+            });
+        });
     };
-
+    
     r.publish = function(topic, message, callback)
     {
         logger.info("publish. topic: " + topic + " message: " + message);
         nrp.emit(topic, message);
-
+        
         callback();
     };
-
+    
     r.subscribe = function(topic, fn, callback)
     {
         logger.info("subscribe. topic: " + topic);
         nrp.on(topic, fn);
-
+        
         callback();
     };
-
+    
     r.unsubscribe = function(topic, fn, callback)
     {
         logger.info("unsubscribe. topic: " + topic);
         nrp.off(topic, fn);
-
+        
         callback();
     };
-
+    
     return r;
 };

package/clients/nrp.js

@@ -0,0 +1,117 @@
+// a revision of
+// https://raw.githubusercontent.com/louischatriot/node-redis-pubsub/master/lib/node-redis-pubsub.js
+// that works with Redis 6+
+
+"use strict";
+var redis = require('redis');
+
+/**
+ * Create a new NodeRedisPubsub instance that can subscribe to channels and publish messages
+ * @param {Object} options Options for the client creations:
+ *                 client - a connected Redis client
+ *                 scope - Optional, two NodeRedisPubsubs with different scopes will not share messages
+ */
+function NodeRedisPubsub(options)
+{
+    if (!(this instanceof NodeRedisPubsub)){ return new NodeRedisPubsub(options); }
+    
+    options || (options = {});
+    
+    this.emitter = options.client.duplicate();
+    this.emitter.setMaxListeners(0);
+    this.receiver = options.client.duplicate();
+    this.receiver.setMaxListeners(0);
+    
+    this.prefix = options.scope ? options.scope + ':' : '';
+}
+
+NodeRedisPubsub.prototype.connect = function(callback)
+{
+    var self = this;
+    
+    (async function() {
+        await self.emitter.connect();
+        await self.receiver.connect();
+        callback();
+    })();
+};
+
+/**
+ * Subscribe to a channel
+ * @param {String} channel The channel to subscribe to, can be a pattern e.g. 'user.*'
+ * @param {Function} handler Function to call with the received message.
+ * @param {Function} cb Optional callback to call once the handler is registered.
+ */
+NodeRedisPubsub.prototype.on = NodeRedisPubsub.prototype.subscribe = function(channel, handler, callback)
+{
+    if (!callback)
+    {
+        callback = function(){};
+    }
+    
+    var self = this;
+    
+    if (channel === "error")
+    {
+        self.errorHandler = handler;
+        self.emitter.on("error", handler);
+        self.receiver.on("error", handler);
+        return callback();
+    }
+    
+    var listener = function(self, handler)
+    {
+        return function(message, channel) {
+            
+            var jsonmsg = message;
+            try{
+                jsonmsg = JSON.parse(message);
+            } catch (ex){
+                if(typeof self.errorHandler === 'function'){
+                    return self.errorHandler("Invalid JSON received! Channel: " + self.prefix + channel + " Message: " + message);
+                }
+            }
+            return handler(jsonmsg, channel);
+        }
+    }(self, handler);
+    
+    (async function() {
+        await self.receiver.pSubscribe(self.prefix + channel, listener);
+    })();
+    
+    callback();
+};
+
+/**
+ * Emit an event
+ * @param {String} channel Channel on which to emit the message
+ * @param {Object} message
+ */
+NodeRedisPubsub.prototype.emit = NodeRedisPubsub.prototype.publish = function (channel, message)
+{
+    var self = this;
+    
+    (async function() {
+        return await self.emitter.publish(self.prefix + channel, JSON.stringify(message));
+    })();
+};
+
+/**
+ * Safely close the redis connections 'soon'
+ */
+NodeRedisPubsub.prototype.quit = function()
+{
+    this.emitter.quit();
+    this.receiver.quit();
+};
+
+/**
+ * Dangerously close the redis connections immediately
+ */
+NodeRedisPubsub.prototype.end = function()
+{
+    this.emitter.end(true);
+    this.receiver.end(true);
+};
+
+module.exports = NodeRedisPubsub;

package/clients/redis.js

@@ -0,0 +1,48 @@
+var redis = require("redis");
+var redisHelper = require("../util/redis");
+
+var clients = {};
+
+/**
+ * Redis client factory.
+ *
+ * @type {*}
+ */
+exports = module.exports = {};
+
+var create = exports.create = function(config, callback)
+{
+    if (typeof(config) === "function") {
+        callback = config;
+        config = {};
+    }
+    
+    if (!config) {
+        config = {};
+    }
+    
+    var redisOptions = redisHelper.redisOptions(config);
+    var url = redisOptions.url;
+    
+    // cached client?
+    var client = clients[url];
+    if (client) {
+        return callback(null, client);
+    }
+    
+    // connect
+    (async function() {
+        await redisHelper.createAndConnect(redisOptions, function(err, client) {
+            
+            if (err) {
+                return callback(err);
+            }
+            
+            // cache it
+            clients[url] = client;
+            
+            // return
+            return callback(null, client);
+        });
+    })();
+};

package/duster/helpers/sample/nyt.js

@@ -6,6 +6,7 @@
 exports = module.exports = function(app, dust, callback)
 {
     var support = require("../support")(dust);
+    var request = require("../../util/request");
 
     // helper functions
     var isDefined = support.isDefined;
@@ -49,7 +50,6 @@ exports = module.exports = function(app, dust, callback)
         return map(chunk, function(chunk) {
             setTimeout(function() {
 
-                var request = require("request");
                 var API_KEY = "3d8d573ec0ae966ea57245357cfcf57f:1:70698955";
 
                 var url = "http://api.nytimes.com/svc/events/v2/listings.json?api-key=" + API_KEY;
@@ -77,17 +77,16 @@ exports = module.exports = function(app, dust, callback)
 
                 //console.log("URL:" + url);
 
-                var request = require("request");
-                request(url, function (error, response, body) {
+                request(url, function (error, response, json) {
 
-                    if (error || response.statusCode !== 200)
+                    if (error || response.status !== 200)
                     {
                         if (error) {
                             console.log("ERROR: " + error);
                         }
 
-                        if (response.statusCode !== 200) {
-                            console.log("STATUS CODE: " + response.statusCode);
+                        if (response.status !== 200) {
+                            console.log("STATUS CODE: " + response.status);
                         }
 
                         chunk.write("There was an error loading this section");
@@ -96,7 +95,6 @@ exports = module.exports = function(app, dust, callback)
                         return;
                     }
 
-                    var json = JSON.parse(body);
                     console.log("BODY: " + JSON.stringify(json, null, "  "));
 
                     var resultObject = {

package/framework/controllers.js

@@ -2,7 +2,6 @@ var http = require("http");
 var path = require("path");
 var fs = require("fs");
 var util = require("../util/util");
-var request = require("request");
 
 var Gitana = require("gitana");
 

package/index.js

@@ -34,7 +34,7 @@ process.logInfo = process.log = function(text, level)
 var Gitana = require("gitana");
 
 // default http timeout
-process.defaultHttpTimeoutMs = 120000; // 2 minutes
+process.defaultHttpTimeoutMs = 60000;
 
 if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
 {
@@ -54,20 +54,18 @@ var HttpsKeepAliveAgent = require('agentkeepalive').HttpsAgent;
 http.globalAgent = new HttpKeepAliveAgent({
     keepAlive: true,
     keepAliveMsecs: 1000,
-    keepAliveTimeout: 30000,
+    maxSockets: 16000,
+    maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    maxSockets: 200,
-    maxFreeSockets: 40,
-    rejectUnauthorized: false
+    freeSocketTimeout: 30000
 });
 https.globalAgent = new HttpsKeepAliveAgent({
     keepAlive: true,
     keepAliveMsecs: 1000,
-    keepAliveTimeout: 30000,
+    maxSockets: 16000,
+    maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    maxSockets: 200,
-    maxFreeSockets: 40,
-    rejectUnauthorized: false
+    freeSocketTimeout: 30000
 });
 
 // disable for now
@@ -126,7 +124,7 @@ exports = module.exports = function()
     // TODO: this is to disable really annoying Express 3.0 deprecated's for multipart() which should hopefully
     // TODO: be resolved soon
     console.warn = function() {};
-
+    
     process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
 
     // assume app-server base path if none provided
@@ -139,6 +137,7 @@ exports = module.exports = function()
     var defaultGitanaProxyScheme = "https";
     var defaultGitanaProxyHost = "api.cloudcms.com";
     var defaultGitanaProxyPort = 443;
+    var defaultGitanaProxyPath = "";
 
     var gitanaJsonPath = path.join(process.env.CLOUDCMS_APPSERVER_BASE_PATH, "gitana.json");
     if (fs.existsSync(gitanaJsonPath))
@@ -150,6 +149,7 @@ exports = module.exports = function()
 
             defaultGitanaProxyHost = parsedUrl.hostname;
             defaultGitanaProxyScheme = parsedUrl.protocol.substring(0, parsedUrl.protocol.length - 1); // remove the :
+            defaultGitanaProxyPath = parsedUrl.path;
 
             if (parsedUrl.port)
             {
@@ -176,10 +176,13 @@ exports = module.exports = function()
     if (!process.env.GITANA_PROXY_PORT) {
         process.env.GITANA_PROXY_PORT = defaultGitanaProxyPort;
     }
-
+    if (!process.env.GITANA_PROXY_PATH) {
+        process.env.GITANA_PROXY_PATH = defaultGitanaProxyPath;
+    }
+    
     if (cluster.isMaster)
     {
-        process.log("Gitana Proxy pointed to: " + util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT));
+        process.log("Gitana Proxy pointed to: " + util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
     }
 
     // all web modules are included by default
@@ -218,6 +221,7 @@ exports = module.exports = function()
     var serverTags = require("./middleware/server-tags/server-tags");
     var storeService = require("./middleware/stores/stores");
     var templates = require("./middleware/templates/templates");
+    var themes = require("./middleware/themes/themes");
     var virtualConfig = require("./middleware/virtual-config/virtual-config");
     var virtualFiles = require("./middleware/virtual-files/virtual-files");
     var wcm = require("./middleware/wcm/wcm");
@@ -491,6 +495,9 @@ exports = module.exports = function()
         // handles deploy/undeploy commands
         app.use(deployment.handler());
 
+        // supports invalidation of configuration
+        app.use(config.invalidateConfigHandler());
+
         // serve back static configuration
         app.use(config.staticConfigHandler());
 
@@ -499,7 +506,10 @@ exports = module.exports = function()
 
         // handles calls to the templates service
         app.use(templates.handler());
-
+    
+        // handles calls to the themes service
+        app.use(themes.handler());
+    
         // handles calls to the modules service
         app.use(modules.handler());
 

package/insight/insight.js

@@ -1,6 +1,3 @@
-var http = require("http");
-var path = require("path");
-var request = require("request");
 var util = require("../util/util");
 
 /**
@@ -131,7 +128,7 @@ var doSend = function(callback)
     }
 
     // url over to cloud cms
-    var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + "/warehouses/" + warehouseId + "/interactions/_create";
+    var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + "/warehouses/" + warehouseId + "/interactions/_create";
     var requestConfig = {
         "url": URL,
         "qs": {},
@@ -145,15 +142,15 @@ var doSend = function(callback)
 
     // make a single attempt to send the data over
     // if it fails, we add it back to the queue
-    util.retryGitanaRequest(log, gitana, requestConfig, 1, function(err, response, body) {
+    util.retryGitanaRequest(log, gitana, requestConfig, 1, function(err, response, json) {
 
-        if (response && response.statusCode === 200 && body)
+        if (response && response.status === 200 && json)
         {
             console.log("Insight sync for warehouse: " + warehouseId + " succeeded");
         }
         else
         {
-            if (err || (body && body.error))
+            if (err || (json && json.error))
             {
                 console.log("Insight sync for warehouse: " + warehouseId + " failed");
 
@@ -169,9 +166,9 @@ var doSend = function(callback)
                 }
                 */
 
-                if (body && body.error)
+                if (json && json.error)
                 {
-                    console.log(" -> body: " + JSON.stringify(body));
+                    console.log(" -> body: " + JSON.stringify(json));
                 }
             }
         }

package/locks/providers/redis.js

@@ -1,6 +1,8 @@
-var path = require("path");
-var redis = require("redis");
 var logFactory = require("../../util/logger");
+//var redisHelper = require("../../util/redis");
+
+var redisClientFactory = require("../../clients/redis");
+const redisHelper = require("../../util/redis");
 
 /**
  * Redis lock service.
@@ -15,94 +17,63 @@ exports = module.exports = function(locksConfig)
         retries: 2000,
         delay: 50
     });
-
-    var nrp = null;
+    
     var client = null;
-
-    var logger = logFactory("REDIS LOCK");
-
-    // allow for global redis default
-    // allow for redis broadcast specific
-    // otherwise default to error
-    if (typeof(process.env.CLOUDCMS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else if (typeof(process.env.CLOUDCMS_LOCKS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_LOCKS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else {
-        logger.setLevel("error");
-    }
-
+    
+    var logger = redisHelper.redisLogger("REDIS_LOCKS", "CLOUDCMS_LOCKS_", "error")
+    
     var r = {};
-
+    
     r.init = function(callback)
     {
-        var redisPort = locksConfig.port;
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_LOCKS_REDIS_PORT;
-        }
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_REDIS_PORT;
-        }
-
-        var redisEndpoint = locksConfig.endpoint;
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_LOCKS_REDIS_ENDPOINT;
-        }
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
-        }
-
-        var redisOptions = {};
-
-        //redis.debug_mode = true;
-
-        client = redis.createClient(redisPort, redisEndpoint, redisOptions);
-
-        callback();
+        redisClientFactory.create(locksConfig, function(err, _client) {
+            
+            if (err) {
+                return callback(err);
+            }
+            
+            client = _client;
+            
+            return callback();
+        });
     };
-
+    
     r.lock = function(key, fn)
     {
         var lockKey = "cloudcms:locks:write:" + key;
-
+        
         var lock = redisLock.createLock(client);
-
+        
         var releaseCallbackFn = function(lock, lockKey) {
             return function() {
                 logger.info("lock.release - " + lockKey);
                 lock.release(function(err) {
-
+                    
                     if (err) {
                         console.log("Failed to release redis lock: " + lockKey);
                         console.log("Error: " + err);
                         return;
                     }
-
+                    
                     logger.info("lock.released - " + lockKey);
                 });
             }
         }(lock, lockKey);
-
+        
         logger.info("lock.acquire - " + lockKey);
         lock.acquire(lockKey, function(err) {
-
+            
             if (err) {
                 console.log("Failed to acquire redis lock: " + lockKey);
                 console.log("Error: " + err);
                 return;
             }
-
+            
             logger.info("lock.acquired - " + lockKey);
-
+            
             fn(releaseCallbackFn);
         });
     };
-
+    
     return r;
 };

package/middleware/admin/admin.js

@@ -17,8 +17,8 @@ exports = module.exports = function()
         if (ref)
         {
             var z = ref.indexOf("://");
-
-            var type = ref.substring(0, z + 3);
+    
+            var type = ref.substring(0, z);
             var identifier = ref.substring(z + 3);
 
             var parts = identifier.split("/").reverse();
@@ -131,8 +131,8 @@ exports = module.exports = function()
                 if (req.url.indexOf("/_admin/cache/reset") === 0 || req.url.indexOf("/_admin/cache/invalidate") === 0)
                 {
                     assertAuthenticated(req, res, function() {
-
-                        doResetCache(req.virtualHost, req.ref, function(err) {
+    
+                        doResetCache(req.virtualHost, req.query.ref, function(err) {
                             completionFn(req.virtualHost, res, err);
                         });
 

package/middleware/awareness/awareness.js

@@ -14,7 +14,10 @@ exports = module.exports = function()
     var REAP_FREQUENCY_MS = 3000; // three seconds
     var REAP_MAX_AGE_MS = 5000; // five seconds
 
-    var pluginPaths = ["./plugins/editorial"];
+    var pluginPaths = [
+        "./plugins/editorial",
+        "./plugins/resources"
+    ];
     var plugins = {};
 
     // ensure reaper only initializes once