cloudcms-server 3.3.1-beta.0 → 3.3.1-beta.11

package/server/index.js

@@ -16,10 +16,14 @@ var session = require('express-session');
 var cookieParser = require('cookie-parser');
 var flash = require("connect-flash");
 
+const redis = require('redis');
+const connectRedis = require('connect-redis');
+
 // we don't bind a single passport - instead, we get the constructor here by hand
 var Passport = require("passport").Passport;
 
 var util = require("../util/util");
+var redisHelper = require("../util/redis");
 
 var launchPad = require("../launchpad/index");
 var cluster = require("cluster");
@@ -37,6 +41,8 @@ var duster = require("../duster/index");
 
 var coreHelpers = require("../duster/helpers/core/index");
 
+var helmet = require("helmet");
+
 var toobusy = require("toobusy-js");
 toobusy.maxLag(500); // 500 ms lag in event queue, quite high but usable for now
 toobusy.interval(250);
@@ -110,6 +116,7 @@ var SETTINGS = {
             "web": "app",
             "content": "tmp",
             "templates": "app",
+            "themes": "app",
             "modules": "app"
         },
         "virtual": {
@@ -118,6 +125,7 @@ var SETTINGS = {
             "web": "tmp",
             "content": "tmp",
             "templates": "tmp",
+            "themes": "tmp",
             "modules": "tmp"
         },
         "oneteam": {
@@ -126,6 +134,7 @@ var SETTINGS = {
             "web": "app",
             "content": "hosts_fs",
             "templates": "app",
+            "themes": "app",
             "modules": "hosts_fs"
         },
         "net-development": {
@@ -134,6 +143,7 @@ var SETTINGS = {
             "web": "hosts_fs",
             "content": "hosts_fs",
             "templates": "hosts_fs",
+            "themes": "hosts_fs",
             "modules": "hosts_fs"
         },
         "net-production": {
@@ -142,6 +152,7 @@ var SETTINGS = {
             "web": "hosts_s3fs",
             "content": "hosts_s3fs",
             "templates": "hosts_s3fs",
+            "themes": "hosts_s3fs",
             "modules": "hosts_s3fs"
         },
         "net-development-s3": {
@@ -149,7 +160,8 @@ var SETTINGS = {
             "config": "hosts_s3",
             "web": "hosts_s3",
             "content": "hosts_s3",
-            "templates": "hosts_s3"
+            "templates": "hosts_s3",
+            "themes": "hosts_s3"
         },
         "net-development-s3fs": {
             "root": "hosts_s3fs",
@@ -157,6 +169,7 @@ var SETTINGS = {
             "web": "hosts_s3fs",
             "content": "hosts_s3fs",
             "templates": "hosts_s3fs",
+            "themes": "hosts_s3fs",
             "modules": "hosts_s3fs"
         }
     },
@@ -308,12 +321,12 @@ var SETTINGS = {
 process.env.PORT = process.env.PORT || 2999;
 
 // allows for specification of alternative transports
-SETTINGS.socketTransports = [
-    'websocket',
-    'xhr-polling',
-    'jsonp-polling',
-    'polling'
-];
+// SETTINGS.socketTransports = [
+//     'websocket',
+//     'xhr-polling',
+//     'jsonp-polling',
+//     'polling'
+// ];
 
 var exports = module.exports;
 
@@ -585,7 +598,50 @@ var startSlave = function(config, afterStartFn)
     if (!process.env.CLOUDCMS_STANDALONE_HOST) {
         process.env.CLOUDCMS_STANDALONE_HOST = "local";
     }
-
+    
+    // auto-configuration for HTTPS
+    if (!process.configuration.https) {
+        process.configuration.https = {};
+    }
+    if (process.env.CLOUDCMS_HTTPS) {
+        process.configuration.https = JSON.parse(process.env.CLOUDCMS_HTTPS);
+    }
+    if (process.env.CLOUDCMS_HTTPS_KEY_FILEPATH) {
+        process.configuration.https.key = fs.readFileSync(process.env.CLOUDCMS_HTTPS_KEY_FILEPATH);
+    }
+    if (process.env.CLOUDCMS_HTTPS_CERT_FILEPATH) {
+        process.configuration.https.cert = fs.readFileSync(process.env.CLOUDCMS_HTTPS_CERT_FILEPATH);
+    }
+    if (process.env.CLOUDCMS_HTTPS_PFX_FILEPATH) {
+        process.configuration.https.pfx = fs.readFileSync(process.env.CLOUDCMS_HTTPS_PFX_FILEPATH);
+    }
+    if (process.env.CLOUDCMS_HTTPS_PASSPHRASE) {
+        process.configuration.https.passphrase = process.env.CLOUDCMS_HTTPS_PASSPHRASE;
+    }
+    if (process.env.CLOUDCMS_HTTPS_REQUEST_CERT === "true") {
+        process.configuration.https.requestCert = true;
+    }
+    if (process.env.CLOUDCMS_HTTPS_CA_FILEPATH) {
+        process.configuration.https.ca = [ fs.readFileSync(process.env.CLOUDCMS_HTTPS_CA_FILEPATH) ];
+    }
+    
+    // if https config is empty, remove it
+    if (Object.keys(process.configuration.https).length === 0) {
+        delete process.configuration.https;
+    }
+    
+    // auto configuration of session store
+    if (!process.configuration.session) {
+        process.configuration.session = {};
+    }
+    if (process.env.CLOUDCMS_SESSION_TYPE) {
+        process.configuration.session.enabled = true;
+        process.configuration.session.type = process.env.CLOUDCMS_SESSION_TYPE;
+    }
+    if (process.env.CLOUDCMS_SESSION_SECRET) {
+        process.configuration.session.secret = process.env.CLOUDCMS_SESSION_SECRET;
+    }
+    
     // session store
     var initializedSession = null;
     if (process.configuration.session)
@@ -618,6 +674,23 @@ var startSlave = function(config, afterStartFn)
                 var SessionFileStore = require('session-file-store')(session);
                 sessionConfig.store = new SessionFileStore(options);
             }
+            else if (process.configuration.session.type === "redis")
+            {
+                (async function() {
+                    var redisOptions = redisHelper.redisOptions();
+                    redisHelper.createAndConnect(redisOptions, function(err, redisClient) {
+            
+                        if (err) {
+                            console.error(err);
+                        }
+                        else
+                        {
+                            var RedisStore = connectRedis(session);
+                            sessionConfig.store = new RedisStore({ client: redisClient });
+                        }
+                    });
+                })();
+            }
             else if (process.configuration.session.type === "memory" || !process.configuration.session.type)
             {
                 var options = {};
@@ -1043,25 +1116,34 @@ var startSlave = function(config, afterStartFn)
                                         // INITIALIZE THE SERVER
                                         //
                                         ////////////////////////////////////////////////////////////////////////////
-
-
-                                        // CORE OBJECTS
-                                        var server = http.Server(app);
-
+    
+                                        // create the server (either HTTP or HTTPS)
+                                        var httpServer = null;
+                                        if (process.configuration.https) {
+                                            // configure helmet to support auto-upgrade of http->https
+                                            app.use(helmet());
+                                            // create https server
+                                            httpServer = https.createServer(process.configuration.https, app);
+                                        } else {
+                                            // legacy
+                                            httpServer = http.Server(app);
+                                        }
+    
                                         // request timeout
                                         var requestTimeout = 30000; // 30 seconds
                                         if (process.configuration && process.configuration.timeout)
                                         {
                                             requestTimeout = process.configuration.timeout;
                                         }
-                                        server.setTimeout(requestTimeout);
-
+                                        httpServer.setTimeout(requestTimeout);
+    
                                         // socket
-                                        server.on("connection", function (socket) {
+                                        httpServer.on("connection", function (socket) {
                                             socket.setNoDelay(true);
                                         });
-                                        var io = process.IO = require("socket.io")(server);
-                                        io.set('transports', config.socketTransports);
+                                        const { Server } = require("socket.io");
+                                        var io = process.IO = new Server(httpServer);
+                                        //io.set('transports', config.socketTransports);
                                         io.use(function (socket, next) {
 
                                             // console.log("New socket being initialized");
@@ -1152,8 +1234,8 @@ var startSlave = function(config, afterStartFn)
 
                                             // APPLY SERVER BEFORE START FUNCTIONS
                                             runFunctions(config.beforeFunctions, [app], function (err) {
-
-                                                server._listenPort = app.get("port");
+    
+                                                httpServer._listenPort = app.get("port");
 
                                                 // AFTER SERVER START
                                                 runFunctions(config.afterFunctions, [app], function (err) {
@@ -1173,7 +1255,7 @@ var startSlave = function(config, afterStartFn)
 
                                                         try
                                                         {
-                                                            server.close();
+                                                            httpServer.close();
                                                         }
                                                         catch (e)
                                                         {
@@ -1213,8 +1295,8 @@ var startSlave = function(config, afterStartFn)
                                                     {
                                                         process.send("server-startup");
                                                     }
-
-                                                    afterStartFn(app, server);
+    
+                                                    afterStartFn(app, httpServer);
 
                                                 });
                                             });

package/server/standalone.js

@@ -46,6 +46,7 @@ server.report(function(callback) {
     console.log("Gitana Scheme: " + process.env.GITANA_PROXY_SCHEME);
     console.log("Gitana Host: " + process.env.GITANA_PROXY_HOST);
     console.log("Gitana Port: " + process.env.GITANA_PROXY_PORT);
+    console.log("Gitana Path: " + process.env.GITANA_PROXY_PATH);
     console.log("CPU Count: " + cpuCount);
 
     var virtualHost = null;
@@ -68,6 +69,7 @@ server.report(function(callback) {
     console.log("Hosts Directory: " + process.env.CLOUDCMS_HOSTS_PATH);
     console.log("LaunchPad Mode: " + process.env.CLOUDCMS_LAUNCHPAD_SETUP);
     console.log("Max Files Detected: " + process.env.CLOUDCMS_MAX_FILES);
+    console.log("Session Type: " + process.configuration.session.type);
 
     console.log("");
 

package/util/auth.js

@@ -1,21 +1,23 @@
-var path = require('path');
-var fs = require('fs');
+// var path = require('path');
+// var fs = require('fs');
 var os = require('os');
 var _util = require("util");
-var util = require("./util");
-var request = require("request");
-var http = require("http");
-var https = require("https");
+// var util = require("./util");
+// var http = require("http");
+// var https = require("https");
 var async = require("async");
+var LRUCache = require("lru-cache");
+
+var request = require("./request");
 
 // trusted profile cache size 100
-var TRUSTED_PROFILE_CACHE = require("lru-cache")({
+var TRUSTED_PROFILE_CACHE = new LRUCache({
     max:100,
     maxAge: 1000 * 60 * 15 // 15 minutes
 });
 
 // user entry cache size 100
-var USER_ENTRY_CACHE = require("lru-cache")({
+var USER_ENTRY_CACHE = new LRUCache({
     max: 100,
     maxAge: 1000 * 60 * 15 // 15 minutes
 });
@@ -137,15 +139,12 @@ var impersonate = exports.impersonate = function(req, key, targetUser, callback)
         var headers = {};
         headers["Authorization"] = req.gitana.platform().getDriver().getHttpHeaders()["Authorization"];
 
-        var agent = util.getAgent(req.gitanaConfig.baseURL);
-
         request({
             "method": "POST",
             "url": req.gitanaConfig.baseURL + "/auth/impersonate/" + targetUser.getDomainId() + "/" + targetUser.getId(),
             "qs": {},
             "json": {},
             "headers": headers,
-            "agent": agent,
             "timeout": process.defaultHttpTimeoutMs
         }, function(err, response, json) {
 
@@ -440,8 +439,6 @@ var __handleSyncUser = function(req, strategy, settings, key, domainId, provider
     var headers = {};
     headers["Authorization"] = authorizationHeader;
 
-    var agent = util.getAgent(req.gitanaConfig.baseURL);
-
     if (!userObject) {
         userObject = {};
     }
@@ -467,7 +464,6 @@ var __handleSyncUser = function(req, strategy, settings, key, domainId, provider
         },
         "json": json,
         "headers": headers,
-        "agent": agent,
         "timeout": process.defaultHttpTimeoutMs
     };
 

package/util/cloudcms.js

@@ -1,10 +1,11 @@
 var path = require('path');
-var fs = require('fs');
+//var fs = require('fs');
 var util = require("./util");
-var request = require("request");
 
-var http = require("http");
-var https = require("https");
+// var http = require("http");
+// var https = require("https");
+
+var request = require("./request");
 
 exports = module.exports = function()
 {
@@ -346,12 +347,10 @@ exports = module.exports = function()
         {
             if (attemptCount === maxAttemptsAllowed)
             {
-                cb({
+                return cb({
                     "message": "Maximum number of connection attempts exceeded(" + maxAttemptsAllowed + ")",
                     "err": previousError
                 });
-
-                return;
             }
 
             contentStore.writeStream(filePath, function(err, tempStream) {
@@ -375,28 +374,24 @@ exports = module.exports = function()
                 // add "authorization" for OAuth2 bearer token
                 var headers2 = gitana.platform().getDriver().getHttpHeaders();
                 headers["Authorization"] = headers2["Authorization"];
-
-                var agent = http.globalAgent;
-                if (process.env.GITANA_PROXY_SCHEME === "https")
-                {
-                    agent = https.globalAgent;
-                }
-
-                var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + uri;
+    
+                var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + uri;
                 request({
                     "method": "GET",
                     "url": URL,
                     "qs": {},
                     "headers": headers,
-                    "timeout": process.defaultHttpTimeoutMs,
-                    "agent": agent
-                }).on('response', function (response) {
+                    "responseType": "stream"
+                }, function(err, response) {
 
-                    //process.log("Status Code: " + response.statusCode);
+                    if (err) {
+                        closeWriteStream(tempStream);
+                        return cb(err);
+                    }
 
-                    if (response.statusCode >= 200 && response.statusCode <= 204)
+                    if (response.status >= 200 && response.status <= 204)
                     {
-                        response.pipe(tempStream).on("close", function (err) {
+                        response.data.pipe(tempStream).on("close", function (err) {
 
                             // TODO: not needed here?
                             // ensure stream is closed
@@ -463,6 +458,8 @@ exports = module.exports = function()
 
                         }).on("error", function (err) {
 
+                            console.log("f2");
+
                             // ensure stream is closed
                             closeWriteStream(tempStream);
 
@@ -475,7 +472,7 @@ exports = module.exports = function()
 
                         var body = "";
 
-                        response.on('data', function (chunk) {
+                        response.data.on('data', function (chunk) {
                             body += chunk;
                         });
 
@@ -507,7 +504,7 @@ exports = module.exports = function()
                                     // fire for retry
                                     return _refreshAccessTokenAndRetry(contentStore, gitana, uri, filePath, attemptCount, maxAttemptsAllowed, {
                                         "message": "Unable to load asset from remote store",
-                                        "code": response.statusCode,
+                                        "code": response.status,
                                         "body": body
                                     }, cb);
                                 }
@@ -515,7 +512,7 @@ exports = module.exports = function()
                                 // otherwise, it's not worth retrying at this time
                                 cb({
                                     "message": "Unable to load asset from remote store",
-                                    "code": response.statusCode,
+                                    "code": response.status,
                                     "body": body
                                 });
 
@@ -531,19 +528,7 @@ exports = module.exports = function()
 
                     }
 
-                }).on('error', function (e) {
-
-                    // ensure stream is closed
-                    closeWriteStream(tempStream);
-
-                    process.log("_writeToDisk request timed out");
-                    process.log(e)
-                }).on('end', function (e) {
-
-                    // ensure stream is closed
-                    closeWriteStream(tempStream);
-
-                }).end();
+                });
 
                 tempStream.on("error", function (e) {
                     process.log("Temp stream errored out");

package/util/proxy-factory.js

@@ -11,6 +11,8 @@ var oauth2 = require("./oauth2")();
 
 var urlTool = require("url");
 
+var LRUCache = require("lru-cache");
+
 var exports = module.exports;
 
 var _LOCK = function(lockIdentifiers, workFunction)
@@ -18,7 +20,7 @@ var _LOCK = function(lockIdentifiers, workFunction)
     process.locks.lock(lockIdentifiers.join("_"), workFunction);
 };
 
-var NAMED_PROXY_HANDLERS_CACHE = require("lru-cache")({
+var NAMED_PROXY_HANDLERS_CACHE = new LRUCache({
     max: 200,
     maxAge: 1000 * 60 * 60 // 60 minutes
 });
@@ -97,15 +99,25 @@ var createProxyHandler = function(proxyTarget, pathPrefix)
 
     // create proxy server instance
     var proxyServer = new httpProxy.createProxyServer(proxyConfig);
-
+    
     // error handling
     proxyServer.on("error", function(err, req, res) {
-        process.log(err);
-        res.writeHead(500, {
-            'Content-Type': 'text/plain'
-        });
-
-        res.end('Something went wrong while proxying the request.');
+        console.log("A proxy error was caught: " + err + ", json: " + JSON.stringify(err) + ", path: " + req.path);
+        
+        // do our best to send something back
+        try
+        {
+            res.writeHead(500, {
+                'Content-Type': 'text/plain'
+            });
+        }
+        catch (e) { }
+        
+        try
+        {
+            res.end('Something went wrong while proxying the request.');
+        }
+        catch (e) { }
     });
 
     // if we're using auth credentials that are picked up in SSO chain, then we listen for a 401

package/util/redis.js

@@ -0,0 +1,113 @@
+var redis = require("redis");
+const logFactory = require("./logger");
+
+exports = module.exports;
+
+var redisLogger = exports.redisLogger = function(name, prefix, defaultLevel)
+{
+    if (!defaultLevel) {
+        defaultLevel = "error";
+    }
+    
+    var level = null;
+    
+    // allow for global redis default
+    // allow for prefix specific
+    if (typeof(process.env["CLOUDCMS_REDIS_DEBUG_LEVEL"]) !== "undefined") {
+        level = "" + process.env["CLOUDCMS_REDIS_DEBUG_LEVEL"].toLowerCase();
+    }
+    
+    if (!level && prefix)
+    {
+        if (typeof(process.env[prefix + "REDIS_DEBUG_LEVEL"]) !== "undefined") {
+            level = "" + process.env[prefix + "REDIS_DEBUG_LEVEL"].toLowerCase();
+        }
+    }
+    
+    if (!level) {
+        level = defaultLevel;
+    }
+    
+    var logger = logFactory(name);
+    logger.setLevel(level);
+    
+    return logger;
+}
+
+var redisOptions = exports.redisOptions = function(config, prefix)
+{
+    if (!config) {
+        config = {};
+    }
+    
+    // redis port
+    var redisPort = config.port;
+    if (prefix)
+    {
+        if (typeof(redisPort) === "undefined" || !redisPort)
+        {
+            // CLOUDCMS_LOCKS_REDIS_PORT;
+            redisPort = process.env[prefix + "_REDIS_PORT"];
+        }
+    }
+    if (typeof(redisPort) === "undefined" || !redisPort)
+    {
+        redisPort = process.env.CLOUDCMS_REDIS_PORT;
+    }
+    
+    // redis host
+    var redisEndpoint = config.endpoint;
+    if (prefix)
+    {
+        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
+        {
+            redisEndpoint = process.env[prefix + "_REDIS_ENDPOINT"];
+        }
+    }
+    if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
+    {
+        redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
+    }
+    
+    // redis url
+    var redisUrl = config.url;
+    if (prefix)
+    {
+        if (typeof(redisUrl) === "undefined" || !redisUrl)
+        {
+            redisUrl = process.env[prefix + "_REDIS_URL"];
+        }
+    }
+    if (typeof(redisUrl) === "undefined" || !redisUrl)
+    {
+        redisUrl = process.env.CLOUDCMS_REDIS_URL;
+    }
+    
+    // build redis URL from components if not otherwise provided
+    if (!redisUrl)
+    {
+        redisUrl = "redis://" + redisEndpoint + ":" + redisPort;
+    }
+    
+    var redisOptions = {};
+    redisOptions.url = redisUrl;
+    
+    return redisOptions;
+}
+
+var createAndConnect = exports.createAndConnect = async function(redisOptions, callback)
+{
+    var client = redis.createClient(redisOptions);
+    
+    var connectErr = null;
+    client.on('error', function(err) {
+        console.log('Redis Client Error', err);
+        connectErr = err;
+    });
+    
+    // connect
+    await client.connect();
+    //console.log("Connected to redis, options: " + JSON.stringify(redisOptions, null, 2) + ", err: " + connectErr + ", client: " + client);
+    
+    return callback(connectErr, client);
+}

package/util/renditions.js

@@ -1,13 +1,14 @@
-var path = require('path');
-var fs = require('fs');
+// var path = require('path');
+// var fs = require('fs');
 var util = require("../util/util");
-var async = require("async");
-var request = require("request");
+//var async = require("async");
 
-var http = require("http");
-var https = require("https");
+// var http = require("http");
+// var https = require("https");
 
-var logFactory = require("../util/logger");
+var logFactory = require("./logger");
+
+var request = require("./request");
 
 /**
  * WCM Resource Dependency Manager
@@ -61,13 +62,7 @@ exports = module.exports = function()
 
         var syncRows = function(rows, callback)
         {
-            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + "/bulk/pagerenditions";
-
-            var agent = http.globalAgent;
-            if (process.env.GITANA_PROXY_SCHEME === "https")
-            {
-                agent = https.globalAgent;
-            }
+            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + "/bulk/pagerenditions";
 
             // add "authorization" for OAuth2 bearer token
             var headers = {};
@@ -82,9 +77,8 @@ exports = module.exports = function()
                     "rows": rows
                 },
                 "headers": headers,
-                "timeout": process.defaultHttpTimeoutMs,
-                "agent": agent
-            }, function (err, response, body) {
+                "timeout": process.defaultHttpTimeoutMs
+            }, function (err, response, json) {
                 callback();
             });
         };