cloudcms-server 3.3.1-beta.9 → 4.0.0-beta.10

package/middleware/cache/providers/redis.js

@@ -1,7 +1,5 @@
-var path = require("path");
-
-var redis = require("redis");
-var logFactory = require("../../../util/logger");
+var redisClientFactory = require("../../../clients/redis");
+const redisHelper = require("../../../util/redis");
 
 /**
  * Redis distributed cache.
@@ -11,123 +9,163 @@ var logFactory = require("../../../util/logger");
 exports = module.exports = function(cacheConfig)
 {
     var client = null;
-
-    var logger = this.logger = logFactory("REDIS CACHE");
-    logger.setLevel("error");
-
-    // allow for global redis default
-    // allow for redis broadcast specific
-    // otherwise default to error
-    if (typeof(process.env.CLOUDCMS_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
-    else if (typeof(process.env.CLOUDCMS_CACHE_REDIS_DEBUG_LEVEL) !== "undefined") {
-        logger.setLevel(("" + process.env.CLOUDCMS_CACHE_REDIS_DEBUG_LEVEL).toLowerCase(), true);
-    }
+    
+    var logger = redisHelper.redisLogger("REDIS_CACHE", "CLOUDCMS_CACHE_", "error")
 
     var r = {};
 
     r.init = function(callback)
     {
-        var redisPort = cacheConfig.port;
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_CACHE_REDIS_PORT;
-        }
-        if (typeof(redisPort) === "undefined" || !redisPort)
-        {
-            redisPort = process.env.CLOUDCMS_REDIS_PORT;
-        }
-
-        var redisEndpoint = cacheConfig.endpoint;
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_CACHE_REDIS_ENDPOINT;
-        }
-        if (typeof(redisEndpoint) === "undefined" || !redisEndpoint)
-        {
-            redisEndpoint = process.env.CLOUDCMS_REDIS_ENDPOINT;
-        }
-
-        var redisOptions = {};
-
-        //redis.debug_mode = true;
-
-        client = redis.createClient(redisPort, redisEndpoint, redisOptions);
-
-        callback();
+        redisClientFactory.create(cacheConfig, function(err, _client) {
+    
+            if (err) {
+                return callback(err);
+            }
+    
+            client = _client;
+    
+            return callback();
+    
+        });
     };
 
     r.write = function(key, value, seconds, callback)
     {
-        if (seconds <= -1)
-        {
-            client.set([key, JSON.stringify(value)], function(err, reply) {
-                if (err) {
-                    logger.error("write error. key: " + key + " value: " + JSON.stringify(value) + ". error:" + err);
+        logger.info('write, key = ' + key + ', value = ' + value + ', seconds = ' + seconds + ', typeofval: ' + typeof(value));
+        (async function() {
+    
+            var reply = null;
+            var err = null;
+    
+            try
+            {
+                var config = {};
+                if (seconds >= 0) {
+                    config["EX"] = seconds;
                 }
+                
+                reply = await client.set(key, JSON.stringify(value), config);
+            }
+            catch (e)
+            {
+                err = e;
+            }
+    
+            if (reply) {
                 logger.info("write -> reply = " + reply);
-                callback(err, reply);
-            });
-        }
-        else
-        {
-            client.set([key, JSON.stringify(value), "EX", seconds], function(err, reply) {
-                if (err) {
-                    logger.error("write.ex error. key: " + key + " value: " + JSON.stringify(value) + ". error:" + err);
-                }
-                logger.info("write.ex -> reply = " + reply);
-                callback(err, reply);
-            });
-        }
+            }
+    
+            if (err) {
+                logger.error("write error. key: " + key + " value: " + JSON.stringify(value) + ". error:" + err + ", value type: " + typeof(value));
+            }
+    
+            callback(err, reply);
+            
+        })();
     };
 
-    r.read = function(key, callback)
-    {
-        client.get([key], function(err, reply) {
+    r.read = function(key, callback)
+    {
+        logger.info('read, key = ' + key);
 
+        (async function() {
+    
+            var err = null;
+            var reply = null;
+    
+            try
+            {
+                reply = await client.get(key);
+            }
+            catch (e)
+            {
+                err = e;
+            }
+    
             if (err) {
-                logger.error("read error. key: " + key + ". error:" + err);
+                logger.error("read error. key: " + key + ". error: " + err);
+            }
+    
+            if (reply) {
+                logger.info("read. key: " + key + " -> reply = " + reply);
             }
-            logger.info("read. key: " + key + " -> reply = " + reply);
-            
+    
             var result = null;
-            try
+            try
             {
-                result = JSON.parse(reply);
-            }
+                result = JSON.parse(reply);
+            }
             catch (ex)
             {
-                result = null;
-                err = ex;
-                if (err) {
+                result = null;
+                err = ex;
+        
+                if (err)
+                {
                     logger.error("error parsing reply. key: " + key + ". error:" + err);
                 }
-            }
-
+            }
+    
             callback(err, result);
-        });
-    };
+        })();
+    };
 
     r.remove = function(key, callback)
     {
-        logger.info("remove. key: " + key);
-        client.del([key], function(err) {
+        logger.info('remove, key = ' + key);
+    
+        (async function() {
+    
+            var err = null;
+            
+            try
+            {
+                await client.del(key);
+                logger.info("remove. key: " + key);
+            }
+            catch (e)
+            {
+                err = e;
+            }
+    
+            if (err) {
+                logger.error("del error. key: " + key + ". error: " + err);
+            }
+    
             callback(err);
-        });
+        })();
     };
     
     r.keys = function(prefix, callback)
-    {
-        logger.info('keys. prefix = ' + prefix);
-        client.keys([prefix + '*'], function(err, reply) {
+    {
+        logger.info('keys, prefix = ' + prefix);
+        
+        (async function() {
+    
+            var err = null;
+            var reply = null;
+    
+            try
+            {
+                reply = await client.keys(prefix + '*');
+            }
+            catch (e)
+            {
+                err = e;
+            }
+    
             if (err) {
-                logger.error("error reading prefix: " + prefix + ". error:" + err);
+                logger.error("error reading keys for prefix: " + prefix + ". error:" + err);
             }
-            logger.info("[keys -> reply = " + reply);
-            callback(err, reply);
-        });
-    };
+    
+            if (reply) {
+                logger.info("keys -> reply = " + reply);
+            }
+    
+            callback(err, reply);
+            
+        })();
+    };
 
     return r;
 };

package/middleware/cache/providers/shared-memory.js

@@ -1,7 +1,7 @@
-var path = require("path");
-var memored = require('../../../temp/memored');
+//var path = require("path");
+var memored = require("memored");
 
-var cluster = require("cluster");
+//var cluster = require("cluster");
 
 /**
  * Shared cluster memory using memored

package/middleware/cloudcms/cloudcms.js

@@ -436,8 +436,18 @@ exports = module.exports = function()
         }
 
         // only allow one "thread" at a time to load the branch
-        _LOCK(cacheKey, function(releaseLockFn) {
-
+        _LOCK(cacheKey, function(err, releaseLockFn) {
+            
+            if (err) {
+                return callback(err);
+            }
+    
+            var branch = CACHED_BRANCHES[cacheKey];
+            if (branch) {
+                callback(null, Chain(branch));
+                return releaseLockFn();
+            }
+            
             var loadFn = function(finished) {
 
                 Chain(repository).trap(function(e) {
@@ -464,23 +474,21 @@ exports = module.exports = function()
 
                         if (err) {
 
+                            callback(err);
+                            
                             // release the lock
-                            releaseLockFn();
-
-                            // do the callback
-                            return callback(err);
+                            return releaseLockFn();
                         }
 
                         // success!
 
                         // store in cache
                         CACHED_BRANCHES[cacheKey] = branch;
-
+    
+                        callback(null, branch);
+                        
                         // release the lock
-                        releaseLockFn();
-
-                        // do the callback
-                        return callback(null, branch);
+                        return releaseLockFn();
                     });
                 }
 
@@ -1439,11 +1447,10 @@ exports = module.exports = function()
 
                     // the range requested (for streaming)
                     //var range = req.headers["range"];
-
+    
                     cloudcmsUtil.preview(contentStore, gitana, repositoryId, branchId, nodeId, nodePath, attachmentId, locale, previewId, size, mimetype, forceReload, function(err, filePath, cacheInfo, releaseLock) {
 
-                        if (err)
-                        {
+                        if (err) {
                             req.log("Error on preview node: " + err.message + ", err: " + JSON.stringify(err));
                         }
 
@@ -1475,8 +1482,7 @@ exports = module.exports = function()
                                 // UZI: file deleted by invalidate before this gets called
                                 contentStore.sendFile(res, filePath, cacheInfo, function(err) {
 
-                                    if (err)
-                                    {
+                                    if (err) {
                                         util.handleSendFileError(req, res, filePath, cacheInfo, req.log, err);
                                     }
 

package/middleware/driver-config/driver-config.js

@@ -1,10 +1,4 @@
-var path = require('path');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
-var Gitana = require("gitana");
-
-var fs = require("fs");
 
 /**
  * Retrieves local driver configuration for hosts from Cloud CMS.

package/middleware/form/form.js

@@ -120,7 +120,7 @@ exports = module.exports = function()
             }
 
             // post form to Cloud CMS using public method
-            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + url;
+            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + url;
 
             var headers = {};
             headers["Authorization"] = req.gitana.platform().getDriver().getHttpHeaders()["Authorization"];
@@ -183,8 +183,8 @@ exports = module.exports = function()
         req.branch(function(err, branch) {
 
             var url = branch.getUri() + "/alpaca/datasource";
-
-            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + url;
+    
+            var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + url;
 
             var headers = {};
             headers["Authorization"] = req.gitana.platform().getDriver().getHttpHeaders()["Authorization"];

package/middleware/modules/modules.js

@@ -55,14 +55,20 @@ exports = module.exports = function()
             });
         }
 
-        if (!moduleConfig.source)
+        var source = moduleConfig.source40;
+        if (!source)
+        {
+            source = moduleConfig.source;
+        }
+
+        if (!source)
         {
             return callback({
                 "message": "Missing module config source settings"
             });
         }
 
-        var sourceType = moduleConfig.source.type;
+        var sourceType = source.type;
         if (!sourceType)
         {
             return callback({
@@ -70,7 +76,7 @@ exports = module.exports = function()
             });
         }
 
-        var sourceUrl = moduleConfig.source.uri;
+        var sourceUrl = source.uri;
         if (!sourceUrl)
         {
             return callback({
@@ -78,12 +84,12 @@ exports = module.exports = function()
             });
         }
 
-        var sourcePath = moduleConfig.source.path;
+        var sourcePath = source.path;
         if (!sourcePath) {
             sourcePath = "/";
         }
 
-        var sourceBranch = moduleConfig.source.branch;
+        var sourceBranch = source.branch;
         if (!sourceBranch) {
             sourceBranch = "master";
         }
@@ -101,15 +107,18 @@ exports = module.exports = function()
 
                 var targetStore = stores["modules"];
                 var targetOffsetPath = moduleId;
-
+    
+                //logFn("From: " + sourceType + ", " + sourceUrl + ", " + sourcePath + ", " + sourceBranch);
+                //logFn("To: " + targetStore + ", path: " + targetOffsetPath);
+                
                 util.gitCheckout(host, sourceType, sourceUrl, sourcePath, sourceBranch, targetStore, targetOffsetPath, false, logFn, function (err) {
 
-                    //logFn("After util.gitCheckout: " + JSON.stringify(err));
+                    logFn("After util.gitCheckout: " + JSON.stringify(err));
 
                     // invalidate any caching within the stores layer
                     storeService.invalidate(host);
 
-                    //logFn("After store.invalidate");
+                    logFn("After store.invalidate");
     
                     // broadcast: module_after_deploy
                     process.broadcast.publish("module_after_deploy", {

package/middleware/perf/perf.js

@@ -194,8 +194,9 @@ exports = module.exports = function()
                             car = filename;
                         }
                         var regex1 = new RegExp("-[0-9a-f]{32}$"); // md5
-                        var regex2 = new RegExp("-[0-9]{13}$"); // timestamp
-                        if (regex1.test(car) || regex2.test(car))
+                        var regex2 = new RegExp("-[0-9]{13}$"); // timestamp?
+                        var regex3 = new RegExp("-[0-9]{10}$"); // epoch millis
+                        if (regex1.test(car) || regex2.test(car) || regex3.test(car))
                         {
                             var x = car.lastIndexOf("-");
 

package/middleware/proxy/proxy.js

@@ -1,16 +1,6 @@
 var path = require('path');
-// var fs = require('fs');
-// var http = require('http');
-// var https = require('https');
-
-// var httpProxy = require('http-proxy');
-
-// var oauth2 = require("../../util/oauth2")();
-
-// var async = require("async");
 
 var util = require("../../util/util");
-// var auth = require("../../util/auth");
 
 var proxyFactory = require("../../util/proxy-factory");
 
@@ -82,20 +72,17 @@ exports = module.exports = function()
         contentStore.existsFile(filePath, function(exists) {
 
             if (!exists) {
-                callback();
-                return;
+                return callback();
             }
 
             contentStore.fileStats(filePath, function(err, stats) {
 
                 if (err) {
-                    callback();
-                    return;
+                    return callback();
                 }
 
-                if (stats.size == 0) {
-                    callback();
-                    return;
+                if (stats.size === 0) {
+                    return callback();
                 }
 
                 var handleGoodStream = function()
@@ -117,8 +104,7 @@ exports = module.exports = function()
                 // check cacheInfo for expireTime
                 contentStore.readFile(filePath + ".cache", function(err, cacheInfoText) {
 
-                    if (err || !cacheInfoText)
-                    {
+                    if (err || !cacheInfoText) {
                         return handleBadStream();
                     }
 
@@ -215,7 +201,7 @@ exports = module.exports = function()
                 contentStore.writeFile(filePath + ".cache", JSON.stringify(cacheInfo), function() {
                     _end.call(res, data, encoding);
                 });
-``            };
+            };
 
             callback();
         });
@@ -237,7 +223,7 @@ exports = module.exports = function()
                 {
                     req.url = "/";
                 }
-
+                
                 // caching scenario
                 _handleCacheRead(req, function (err, readStream) {
 
@@ -257,6 +243,7 @@ exports = module.exports = function()
                                 "message": "Missing baseURL from request bound gitana-config"
                             });
                         }
+                        
                         proxyFactory.acquireProxyHandler(proxyTarget, null, function(err, proxyHandler) {
                             if (err) {
                                 return next(err);

package/middleware/stores/store.js

@@ -1,4 +1,3 @@
-var request = require('request');
 var path = require('path');
 
 var util = require("../../util/util");

package/middleware/stores/stores.js

@@ -46,7 +46,7 @@ exports = module.exports = function()
         var engineId = storeConfiguration[storeType];
 
         var engine = ENGINES[engineId];
-    
+        
         var engineType = process.configuration.storeEngines[engineId].type;
         var engineConfiguration = process.configuration.storeEngines[engineId].config;
 
@@ -176,7 +176,7 @@ exports = module.exports = function()
             if (!stores) {
                 return afterCheck();
             }
-
+    
             return process.cache.read("module-descriptors-" + host, function(err, moduleDescriptorsForHost) {
 
                 if (err) {
@@ -417,7 +417,7 @@ exports = module.exports = function()
                         var moduleStoreType = moduleDescriptors[i].store;
                         var modulePath = moduleDescriptors[i].path;
 
-                        //console.log("Config Store - Module Path: " + modulePath + ", type: " + moduleStoreType);
+                        console.log("Config Store - Module Path: " + modulePath + ", type: " + moduleStoreType);
 
                         var storePath = path.join(modulePath, "config");
                         if (moduleStoreType === "modules")
@@ -520,7 +520,7 @@ exports = module.exports = function()
                         callback();
                     });
                 };
-    
+
                 process.cache.read("module-descriptors-" + host, function(err, moduleDescriptors) {
 
                     moduleDescriptors = null;

package/middleware/virtual-config/virtual-config.js

@@ -1,6 +1,3 @@
-//var path = require('path');
-//var http = require('http');
-//var request = require('request');
 var util = require("../../util/util");
 
 /**
@@ -33,9 +30,12 @@ exports = module.exports = function()
             // force key to "virtualdriver"
             configuration.virtualDriver.key = VIRTUAL_DRIVER_CACHE_KEY;
 
+            //console.log("a1: " + JSON.stringify(configuration.virtualDriver, null, 2));
+            
             // either connect anew or re-use an existing connection to Cloud CMS for this application
             Gitana.connect(configuration.virtualDriver, function(err) {
-
+    
+                //console.log("a2:" + err);
                 if (err)
                 {
                     return callback(err);
@@ -83,7 +83,7 @@ exports = module.exports = function()
 
                 var URL = configuration.virtualDriver.baseURL;
                 if (!URL) {
-                    URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT);
+                    URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
                 }
                 URL += "/virtual/driver/config";
                 var requestConfig = {
@@ -143,10 +143,10 @@ exports = module.exports = function()
         }
     };
 
-    var _LOCK = function(lockKey, workFunction)
-    {
-        process.locks.lock(lockKey, workFunction);
-    };
+    // var _LOCK = function(lockKey, workFunction)
+    // {
+    //     process.locks.lock(lockKey, workFunction);
+    // };
 
     var r = {};
 
@@ -213,7 +213,7 @@ exports = module.exports = function()
                         }
                         if (!gitanaJson.baseURL)
                         {
-                            gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT));
+                            gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
                         }
 
                         // mark as retrieved from virtual driver
@@ -336,7 +336,7 @@ exports = module.exports = function()
             // defaults
             if (!configuration.baseURL)
             {
-                configuration.baseURL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT);
+                configuration.baseURL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
             }
             if (!configuration.key) {
                 configuration.key = "virtual";

package/middleware/virtual-files/virtual-files.js

@@ -1,6 +1,3 @@
-var path = require('path');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
 
 /**