cloudcms-server 3.2.316 → 3.2.318

package/locks/providers/memory.js

@@ -23,10 +23,10 @@ exports = module.exports = function(lockConfig)
         lock.acquire(key, function(releaseCallbackFn) {
             fn(null, releaseCallbackFn);
         }, function(err, ret) {
-            // lock was released
             if (err) {
                 console.error("Memory Lock heard error: ", err, " return value: ", ret);
             }
+            fn(err);
         });
     };
 

package/middleware/virtual-config/virtual-config.js

@@ -1,5 +1,7 @@
 var util = require("../../util/util");
 
+var workQueueFactory = require("../../util/workqueue");
+
 /**
  * Retrieves virtual driver configuration for hosts from Cloud CMS.
  *
@@ -7,7 +9,11 @@ var util = require("../../util/util");
  */
 exports = module.exports = function()
 {
+    // ensures that we only load 2 virtual config at a time
+    var enqueueLoadVirtualConfig = workQueueFactory(2);
+
     var SENTINEL_NOT_FOUND_VALUE = "null";
+    var BLACKLIST_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days
 
     var VIRTUAL_DRIVER_CACHE_KEY = "virtualdriver";
 
@@ -53,98 +59,96 @@ exports = module.exports = function()
     {
         var configuration = process.configuration;
 
-        if (configuration.virtualDriver && configuration.virtualDriver.enabled)
+        if (!configuration.virtualDriver || !configuration.virtualDriver.enabled)
         {
-            connectAsVirtualDriver(function(err, gitana) {
+            return callback();
+        }
 
-                if (err)
-                {
-                    return callback(err);
-                }
+        // no appkey, cannot load
+        if (!configuration.virtualDriver.appKey)
+        {
+            return callback();
+        }
 
-                // no appkey, cannot load
-                if (!configuration.virtualDriver.appKey)
-                {
-                    return callback();
-                }
+        connectAsVirtualDriver(function(err, gitana) {
 
-                // Basic Authentication request back to server
-                var qs = {};
-                qs.h = host;
-                qs.a = configuration.virtualDriver.appKey;
+            if (err)
+            {
+                return callback(err);
+            }
 
-                if (configuration.virtualDriver && configuration.virtualDriver.webhost)
-                {
-                    qs.w = configuration.virtualDriver.webhost;
-                }
+            // Basic Authentication request back to server
+            var qs = {};
+            qs.h = host;
+            qs.a = configuration.virtualDriver.appKey;
 
-                var URL = configuration.virtualDriver.baseURL;
-                if (!URL) {
-                    URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
-                }
-                URL += "/virtual/driver/config";
-                var requestConfig = {
-                    "url": URL,
-                    "qs": qs
-                };
+            if (configuration.virtualDriver && configuration.virtualDriver.webhost)
+            {
+                qs.w = configuration.virtualDriver.webhost;
+            }
 
-                util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, body) {
+            var URL = configuration.virtualDriver.baseURL;
+            if (!URL) {
+                URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
+            }
+            URL += "/virtual/driver/config";
+            var requestConfig = {
+                "url": URL,
+                "qs": qs
+            };
 
-                    if (response && response.status === 200 && body)
-                    {
-                        var config = body.config;
-                        if (!config)
-                        {
-                            // nothing found
-                            callback();
-                        }
-                        else
-                        {
-                            // make sure we update baseURL
-                            config.baseURL = configuration.virtualDriver.baseURL;
+            util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, body) {
 
-                            // hand back
-                            callback(null, config);
-                        }
+                if (response && response.status === 200 && body)
+                {
+                    var config = body.config;
+                    if (!config)
+                    {
+                        // nothing found
+                        callback();
                     }
                     else
                     {
-                        logMethod("Load virtual driver config failed");
-                        if (response && response.status)
-                        {
-                            logMethod("Response status code: " + response.status);
-                        }
-                        if (err) {
-                            logMethod("Err: " + JSON.stringify(err));
-                        }
-                        if (body) {
-                            logMethod("Body: " + body);
-                        }
-                        var message = null;
-                        if (body) {
-                            message = JSON.stringify(body);
-                        }
-                        if (!message) {
-                            message = "Unable to load virtual driver configuration";
-                        }
-
-                        // force disconnect of virtual driver so that it has to log in again
-                        // this prevents the attempt to use the refresh token
-                        disconnectVirtualDriver();
+                        // make sure we update baseURL
+                        config.baseURL = configuration.virtualDriver.baseURL;
 
-                        // fire callback
-                        callback({
-                            "message": message,
-                            "err": err
-                        });
+                        // hand back
+                        callback(null, config);
+                    }
+                }
+                else
+                {
+                    logMethod("Load virtual driver config failed");
+                    if (response && response.status)
+                    {
+                        logMethod("Response status code: " + response.status);
                     }
-                });
+                    if (err) {
+                        logMethod("Err: " + JSON.stringify(err));
+                    }
+                    if (body) {
+                        logMethod("Body: " + body);
+                    }
+                    var message = null;
+                    if (body) {
+                        message = JSON.stringify(body);
+                    }
+                    if (!message) {
+                        message = "Unable to load virtual driver configuration";
+                    }
+
+                    // force disconnect of virtual driver so that it has to log in again
+                    // this prevents the attempt to use the refresh token
+                    disconnectVirtualDriver();
+
+                    // fire callback
+                    callback({
+                        "message": message,
+                        "err": err
+                    });
+                }
             });
-        }
-        else
-        {
-            callback();
-        }
+        });
     };
 
     var r = {};
@@ -158,166 +162,171 @@ exports = module.exports = function()
 
         var VCSENTINEL_CACHE_KEY = "vcSentinelFailed-" + host;
 
-        rootStore.existsFile("gitana.json", function(exists) {
-
-            var loadFromRemote = function(finishedLoading) {
-
-                // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
-                process.cache.read(VCSENTINEL_CACHE_KEY, function (err, doesNotExist) {
-
-                    if (doesNotExist) {
-                        return finishedLoading({
-                            "message": "No virtual config found for host (from previous attempt)"
-                        }, null, true);
-                    }
+        // so that only N number of virtual configs are loaded at a time
+        enqueueLoadVirtualConfig(function(host, rootStore, logMethod, callback) {
 
-                    // load the gitana.json file from Cloud CMS
-                    loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig) {
+            rootStore.existsFile("gitana.json", function(exists) {
 
-                        if (err)
-                        {
-                            // something failed, perhaps a network issue
-                            // don't store anything
-                            return finishedLoading(err);
-                        }
+                var loadFromRemote = function(finishedLoading) {
 
-                        if (!virtualConfig)
-                        {
-                            // mark that it failed (30 minute TTL)
-                            return process.cache.write(VCSENTINEL_CACHE_KEY, true, 30 * 60, function() {
-                                finishedLoading({
-                                    "message": "No virtual config found for host: " + host
-                                });
-                            });
-                        }
+                    // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
+                    process.cache.read(VCSENTINEL_CACHE_KEY, function (err, doesNotExist) {
 
-                        // populate gitana.json
-                        var gitanaJson = {
-                            "clientKey": virtualConfig.clientKey
-                        };
-                        if (virtualConfig.clientSecret) {
-                            gitanaJson.clientSecret = virtualConfig.clientSecret;
+                        if (doesNotExist) {
+                            return finishedLoading({
+                                "message": "No virtual config found for host (from previous attempt)"
+                            }, null, true);
                         }
-                        if (virtualConfig.username) {
-                            gitanaJson.username = virtualConfig.username;
-                        }
-                        if (virtualConfig.password) {
-                            gitanaJson.password = virtualConfig.password;
-                        }
-                        if (virtualConfig.application) {
-                            gitanaJson.application = virtualConfig.application;
-                        }
-                        if (virtualConfig.baseURL) {
-                            gitanaJson.baseURL = virtualConfig.baseURL;
-                        }
-                        if (!gitanaJson.baseURL)
-                        {
-                            gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
-                        }
-
-                        // mark as retrieved from virtual driver
-                        gitanaJson._virtual = true;
 
-                        // write the gitana.json file
-                        rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
+                        // load the gitana.json file from Cloud CMS
+                        loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig) {
 
-                            // if we failed to write the file, then delete and call back with error
                             if (err)
                             {
-                                return rootStore.deleteFile("gitana.json", function() {
-                                    finishedLoading(err);
+                                // something failed, perhaps a network issue
+                                // don't store anything
+                                return finishedLoading(err);
+                            }
+
+                            if (!virtualConfig)
+                            {
+                                // mark that it failed (30 minute TTL)
+                                return process.cache.write(VCSENTINEL_CACHE_KEY, true, BLACKLIST_TTL_SECONDS, function() {
+                                    finishedLoading({
+                                        "message": "No virtual config found for host: " + host
+                                    });
                                 });
                             }
 
-                            // make sure the file wrote successfully
-                            // check stats, ensure non-error and file size > 0
-                            rootStore.fileStats("gitana.json", function(err, stats) {
+                            // populate gitana.json
+                            var gitanaJson = {
+                                "clientKey": virtualConfig.clientKey
+                            };
+                            if (virtualConfig.clientSecret) {
+                                gitanaJson.clientSecret = virtualConfig.clientSecret;
+                            }
+                            if (virtualConfig.username) {
+                                gitanaJson.username = virtualConfig.username;
+                            }
+                            if (virtualConfig.password) {
+                                gitanaJson.password = virtualConfig.password;
+                            }
+                            if (virtualConfig.application) {
+                                gitanaJson.application = virtualConfig.application;
+                            }
+                            if (virtualConfig.baseURL) {
+                                gitanaJson.baseURL = virtualConfig.baseURL;
+                            }
+                            if (!gitanaJson.baseURL)
+                            {
+                                gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
+                            }
+
+                            // mark as retrieved from virtual driver
+                            gitanaJson._virtual = true;
+
+                            // write the gitana.json file
+                            rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
 
-                                // if we failed to read stats, then delete and call back with error
-                                if (err || stats.size === 0)
+                                // if we failed to write the file, then delete and call back with error
+                                if (err)
                                 {
                                     return rootStore.deleteFile("gitana.json", function() {
-                                        finishedLoading({
-                                            "message": "There was a problem writing the driver configuration file.  Please reload."
-                                        });
+                                        finishedLoading(err);
                                     });
                                 }
 
-                                finishedLoading(null, gitanaJson);
+                                // make sure the file wrote successfully
+                                // check stats, ensure non-error and file size > 0
+                                rootStore.fileStats("gitana.json", function(err, stats) {
+
+                                    // if we failed to read stats, then delete and call back with error
+                                    if (err || stats.size === 0)
+                                    {
+                                        return rootStore.deleteFile("gitana.json", function() {
+                                            finishedLoading({
+                                                "message": "There was a problem writing the driver configuration file.  Please reload."
+                                            });
+                                        });
+                                    }
+
+                                    finishedLoading(null, gitanaJson);
+                                });
                             });
                         });
                     });
-                });
-            };
-
-            if (exists)
-            {
-                // read gitana json and send back
-                rootStore.readFile("gitana.json", function(err, data) {
-
-                    if (err)
-                    {
-                        return callback(err);
-                    }
-
-                    if (!data)
-                    {
-                        return callback({
-                            "message": "The gitana.json data read from disk was null or empty"
-                        })
-                    }
+                };
 
-                    // make sure not size 0
-                    rootStore.fileStats("gitana.json", function(err, stats) {
+                if (exists)
+                {
+                    // read gitana json and send back
+                    rootStore.readFile("gitana.json", function(err, data) {
 
                         if (err)
                         {
                             return callback(err);
                         }
 
-                        // if we failed to read stats or file size 0, then delete and call back with error
-                        if (err || stats.size === 0)
+                        if (!data)
                         {
-                            return rootStore.deleteFile("gitana.json", function() {
-                                callback({
-                                    "message": "There was a problem writing the driver configuration file.  Please reload."
-                                });
-                            });
+                            return callback({
+                                "message": "The gitana.json data read from disk was null or empty"
+                            })
                         }
 
-                        // remove vcSentinel if it exists
-                        process.cache.remove(VCSENTINEL_CACHE_KEY);
+                        // make sure not size 0
+                        rootStore.fileStats("gitana.json", function(err, stats) {
 
-                        var gitanaJson = JSON.parse("" + data);
+                            if (err)
+                            {
+                                return callback(err);
+                            }
 
-                        // auto-upgrade the host?
-                        if (gitanaJson.baseURL)
-                        {
-                            var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
-                            if (newBaseURL !== gitanaJson.baseURL)
+                            // if we failed to read stats or file size 0, then delete and call back with error
+                            if (err || stats.size === 0)
                             {
-                                console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
+                                return rootStore.deleteFile("gitana.json", function() {
+                                    callback({
+                                        "message": "There was a problem writing the driver configuration file.  Please reload."
+                                    });
+                                });
+                            }
 
-                                gitanaJson.baseURL = newBaseURL;
+                            // remove vcSentinel if it exists
+                            process.cache.remove(VCSENTINEL_CACHE_KEY);
 
-                                // write the gitana.json file
-                                rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
-                                    // nada
-                                });
+                            var gitanaJson = JSON.parse("" + data);
+
+                            // auto-upgrade the host?
+                            if (gitanaJson.baseURL)
+                            {
+                                var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
+                                if (newBaseURL !== gitanaJson.baseURL)
+                                {
+                                    console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
+
+                                    gitanaJson.baseURL = newBaseURL;
+
+                                    // write the gitana.json file
+                                    rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
+                                        // nada
+                                    });
+                                }
                             }
-                        }
 
-                        // otherwise, fine!
-                        callback(null, gitanaJson);
+                            // otherwise, fine!
+                            callback(null, gitanaJson);
+                        });
                     });
-                });
-            }
-            else
-            {
-                loadFromRemote(function(err, gitanaJson, doesNotExist) {
-                    callback(err, gitanaJson, doesNotExist);
-                });
-            }
+                }
+                else
+                {
+                    loadFromRemote(function(err, gitanaJson, doesNotExist) {
+                        callback(err, gitanaJson, doesNotExist);
+                    });
+                }
+            });
+
         });
     };
 
@@ -345,6 +354,16 @@ exports = module.exports = function()
             {
                 if (doesNotExist)
                 {
+                    // console.log("BLOCK, method: " + req.method + ", url: " + req.url);
+                    // if (req.headers)
+                    // {
+                    //     console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
+                    // }
+                    // if (req.query)
+                    // {
+                    //     console.log(" -> query: " + JSON.stringify(req.query, null, 2));
+                    // }
+                    //
                     // are we being spoofed? kill the connection
                     res.blocked = true;
                     res.writeHead(503, { 'Content-Type': 'application/json' });
@@ -413,7 +432,7 @@ exports = module.exports = function()
 
                         // mark with sentinel (30 minutes)
                         req.log("[BLACKLIST] Adding: " + req.virtualHost);
-                        process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, 30 * 60, function (err) {
+                        process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, BLACKLIST_TTL_SECONDS, function (err) {
                             completionFunction(null, null, true);
                         });
                     });

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "cloudcms-server",
   "description": "Cloud CMS Application Server Module",
-  "version": "3.2.316",
+  "version": "3.2.318",
   "repository": {
     "type": "git",
     "url": "git://github.com/gitana/cloudcms-server.git"

package/server/index.js

@@ -992,6 +992,36 @@ var startServer = function(config, startServerFinishedFn)
     
                         next();
                     });
+
+                    // kills immediately based on path, headers or other detections
+                    app.use(function(req, res, next) {
+
+                        var kill = false;
+                        if (req.path.endsWith("/env"))
+                        {
+                            kill = true;
+                        }
+
+                        if (kill)
+                        {
+                            // console.log("KILL, method: " + req.method + ", url: " + req.url);
+                            // if (req.headers)
+                            // {
+                            //     console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
+                            // }
+                            // if (req.query)
+                            // {
+                            //     console.log(" -> query: " + JSON.stringify(req.query, null, 2));
+                            // }
+                            //
+                            // are we being spoofed? kill the connection
+                            res.blocked = true;
+                            res.writeHead(503, { 'Content-Type': 'application/json' });
+                            return res.end(JSON.stringify({"error": true, "message": "Bad Request."}));
+                        }
+
+                        next();
+                    });
     
                     // common interceptors and config
                     main.common1(app);

package/temp/clusterlock/index.js

@@ -35,8 +35,12 @@ var _setup = function() {
                 };
 
                 _sendMessageToWorker(message);
+            }, function(err, ret) {
+                // lock was released
+                if (err) {
+                    console.error("Cluster Lock heard error: ", err, " return value: ", ret);
+                }
             });
-
         };
 
         var _release = function(message)

package/util/loaders.js

@@ -94,6 +94,11 @@ var exclusive = exports.exclusive = function(loader, key, timeout)
                 }, 0);
                 callback.call(this, err, value);
             });
+        }, function(err, ret) {
+            if (err) {
+                console.error("Loaders async lock acquire heard error: ", err, " return value: ", ret);
+            }
+            callback.call(this, err, ret);
         }, opts);
     };
 };

package/util/workqueue.js

@@ -0,0 +1,80 @@
+var exports = module.exports;
+
+exports = function(maxSize)
+{
+    if (!maxSize) {
+        maxSize = 3;
+    }
+
+    var blockExecution = false;
+
+    var pendingWorkFns = [];
+    var activeCount = 0;
+
+    var processWork = function () {
+
+        // if another "thread" is running the processor, don't bother
+        if (blockExecution)
+        {
+            return;
+        }
+
+        blockExecution = true;
+
+        // add as many pending work items as we can, loop until full or no more pending
+        var process = true;
+        do
+        {
+            // if nothing to work on, bail
+            if (pendingWorkFns.length === 0)
+            {
+                process = false;
+            }
+
+            // if we're full, bail
+            if (activeCount >= maxSize)
+            {
+                process = false;
+            }
+
+            if (process)
+            {
+                // increment active count
+                activeCount++;
+
+                //console.log("Active work items: " + activeCount);
+
+                // define execution function and splice/bind to 0th element from pending list
+                var executionFn = function(workFn) {
+                    return function() {
+                        workFn(function () {
+
+                            // decrement active count
+                            activeCount--;
+
+                            //console.log("Active work items: " + activeCount);
+
+                            // process more work on timeout
+                            window.setTimeout(processWork);
+                        });
+
+                    };
+                }(pendingWorkFns.splice(0, 1)[0]);
+
+                // execute on timeout
+                window.setTimeout(executionFn);
+            }
+
+        } while (process);
+
+        blockExecution = false;
+    };
+
+    return function(workFn) {
+        pendingWorkFns.push(workFn);
+
+        // execute on timeout
+        window.setTimeout(processWork);
+    };
+
+};