cloudcms-server 4.0.0-beta.11 → 4.0.0-beta.14

package/middleware/virtual-config/virtual-config.js

@@ -1,5 +1,9 @@
 var util = require("../../util/util");
 
+var workQueueFactory = require("../../util/workqueue");
+
+//var debugLog = process.debugLog;
+
 /**
  * Retrieves virtual driver configuration for hosts from Cloud CMS.
  *
@@ -7,7 +11,12 @@ var util = require("../../util/util");
  */
 exports = module.exports = function()
 {
+    // ensures that we only load 2 virtual config at a time
+    var enqueueLoadVirtualConfig = workQueueFactory(2);
+
     var SENTINEL_NOT_FOUND_VALUE = "null";
+    var BLACKLIST_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days
+    var DISABLED_TTL_SECONDS = 60 * 10; // 10 minutes
 
     var VIRTUAL_DRIVER_CACHE_KEY = "virtualdriver";
 
@@ -30,12 +39,9 @@ exports = module.exports = function()
             // force key to "virtualdriver"
             configuration.virtualDriver.key = VIRTUAL_DRIVER_CACHE_KEY;
 
-            //console.log("a1: " + JSON.stringify(configuration.virtualDriver, null, 2));
-            
             // either connect anew or re-use an existing connection to Cloud CMS for this application
             Gitana.connect(configuration.virtualDriver, function(err) {
-    
-                //console.log("a2:" + err);
+
                 if (err)
                 {
                     return callback(err);
@@ -56,59 +62,66 @@ exports = module.exports = function()
     {
         var configuration = process.configuration;
 
-        if (configuration.virtualDriver && configuration.virtualDriver.enabled)
+        if (!configuration.virtualDriver || !configuration.virtualDriver.enabled)
         {
-            connectAsVirtualDriver(function(err, gitana) {
+            return callback();
+        }
 
-                if (err)
-                {
-                    return callback(err);
-                }
+        // no appkey, cannot load
+        if (!configuration.virtualDriver.appKey)
+        {
+            return callback();
+        }
 
-                // no appkey, cannot load
-                if (!configuration.virtualDriver.appKey)
-                {
-                    return callback();
-                }
+        connectAsVirtualDriver(function(err, gitana) {
 
-                // Basic Authentication request back to server
-                var qs = {};
-                qs.h = host;
-                qs.a = configuration.virtualDriver.appKey;
+            if (err)
+            {
+                return callback(err);
+            }
 
-                if (configuration.virtualDriver && configuration.virtualDriver.webhost)
-                {
-                    qs.w = configuration.virtualDriver.webhost;
-                }
+            // Basic Authentication request back to server
+            var qs = {};
+            qs.h = host;
+            qs.a = configuration.virtualDriver.appKey;
 
-                var URL = configuration.virtualDriver.baseURL;
-                if (!URL) {
-                    URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
-                }
-                URL += "/virtual/driver/config";
-                var requestConfig = {
-                    "url": URL,
-                    "qs": qs
-                };
+            if (configuration.virtualDriver && configuration.virtualDriver.webhost)
+            {
+                qs.w = configuration.virtualDriver.webhost;
+            }
 
-                util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, json) {
+            var URL = configuration.virtualDriver.baseURL;
+            if (!URL) {
+                URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
+            }
+            URL += "/virtual/driver/config";
+            var requestConfig = {
+                "url": URL,
+                "qs": qs
+            };
 
-                    if (response && response.status === 200 && json)
-                    {
-                        var config = json.config;
-                        if (!config)
-                        {
-                            // nothing found
-                            return callback();
-                        }
+            util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, body) {
 
+                if (response && response.status === 200 && body)
+                {
+                    var config = body.config;
+                    if (!config)
+                    {
+                        // nothing found
+                        callback();
+                    }
+                    else
+                    {
                         // make sure we update baseURL
                         config.baseURL = configuration.virtualDriver.baseURL;
 
                         // hand back
-                        return callback(null, config);
+                        var disabled = body.disabled ? true : false;
+                        callback(null, config, disabled);
                     }
-
+                }
+                else
+                {
                     logMethod("Load virtual driver config failed");
                     if (response && response.status)
                     {
@@ -117,10 +130,13 @@ exports = module.exports = function()
                     if (err) {
                         logMethod("Err: " + JSON.stringify(err));
                     }
-                    if (json) {
-                        logMethod("Body: " + json);
+                    if (body) {
+                        logMethod("Body: " + body);
+                    }
+                    var message = null;
+                    if (body) {
+                        message = JSON.stringify(body);
                     }
-                    var message = json;
                     if (!message) {
                         message = "Unable to load virtual driver configuration";
                     }
@@ -130,24 +146,15 @@ exports = module.exports = function()
                     disconnectVirtualDriver();
 
                     // fire callback
-                    return callback({
+                    callback({
                         "message": message,
                         "err": err
                     });
-                });
+                }
             });
-        }
-        else
-        {
-            callback();
-        }
+        });
     };
 
-    // var _LOCK = function(lockKey, workFunction)
-    // {
-    //     process.locks.lock(lockKey, workFunction);
-    // };
-
     var r = {};
 
     /**
@@ -159,166 +166,188 @@ exports = module.exports = function()
 
         var VCSENTINEL_CACHE_KEY = "vcSentinelFailed-" + host;
 
-        rootStore.existsFile("gitana.json", function(exists) {
+        // so that only N number of virtual configs are loaded at a time
+        var workFn = function(host, rootStore, logMethod) {
 
-            var loadFromRemote = function(finishedLoading) {
+            return function(done)
+            {
+                rootStore.existsFile("gitana.json", function(exists) {
 
-                // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
-                process.cache.read(VCSENTINEL_CACHE_KEY, function (err, failedRecently) {
+                    var loadFromRemote = function(finishedLoading) {
 
-                    if (failedRecently) {
-                        return finishedLoading({
-                            "message": "No virtual config found for host (from previous attempt)"
-                        });
-                    }
+                        // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
+                        process.cache.read(VCSENTINEL_CACHE_KEY, function (err, doesNotExist) {
 
-                    // load the gitana.json file from Cloud CMS
-                    loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig) {
+                            if (doesNotExist) {
+                                return finishedLoading({
+                                    "message": "No virtual config found for host (from previous attempt)"
+                                }, null, true);
+                            }
 
-                        if (err)
-                        {
-                            // something failed, perhaps a network issue
-                            // don't store anything
-                            return finishedLoading(err);
-                        }
+                            // load the gitana.json file from Cloud CMS
+                            loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig, disabled) {
 
-                        if (!virtualConfig)
-                        {
-                            // mark that it failed (5 seconds TTL)
-                            return process.cache.write(VCSENTINEL_CACHE_KEY, SENTINEL_NOT_FOUND_VALUE, 5, function() {
-                                finishedLoading({
-                                    "message": "No virtual config found for host: " + host
-                                });
-                            });
-                        }
+                                if (err)
+                                {
+                                    // something failed, perhaps a network issue
+                                    // don't store anything
+                                    return finishedLoading(err);
+                                }
 
-                        // populate gitana.json
-                        var gitanaJson = {
-                            "clientKey": virtualConfig.clientKey
-                        };
-                        if (virtualConfig.clientSecret) {
-                            gitanaJson.clientSecret = virtualConfig.clientSecret;
-                        }
-                        if (virtualConfig.username) {
-                            gitanaJson.username = virtualConfig.username;
-                        }
-                        if (virtualConfig.password) {
-                            gitanaJson.password = virtualConfig.password;
-                        }
-                        if (virtualConfig.application) {
-                            gitanaJson.application = virtualConfig.application;
-                        }
-                        if (virtualConfig.baseURL) {
-                            gitanaJson.baseURL = virtualConfig.baseURL;
-                        }
-                        if (!gitanaJson.baseURL)
-                        {
-                            gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
-                        }
+                                if (!virtualConfig)
+                                {
+                                    // mark that it failed (30 day blacklist TTL)
+                                    return process.cache.write(VCSENTINEL_CACHE_KEY, true, BLACKLIST_TTL_SECONDS, function() {
+                                        finishedLoading({
+                                            "message": "No virtual config found for host: " + host
+                                        });
+                                    });
+                                }
 
-                        // mark as retrieved from virtual driver
-                        gitanaJson._virtual = true;
+                                if (disabled)
+                                {
+                                    // mark that the virtual config is disabled (10 minutes TLL)
+                                    return process.cache.write(VCSENTINEL_CACHE_KEY, true, DISABLED_TTL_SECONDS, function() {
+                                        finishedLoading({
+                                            "message": "The virtual config was found for host: " + host + " but it has been marked as disabled"
+                                        });
+                                    });
+                                }
 
-                        // write the gitana.json file
-                        rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
+                                // populate gitana.json
+                                var gitanaJson = {
+                                    "clientKey": virtualConfig.clientKey
+                                };
+                                if (virtualConfig.clientSecret) {
+                                    gitanaJson.clientSecret = virtualConfig.clientSecret;
+                                }
+                                if (virtualConfig.username) {
+                                    gitanaJson.username = virtualConfig.username;
+                                }
+                                if (virtualConfig.password) {
+                                    gitanaJson.password = virtualConfig.password;
+                                }
+                                if (virtualConfig.application) {
+                                    gitanaJson.application = virtualConfig.application;
+                                }
+                                if (virtualConfig.baseURL) {
+                                    gitanaJson.baseURL = virtualConfig.baseURL;
+                                }
+                                if (!gitanaJson.baseURL)
+                                {
+                                    gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
+                                }
+
+                                // mark as retrieved from virtual driver
+                                gitanaJson._virtual = true;
+
+                                // write the gitana.json file
+                                rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
+
+                                    // if we failed to write the file, then delete and call back with error
+                                    if (err)
+                                    {
+                                        return rootStore.deleteFile("gitana.json", function() {
+                                            finishedLoading(err);
+                                        });
+                                    }
+
+                                    // make sure the file wrote successfully
+                                    // check stats, ensure non-error and file size > 0
+                                    rootStore.fileStats("gitana.json", function(err, stats) {
+
+                                        // if we failed to read stats, then delete and call back with error
+                                        if (err || stats.size === 0)
+                                        {
+                                            return rootStore.deleteFile("gitana.json", function() {
+                                                finishedLoading({
+                                                    "message": "There was a problem writing the driver configuration file.  Please reload."
+                                                });
+                                            });
+                                        }
+
+                                        finishedLoading(null, gitanaJson);
+                                    });
+                                });
+                            });
+                        });
+                    };
+
+                    if (exists)
+                    {
+                        // read gitana json and send back
+                        rootStore.readFile("gitana.json", function(err, data) {
 
-                            // if we failed to write the file, then delete and call back with error
                             if (err)
                             {
-                                return rootStore.deleteFile("gitana.json", function() {
-                                    finishedLoading(err);
-                                });
+                                return done(err);
+                            }
+
+                            if (!data)
+                            {
+                                return done({
+                                    "message": "The gitana.json data read from disk was null or empty"
+                                })
                             }
 
-                            // make sure the file wrote successfully
-                            // check stats, ensure non-error and file size > 0
+                            // make sure not size 0
                             rootStore.fileStats("gitana.json", function(err, stats) {
 
-                                // if we failed to read stats, then delete and call back with error
+                                if (err)
+                                {
+                                    return done(err);
+                                }
+
+                                // if we failed to read stats or file size 0, then delete and call back with error
                                 if (err || stats.size === 0)
                                 {
                                     return rootStore.deleteFile("gitana.json", function() {
-                                        finishedLoading({
+                                        done({
                                             "message": "There was a problem writing the driver configuration file.  Please reload."
                                         });
                                     });
                                 }
 
-                                finishedLoading(null, gitanaJson);
-                            });
-                        });
-                    });
-                });
-            };
+                                // remove vcSentinel if it exists
+                                process.cache.remove(VCSENTINEL_CACHE_KEY);
 
-            if (exists)
-            {
-                // read gitana json and send back
-                rootStore.readFile("gitana.json", function(err, data) {
+                                var gitanaJson = JSON.parse("" + data);
 
-                    if (err)
-                    {
-                        return callback(err);
-                    }
-
-                    if (!data)
-                    {
-                        return callback({
-                            "message": "The gitana.json data read from disk was null or empty"
-                        })
-                    }
+                                // auto-upgrade the host?
+                                if (gitanaJson.baseURL)
+                                {
+                                    var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
+                                    if (newBaseURL !== gitanaJson.baseURL)
+                                    {
+                                        console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
 
-                    // make sure not size 0
-                    rootStore.fileStats("gitana.json", function(err, stats) {
+                                        gitanaJson.baseURL = newBaseURL;
 
-                        if (err)
-                        {
-                            return callback(err);
-                        }
+                                        // write the gitana.json file
+                                        rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
+                                            // nada
+                                        });
+                                    }
+                                }
 
-                        // if we failed to read stats or file size 0, then delete and call back with error
-                        if (err || stats.size === 0)
-                        {
-                            return rootStore.deleteFile("gitana.json", function() {
-                                callback({
-                                    "message": "There was a problem writing the driver configuration file.  Please reload."
-                                });
+                                // otherwise, fine!
+                                done(null, gitanaJson);
                             });
-                        }
-
-                        // remove vcSentinel if it exists
-                        process.cache.remove(VCSENTINEL_CACHE_KEY);
-
-                        var gitanaJson = JSON.parse("" + data);
-
-                        // auto-upgrade the host?
-                        if (gitanaJson.baseURL)
-                        {
-                            var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
-                            if (newBaseURL !== gitanaJson.baseURL)
-                            {
-                                console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
-
-                                gitanaJson.baseURL = newBaseURL;
-
-                                // write the gitana.json file
-                                rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, "   "), function (err) {
-                                    // nada
-                                });
-                            }
-                        }
-
-                        // otherwise, fine!
-                        callback(null, gitanaJson);
-                    });
-                });
-            }
-            else
-            {
-                loadFromRemote(function(err, gitanaJson) {
-                    callback(err, gitanaJson);
+                        });
+                    }
+                    else
+                    {
+                        loadFromRemote(function(err, gitanaJson, doesNotExist) {
+                            done(err, gitanaJson, doesNotExist);
+                        });
+                    }
                 });
             }
+
+        }(host, rootStore, logMethod);
+
+        enqueueLoadVirtualConfig(workFn, function(err, gitanaJson, doesNotExist) {
+            callback(err, gitanaJson, doesNotExist);
         });
     };
 
@@ -342,15 +371,38 @@ exports = module.exports = function()
                 configuration.key = "virtual";
             }
 
-            var completionFunction = function (err, gitanaConfig) {
-                if (err) {
-                    if (err.message) {
+            var completionFunction = function (err, gitanaConfig, doesNotExist)
+            {
+                if (doesNotExist)
+                {
+                    // console.log("BLOCK, method: " + req.method + ", url: " + req.url);
+                    // if (req.headers)
+                    // {
+                    //     console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
+                    // }
+                    // if (req.query)
+                    // {
+                    //     console.log(" -> query: " + JSON.stringify(req.query, null, 2));
+                    // }
+                    //
+                    // are we being spoofed? kill the connection
+                    res.blocked = true;
+                    res.writeHead(503, { 'Content-Type': 'application/json' });
+                    return res.end(JSON.stringify({"error": true, "message": "Bad Request."}));
+                }
+
+                if (err)
+                {
+                    if (err.message)
+                    {
                         req.log(err.message);
                     }
+
                     return next();
                 }
 
-                if (gitanaConfig) {
+                if (gitanaConfig)
+                {
                     // store config
                     req.gitanaConfig = gitanaConfig;
 
@@ -372,7 +424,7 @@ exports = module.exports = function()
                     if (cachedValue === SENTINEL_NOT_FOUND_VALUE)
                     {
                         // null means there verifiably isn't anything on disk (null used as sentinel marker)
-                        completionFunction();
+                        completionFunction(null, null, true);
                     }
                     else
                     {
@@ -383,29 +435,27 @@ exports = module.exports = function()
                 else
                 {
                     // try to load from disk
-                    acquireGitanaJson(req.virtualHost, req.rootStore, req.log, function (err, gitanaConfig)
+                    acquireGitanaJson(req.virtualHost, req.rootStore, req.log, function (err, gitanaConfig, doesNotExist)
                     {
-                        if (err)
+                        if (err && !doesNotExist)
                         {
                             return completionFunction(err);
                         }
 
                         if (gitanaConfig)
                         {
-                            process.driverConfigCache.write(req.virtualHost, {
+                            return process.driverConfigCache.write(req.virtualHost, {
                                 "config": gitanaConfig
                             }, function (err) {
-                                completionFunction(null, gitanaConfig);
-                            });
-                        }
-                        else
-                        {
-                            // mark with sentinel
-                            process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, 5, function (err)
-                            {
-                                completionFunction();
+                                completionFunction(err, gitanaConfig);
                             });
                         }
+
+                        // mark with sentinel (30 minutes)
+                        req.log("[BLACKLIST] Adding: " + req.virtualHost);
+                        process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, BLACKLIST_TTL_SECONDS, function (err) {
+                            completionFunction(null, null, true);
+                        });
                     });
                 }
             });