cloudcms-server 4.0.0-beta.15 → 4.0.0-beta.17

package/middleware/awareness/plugins/editorial.js

@@ -21,6 +21,10 @@ exports.bindSocket = function(socket, provider, io)
             commitEditorialWorkspace(socket, provider, sessionKey, repositoryId, branchId, callback);
         });
 
+        socket.on("touchEditorialWorkspace", function(sessionKey, repositoryId, branchId, callback) {
+            touchEditorialWorkspace(socket, provider, sessionKey, repositoryId, branchId, callback);
+        });
+
         socket.on("editorialWorkspaceInfo", function(sessionKey, repositoryId, branchId, callback) {
             editorialWorkspaceInfo(socket, provider, sessionKey, repositoryId, branchId, callback);
         });
@@ -219,6 +223,52 @@ exports.bindSocket = function(socket, provider, io)
         });
     };
 
+    /**
+     * Touches an editorial workspace.
+     *
+     * This is a "keep alive" call to prevent a workspace from being cleaned up while it is being
+     * worked on.
+     *
+     * @param socket
+     * @param provider
+     * @param sessionKey
+     * @param repositoryId
+     * @param branchId
+     * @param callback
+     */
+    var touchEditorialWorkspace = function(socket, provider, sessionKey, repositoryId, branchId, callback)
+    {
+        var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + "/oneteam/editorial/workspace/touch";
+
+        var json = {};
+        json.repositoryId = repositoryId;
+        json.branchId = branchId;
+        json.key = sessionKey;
+
+        var headers = {};
+        var gitanaTicket = extractTicket(socket);
+        if (gitanaTicket)
+        {
+            headers["GITANA_TICKET"] = gitanaTicket;
+        }
+
+        request({
+            "method": "POST",
+            "url": URL,
+            "qs": {},
+            "json": json,
+            "headers": headers,
+            "timeout": process.defaultHttpTimeoutMs
+        }, function(err, response, json) {
+
+            if (err || (json && json.error)) {
+                return callback(err);
+            }
+
+            callback(null, json);
+        });
+    };
+
     var extractTicket = function(socket)
     {
         var ticket = null;

package/middleware/virtual-config/virtual-config.js

@@ -12,7 +12,7 @@ var workQueueFactory = require("../../util/workqueue");
 exports = module.exports = function()
 {
     // ensures that we only load 2 virtual config at a time
-    var enqueueLoadVirtualConfig = workQueueFactory(2);
+    var enqueueLoadVirtualConfig = workQueueFactory("loadVirtualConfigQueue", 2);
 
     var SENTINEL_NOT_FOUND_VALUE = "null";
     var BLACKLIST_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "cloudcms-server",
   "description": "Cloud CMS Application Server Module",
-  "version": "4.0.0-beta.15",
+  "version": "4.0.0-beta.17",
   "repository": {
     "type": "git",
     "url": "git://github.com/gitana/cloudcms-server.git"

package/server/index.js

@@ -586,12 +586,55 @@ var _start = function(overrides, callback) {
     if (!process.env.CLOUDCMS_STANDALONE_HOST) {
         process.env.CLOUDCMS_STANDALONE_HOST = "local";
     }
-    
-    
+
+    // http timeout
+    // default http timeout (2 minutes)
+    process.defaultHttpTimeoutMs = 2 * 60 * 1000;
+    if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
+    {
+        try {
+            process.defaultHttpTimeoutMs = parseInt(process.env.DEFAULT_HTTP_TIMEOUT_MS);
+        } catch (e) { }
+    }
+    else if (process.configuration.timeout)
+    {
+        process.defaultHttpTimeoutMs = process.configuration.timeout;
+    }
+
+    // socket keep alive (3 minutes)
+    process.defaultKeepAliveMs = (3 * 60 * 1000);
+    if (process.env.DEFAULT_KEEP_ALIVE_MS)
+    {
+        try {
+            process.defaultKeepAliveMs = parseInt(process.env.DEFAULT_KEEP_ALIVE_MS);
+        } catch (e) { }
+    }
+    else if (process.configuration.keepAliveMs)
+    {
+        process.defaultKeepAliveMs = process.configuration.keepAliveMs;
+    }
+
+
+    ///////////////////////
+    // auto-configuration for HTTP
+    ///////////////////////
+
+    if (!process.configuration.http) {
+        process.configuration.http = {};
+    }
+    process.configuration.http.keepAliveTimeout = process.defaultKeepAliveMs;
+    process.configuration.http.requestTimeout = process.defaultKeepAliveMs;
+
+    ///////////////////////
     // auto-configuration for HTTPS
+    ///////////////////////
+
     if (!process.configuration.https) {
         process.configuration.https = {};
     }
+    process.configuration.https.keepAliveTimeout = process.defaultKeepAliveMs;
+    process.configuration.https.requestTimeout = process.defaultKeepAliveMs;
+
     if (process.env.CLOUDCMS_HTTPS) {
         process.configuration.https = JSON.parse(process.env.CLOUDCMS_HTTPS);
     }
@@ -614,12 +657,6 @@ var _start = function(overrides, callback) {
         process.configuration.https.ca = [ fs.readFileSync(process.env.CLOUDCMS_HTTPS_CA_FILEPATH) ];
     }
     
-    // if https config is empty, remove it
-    if (Object.keys(process.configuration.https).length === 0) {
-        delete process.configuration.https;
-    }
-    
-    
     // auto configuration of session store
     if (!process.configuration.session) {
         process.configuration.session = {};
@@ -1214,20 +1251,17 @@ var createHttpServer = function(app, done)
     else
     {
         // legacy
-        httpServer = http.Server(app);
+        //httpServer = http.Server(app);
+        httpServer = http.createServer(process.configuration.http, app);
     }
     
-    // request timeout
-    var requestTimeout = 120000; // 2 minutes
-    if (process.configuration && process.configuration.timeout)
-    {
-        requestTimeout = process.configuration.timeout;
-    }
-    httpServer.setTimeout(requestTimeout, function(socket) {
+    // socket timeout
+    httpServer.setTimeout(process.defaultHttpTimeoutMs, function(socket) {
         try { socket.end(); } catch (e) { }
         try { socket.destroy(); } catch (e) { }
     });
 
+
     var c = 0;
 
     // socket

package/util/cloudcms.js

@@ -11,8 +11,8 @@ var workQueueFactory = require("./workqueue");
 
 exports = module.exports = function()
 {
-    // ensures that we only load 3 preview icons at a time
-    var previewQueueFn = workQueueFactory(3);
+    // ensures that we only load 5 preview icons at a time
+    var previewQueueFn = workQueueFactory("previewQueue", 5, false);
 
     var toCacheFilePath = function(filePath)
     {
@@ -356,7 +356,8 @@ exports = module.exports = function()
                 });
             }
             
-            var failFast = function(contentStore, filePath, cb) {
+            var failFast = function(contentStore, filePath, cb)
+            {
                 var triggered = false;
                 
                 return function(tempStream, err)
@@ -383,7 +384,7 @@ exports = module.exports = function()
             contentStore.writeStream(filePath, function(err, tempStream) {
 
                 if (err) {
-                    return failFast(tempStream, err);
+                    return failFast(tempStream, err, cb);
                 }
 
                 var cacheFilePath = toCacheFilePath(filePath);
@@ -406,9 +407,10 @@ exports = module.exports = function()
                     "method": "GET",
                     "url": URL,
                     "qs": {},
+                    "timeout": 3000,
                     "headers": headers,
                     "responseType": "stream"
-                }, function(err, response) {
+                }, function(err, response, stream) {
 
                     if (err) {
                         closeWriteStream(tempStream);
@@ -417,34 +419,34 @@ exports = module.exports = function()
     
                     if (response.status >= 200 && response.status <= 204)
                     {
-                        response.data.pipe(tempStream).on("close", function (err) {
-            
+                        var handle = function (err) {
+
                             if (err) {
                                 // some went wrong at disk io level?
-                                return failFast(tempStream, err);
+                                return failFast(tempStream, err, cb);
                             }
-            
+
                             contentStore.existsFile(filePath, function (exists) {
-                
+
                                 if (exists) {
-                    
+
                                     // write cache file
                                     var cacheInfo = buildCacheInfo(response);
                                     if (!cacheInfo) {
                                         return cb(null, filePath, null);
                                     }
-                    
+
                                     contentStore.writeFile(cacheFilePath, JSON.stringify(cacheInfo, null, "    "), function (err) {
-                        
+
                                         if (err) {
                                             // failed to write cache file, thus the whole thing is invalid
                                             return safeRemove(contentStore, cacheFilePath, function () {
                                                 failFast(tempStream, {
                                                     "message": "Failed to write cache file: " + cacheFilePath + ", err: " + JSON.stringify(err)
-                                                });
+                                                }, cb);
                                             });
                                         }
-                        
+
                                         cb(null, filePath, cacheInfo);
                                     });
                                 } else {
@@ -453,13 +455,27 @@ exports = module.exports = function()
                                     safeRemove(contentStore, cacheFilePath, function () {
                                         failFast(tempStream, {
                                             "message": "Failed to verify written cached file: " + filePath
-                                        });
+                                        }, cb);
                                     });
                                 }
                             });
-            
+                        };
+
+                        // stream.on("end", function() {
+                        //     console.log("g1");
+                        // });
+                        // stream.on("finish", function() {
+                        //     console.log("g2");
+                        // });
+                        stream.on("close", handle);
+
+                        stream.pipe(tempStream).on("close", handle).on("end", function() {
+                            //console.log("a4");
+                        }).on("finish", function() {
+                            //console.log("a5");
                         }).on("error", function (err) {
-                            failFast(tempStream, err);
+                            console.log("a3");
+                            failFast(tempStream, err, cb);
                         });
                     }
                     else
@@ -467,13 +483,13 @@ exports = module.exports = function()
                         // some kind of http error (usually permission denied or invalid_token)
         
                         var body = "";
-        
-                        response.data.on('data', function (chunk) {
+
+                        stream.on('data', function (chunk) {
                             body += chunk;
                         });
-        
-                        response.data.on('end', function () {
-            
+
+                        stream.on('end', function () {
+
                             var afterCleanup = function () {
                 
                                 // see if it is "invalid_token"
@@ -515,11 +531,10 @@ exports = module.exports = function()
                                 });
                             });
                         });
-        
                     }
                 });
                 // }).on('error', function (e) {
-                //     failFast(tempStream, e);
+                //     failFast(tempStream, e, cb);
                 //
                 // }).on('end', function (e) {
                 //
@@ -532,7 +547,7 @@ exports = module.exports = function()
                     process.log("Temp stream errored out");
                     process.log(e);
     
-                    failFast(tempStream, e);
+                    failFast(tempStream, e, cb);
 
                     // ensure stream is closed
                     //closeWriteStream(tempStream);

package/util/request.js

@@ -103,7 +103,6 @@ module.exports = function(config, callback)
         requestConfig.responseType = config.responseType;
     }
 
-
     /*
     if (requestConfig.url.toLowerCase().indexOf("https:") > -1)
     {
@@ -114,6 +113,10 @@ module.exports = function(config, callback)
         requestConfig.httpAgent = http.globalAgent;
     }
     */
+
+    if (config.timeout) {
+        requestConfig.timeout = config.timeout;
+    }
     
     return axios.request(requestConfig).then(function(response) {
         callback(null, response, response.data);

package/util/workqueue.js

@@ -1,23 +1,23 @@
-module.exports = function(maxSize)
+var util = require("./util");
+
+module.exports = function(name, maxSize, debug)
 {
     if (!maxSize) {
         maxSize = 3;
     }
 
-    var blockExecution = false;
-
     var pendingWorkQueue = [];
     var activeCount = 0;
 
-    var processWork = function () {
-
-        // if another "thread" is running the processor, don't bother
-        if (blockExecution)
+    var debugLog = function(text)
+    {
+        if (debug)
         {
-            return;
+            console.log("[WORKQUEUE: " + name + "] " + text);
         }
+    }
 
-        blockExecution = true;
+    var dispatcherFn = function () {
 
         // add as many pending work items as we can, loop until full or no more pending
         var process = true;
@@ -28,73 +28,84 @@ module.exports = function(maxSize)
             {
                 process = false;
             }
-
-            // if we're full, bail
-            if (activeCount >= maxSize)
-            {
-                process = false;
-            }
-
-            if (process)
+            else
             {
-                // increment active count
-                activeCount++;
-
-                // console.log("Active work items: " + activeCount);
-
-                // define execution function and splice/bind to 0th element from pending list
-                var executionFn = function(work) {
-                    return function() {
-                        var workFn = work.workFn;
-                        var callbackFn = work.callbackFn;
-
-                        // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] start work");
-
-                        workFn(function(err, obj1, obj2) {
-
-                            // decrement active count
-                            activeCount--;
-
-                            // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] finish work");
-
-                            // fire optional callback
-                            if (callbackFn) {
-                                // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] fire work callback");
-                                window.setTimeout(function() {
-                                    callbackFn(err, obj1, obj2);
-                                });
-                            }
-
-                            // process more work on timeout
-                            window.setTimeout(processWork);
-                        });
-
-                    };
-                }(pendingWorkQueue.splice(0, 1)[0]);
-
-                // execute on timeout
-                window.setTimeout(executionFn);
+                // var ids = [];
+                // for (var z = 0; z < pendingWorkQueue.length; z++)
+                // {
+                //     ids.push(pendingWorkQueue[z].id);
+                // }
+                //
+                // debugLog("Dispatcher top, queue: " + ids.join(","));
+
+                //debugLog("dispatcher, pending: " + pendingWorkQueue.length + ", actives: " + activeCount + "]");
+
+                // if we're full, bail
+                if (activeCount >= maxSize)
+                {
+                    process = false;
+                }
+
+                if (process)
+                {
+                    // increment active count
+                    activeCount++;
+
+                    // define execution function and splice/bind to 0th element from pending list
+                    var executionFn = function(work) {
+                        return function() {
+                            var workFn = work.workFn;
+                            var callbackFn = work.callbackFn;
+
+                            debugLog("Start: " + work.id + ", queue: " + pendingWorkQueue.length + ", actives: " + activeCount);
+
+                            workFn(function(err, obj1, obj2) {
+
+                                // fire optional callback
+                                if (callbackFn) {
+                                    window.setTimeout(function() {
+                                        callbackFn(err, obj1, obj2);
+                                    });
+                                }
+
+                                // decrement active count
+                                activeCount--;
+
+                                debugLog("Complete: " + work.id + ", queue: " + pendingWorkQueue.length + ", actives: " + activeCount);
+                            });
+
+                        };
+                    }(pendingWorkQueue.splice(0, 1)[0]);
+
+                    // execute on timeout
+                    window.setTimeout(executionFn);
+                }
             }
 
         } while (process);
 
-        blockExecution = false;
+        // run again on a brief timeout
+        window.setTimeout(dispatcherFn, 50);
     };
 
+    // launch dispatcher
+    window.setTimeout(dispatcherFn);
+
+    // hand back a function to register work onto the queue
     return function(workFn, callbackFn) {
 
-        var pendingWork = {
+        var work = {
+            "id": util.guid(),
             "workFn": workFn
         };
 
         if (callbackFn) {
-            pendingWork.callbackFn = callbackFn;
+            work.callbackFn = callbackFn;
         }
 
-        pendingWorkQueue.push(pendingWork);
+        pendingWorkQueue.push(work);
 
-        // execute on timeout
-        window.setTimeout(processWork);
+        //debugLog("Added to pending queue, id: " + work.id + ", pending: " + pendingWorkQueue.length);
     };
 
 };