cloudcms-server 4.0.0-beta.2 → 4.0.0-beta.20

package/util/cloudcms.js

@@ -7,8 +7,13 @@ var https = require("https");
 
 var request = require("./request");
 
+var workQueueFactory = require("./workqueue");
+
 exports = module.exports = function()
 {
+    // ensures that we only load 5 preview icons at a time
+    var previewQueueFn = workQueueFactory("previewQueue", 5, false);
+
     var toCacheFilePath = function(filePath)
     {
         var filename = path.basename(filePath);
@@ -311,8 +316,13 @@ exports = module.exports = function()
      * @param filePath
      * @param callback
      */
-    var writeToDisk = function(contentStore, gitana, uri, filePath, callback)
+    var writeToDisk = function(contentStore, gitana, uri, filePath, queueFn, callback)
     {
+        if (!callback) {
+            queueFn = callback;
+            queueFn = null;
+        }
+
         var _refreshAccessTokenAndRetry = function(contentStore, gitana, uri, filePath, attemptCount, maxAttemptsAllowed, previousError, cb)
         {
             // tell gitana driver to refresh access token
@@ -346,7 +356,8 @@ exports = module.exports = function()
                 });
             }
             
-            var failFast = function(contentStore, filePath, cb) {
+            var failFast = function(contentStore, filePath, cb)
+            {
                 var triggered = false;
                 
                 return function(tempStream, err)
@@ -373,7 +384,7 @@ exports = module.exports = function()
             contentStore.writeStream(filePath, function(err, tempStream) {
 
                 if (err) {
-                    return failFast(tempStream, err);
+                    return failFast(tempStream, err, cb);
                 }
 
                 var cacheFilePath = toCacheFilePath(filePath);
@@ -396,10 +407,11 @@ exports = module.exports = function()
                     "method": "GET",
                     "url": URL,
                     "qs": {},
+                    "timeout": 3000,
                     "headers": headers,
                     "responseType": "stream"
-                }, function(err, response) {
-    
+                }, function(err, response, stream) {
+
                     if (err) {
                         closeWriteStream(tempStream);
                         return cb(err);
@@ -407,34 +419,34 @@ exports = module.exports = function()
     
                     if (response.status >= 200 && response.status <= 204)
                     {
-                        response.data.pipe(tempStream).on("close", function (err) {
-            
+                        var handle = function (err) {
+
                             if (err) {
                                 // some went wrong at disk io level?
-                                return failFast(tempStream, err);
+                                return failFast(tempStream, err, cb);
                             }
-            
+
                             contentStore.existsFile(filePath, function (exists) {
-                
+
                                 if (exists) {
-                    
+
                                     // write cache file
                                     var cacheInfo = buildCacheInfo(response);
                                     if (!cacheInfo) {
                                         return cb(null, filePath, null);
                                     }
-                    
+
                                     contentStore.writeFile(cacheFilePath, JSON.stringify(cacheInfo, null, "    "), function (err) {
-                        
+
                                         if (err) {
                                             // failed to write cache file, thus the whole thing is invalid
                                             return safeRemove(contentStore, cacheFilePath, function () {
                                                 failFast(tempStream, {
                                                     "message": "Failed to write cache file: " + cacheFilePath + ", err: " + JSON.stringify(err)
-                                                });
+                                                }, cb);
                                             });
                                         }
-                        
+
                                         cb(null, filePath, cacheInfo);
                                     });
                                 } else {
@@ -443,13 +455,27 @@ exports = module.exports = function()
                                     safeRemove(contentStore, cacheFilePath, function () {
                                         failFast(tempStream, {
                                             "message": "Failed to verify written cached file: " + filePath
-                                        });
+                                        }, cb);
                                     });
                                 }
                             });
-            
+                        };
+
+                        // stream.on("end", function() {
+                        //     console.log("g1");
+                        // });
+                        // stream.on("finish", function() {
+                        //     console.log("g2");
+                        // });
+                        stream.on("close", handle);
+
+                        stream.pipe(tempStream).on("close", handle).on("end", function() {
+                            //console.log("a4");
+                        }).on("finish", function() {
+                            //console.log("a5");
                         }).on("error", function (err) {
-                            failFast(tempStream, err);
+                            console.log("a3");
+                            failFast(tempStream, err, cb);
                         });
                     }
                     else
@@ -457,13 +483,13 @@ exports = module.exports = function()
                         // some kind of http error (usually permission denied or invalid_token)
         
                         var body = "";
-        
-                        response.data.on('data', function (chunk) {
+
+                        stream.on('data', function (chunk) {
                             body += chunk;
                         });
-        
-                        response.data.on('end', function () {
-            
+
+                        stream.on('end', function () {
+
                             var afterCleanup = function () {
                 
                                 // see if it is "invalid_token"
@@ -505,11 +531,10 @@ exports = module.exports = function()
                                 });
                             });
                         });
-        
                     }
                 });
                 // }).on('error', function (e) {
-                //     failFast(tempStream, e);
+                //     failFast(tempStream, e, cb);
                 //
                 // }).on('end', function (e) {
                 //
@@ -522,7 +547,7 @@ exports = module.exports = function()
                     process.log("Temp stream errored out");
                     process.log(e);
     
-                    failFast(tempStream, e);
+                    failFast(tempStream, e, cb);
 
                     // ensure stream is closed
                     //closeWriteStream(tempStream);
@@ -532,10 +557,27 @@ exports = module.exports = function()
 
         };
 
-        _writeToDisk(contentStore, gitana, uri, filePath, 0, 2, null, function(err, filePath, cacheInfo) {
+        var workFn = function(contentStore, gitana, uri, filePath)
+        {
+            return function(done)
+            {
+                _writeToDisk(contentStore, gitana, uri, filePath, 0, 2, null, function(err, filePath, cacheInfo) {
+                    done(err, filePath, cacheInfo);
+                });
+            }
+        }(contentStore, gitana, uri, filePath);
+
+        if (queueFn)
+        {
+            return queueFn(workFn, function(err, filePath, cacheInfo) {
+                callback(err, filePath, cacheInfo);
+            });
+        }
+
+        // otherwise, invoke right away
+        workFn(function(err, filePath, cacheInfo) {
             callback(err, filePath, cacheInfo);
         });
-
     };
 
     /**
@@ -555,7 +597,7 @@ exports = module.exports = function()
     var downloadNode = function(contentStore, gitana, repositoryId, branchId, nodeId, attachmentId, nodePath, locale, forceReload, callback)
     {
         // ensure path starts with "/"
-        if (nodePath && nodePath.substring(0, 1) !== "/") {
+        if (nodePath && !nodePath.startsWith("/")) {
             nodePath = "/" + nodePath;
         }
 
@@ -601,7 +643,7 @@ exports = module.exports = function()
                     }
 
                     // grab from Cloud CMS and write to disk
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, cacheInfo) {
+                    writeToDisk(contentStore, gitana, uri, filePath, null, function (err, filePath, cacheInfo) {
 
                         if (err) {
                             process.log("writeToDisk error, err: " + err.message + ", body: " + err.body);
@@ -666,7 +708,7 @@ exports = module.exports = function()
         }
 
         // ensure path starts with "/"
-        if (nodePath && nodePath.substring(0, 1) !== "/") {
+        if (nodePath && !nodePath.startsWith("/")) {
             nodePath = "/" + nodePath;
         }
 
@@ -719,11 +761,11 @@ exports = module.exports = function()
                         uri += "&mimetype=" + mimetype;
                     }
 
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, responseHeaders) {
+                    writeToDisk(contentStore, gitana, uri, filePath, previewQueueFn, function (err, filePath, responseHeaders) {
 
                         if (err) {
                             
-                            if (err.code === 404) {
+                            if (err.status === 404) {
                                 return callback();
                             }
 
@@ -876,7 +918,7 @@ exports = module.exports = function()
                     uri += "?a=true";
 
                     // grab from Cloud CMS and write to disk
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, cacheInfo) {
+                    writeToDisk(contentStore, gitana, uri, filePath, null, function (err, filePath, cacheInfo) {
 
                         if (err) {
                             callback(err);
@@ -978,7 +1020,7 @@ exports = module.exports = function()
                         uri += "&mimetype=" + mimetype;
                     }
 
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, responseHeaders) {
+                    writeToDisk(contentStore, gitana, uri, filePath, previewQueueFn, function (err, filePath, responseHeaders) {
 
                         if (err) {
                             callback(err);

package/util/loaders.js

@@ -0,0 +1,113 @@
+var exports = module.exports;
+
+var AsyncLock = require("async-lock");
+
+var SENTINEL_NOT_FOUND_VALUE = "null";
+
+/**
+ * Applies caching to a loader.
+ *
+ * A loader function is invoked like:
+ *
+ *    loader(function(err, value) {
+ *
+ *    });
+ *
+ * Its job is to load something from a remote place and then fire the callback.
+ *
+ * This method wraps caching around the loader for the given key.  It returns a new loader
+ * that checks a given cache (key) for a value ahead of invoking the actual underlying loader.
+ *
+ * @param loader
+ * @param cache
+ * @param key
+ */
+var cached = exports.cached = function(loader, cache, key)
+{
+    return function(callback)
+    {
+        cache.read(key, function(err, value) {
+
+            if (value === SENTINEL_NOT_FOUND_VALUE) {
+                return callback();
+            }
+
+            if (value) {
+                return callback(null, value);
+            }
+
+            loader(function(err, value) {
+
+                if (err) {
+                    return callback(err);
+                }
+
+                if (!value) {
+                    return cache.write(key, SENTINEL_NOT_FOUND_VALUE, function () {
+                        callback();
+                    });
+                }
+
+                // write to cache
+                return cache.write(key, value, function () {
+                    callback.call(this, null, value);
+                });
+            });
+        });
+    };
+};
+
+var lock = new AsyncLock();
+
+/**
+ * Applies caching to a loader.
+ *
+ * A loader function is invoked like:
+ *
+ *    loader(function(err, value) {
+ *
+ *    });
+ *
+ * Its job is to load something from a remote place and then fire the callback.
+ *
+ * This method wraps an exclusive mutex lock around the given loader.  This makes it so that only one
+ * invocation of this loader may run per key within the event loop.
+ *
+ * @param loader
+ * @param key
+ */
+var exclusive = exports.exclusive = function(loader, key, timeout)
+{
+    return function(callback)
+    {
+        var opts = {};
+        // up to 50000 tasks in the queue
+        opts.maxPending = 50000;
+        if (timeout) {
+            opts.timeout = timeout;
+        }
+
+        lock.acquire(key, function(releaseFn) {
+            loader(function(err, value) {
+                setTimeout(function() {
+                    releaseFn();
+                }, 0);
+                callback.call(this, err, value);
+            });
+        }, function(err, value) {
+            // lock was released
+            if (err) {
+                console.error("Failed to acquire lock for key: " + key + ", error: ", err, " return value: ", value);
+            }
+        }, opts);
+    };
+};
+
+var cachedExclusive = exports.cachedExclusive = function(loader, cache, key, timeout)
+{
+    var cachedLoader1 = cached(loader, cache, key);
+    var exclusiveLoader = exclusive(cachedLoader1, key, timeout);
+    var cachedLoader2 = cached(exclusiveLoader, cache, key);
+
+    return cachedLoader2;
+};