cloudcms-server 4.0.0-beta.11 → 4.0.0-beta.14

package/util/cloudcms.js

@@ -7,8 +7,13 @@ var https = require("https");
 
 var request = require("./request");
 
+var workQueueFactory = require("./workqueue");
+
 exports = module.exports = function()
 {
+    // ensures that we only load 3 preview icons at a time
+    var previewQueueFn = workQueueFactory(3);
+
     var toCacheFilePath = function(filePath)
     {
         var filename = path.basename(filePath);
@@ -311,8 +316,13 @@ exports = module.exports = function()
      * @param filePath
      * @param callback
      */
-    var writeToDisk = function(contentStore, gitana, uri, filePath, callback)
+    var writeToDisk = function(contentStore, gitana, uri, filePath, queueFn, callback)
     {
+        if (!callback) {
+            queueFn = callback;
+            queueFn = null;
+        }
+
         var _refreshAccessTokenAndRetry = function(contentStore, gitana, uri, filePath, attemptCount, maxAttemptsAllowed, previousError, cb)
         {
             // tell gitana driver to refresh access token
@@ -532,10 +542,27 @@ exports = module.exports = function()
 
         };
 
-        _writeToDisk(contentStore, gitana, uri, filePath, 0, 2, null, function(err, filePath, cacheInfo) {
+        var workFn = function(contentStore, gitana, uri, filePath)
+        {
+            return function(done)
+            {
+                _writeToDisk(contentStore, gitana, uri, filePath, 0, 2, null, function(err, filePath, cacheInfo) {
+                    done(err, filePath, cacheInfo);
+                });
+            }
+        }(contentStore, gitana, uri, filePath);
+
+        if (queueFn)
+        {
+            return queueFn(workFn, function(err, filePath, cacheInfo) {
+                callback(err, filePath, cacheInfo);
+            });
+        }
+
+        // otherwise, invoke right away
+        workFn(function(err, filePath, cacheInfo) {
             callback(err, filePath, cacheInfo);
         });
-
     };
 
     /**
@@ -601,7 +628,7 @@ exports = module.exports = function()
                     }
 
                     // grab from Cloud CMS and write to disk
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, cacheInfo) {
+                    writeToDisk(contentStore, gitana, uri, filePath, null, function (err, filePath, cacheInfo) {
 
                         if (err) {
                             process.log("writeToDisk error, err: " + err.message + ", body: " + err.body);
@@ -719,7 +746,7 @@ exports = module.exports = function()
                         uri += "&mimetype=" + mimetype;
                     }
 
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, responseHeaders) {
+                    writeToDisk(contentStore, gitana, uri, filePath, previewQueueFn, function (err, filePath, responseHeaders) {
 
                         if (err) {
                             
@@ -876,7 +903,7 @@ exports = module.exports = function()
                     uri += "?a=true";
 
                     // grab from Cloud CMS and write to disk
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, cacheInfo) {
+                    writeToDisk(contentStore, gitana, uri, filePath, null, function (err, filePath, cacheInfo) {
 
                         if (err) {
                             callback(err);
@@ -978,7 +1005,7 @@ exports = module.exports = function()
                         uri += "&mimetype=" + mimetype;
                     }
 
-                    writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, responseHeaders) {
+                    writeToDisk(contentStore, gitana, uri, filePath, previewQueueFn, function (err, filePath, responseHeaders) {
 
                         if (err) {
                             callback(err);

package/util/loaders.js

@@ -0,0 +1,113 @@
+var exports = module.exports;
+
+var AsyncLock = require("async-lock");
+
+var SENTINEL_NOT_FOUND_VALUE = "null";
+
+/**
+ * Applies caching to a loader.
+ *
+ * A loader function is invoked like:
+ *
+ *    loader(function(err, value) {
+ *
+ *    });
+ *
+ * Its job is to load something from a remote place and then fire the callback.
+ *
+ * This method wraps caching around the loader for the given key.  It returns a new loader
+ * that checks a given cache (key) for a value ahead of invoking the actual underlying loader.
+ *
+ * @param loader
+ * @param cache
+ * @param key
+ */
+var cached = exports.cached = function(loader, cache, key)
+{
+    return function(callback)
+    {
+        cache.read(key, function(err, value) {
+
+            if (value === SENTINEL_NOT_FOUND_VALUE) {
+                return callback();
+            }
+
+            if (value) {
+                return callback(null, value);
+            }
+
+            loader(function(err, value) {
+
+                if (err) {
+                    return callback(err);
+                }
+
+                if (!value) {
+                    return cache.write(key, SENTINEL_NOT_FOUND_VALUE, function () {
+                        callback();
+                    });
+                }
+
+                // write to cache
+                return cache.write(key, value, function () {
+                    callback.call(this, null, value);
+                });
+            });
+        });
+    };
+};
+
+var lock = new AsyncLock();
+
+/**
+ * Applies caching to a loader.
+ *
+ * A loader function is invoked like:
+ *
+ *    loader(function(err, value) {
+ *
+ *    });
+ *
+ * Its job is to load something from a remote place and then fire the callback.
+ *
+ * This method wraps an exclusive mutex lock around the given loader.  This makes it so that only one
+ * invocation of this loader may run per key within the event loop.
+ *
+ * @param loader
+ * @param key
+ */
+var exclusive = exports.exclusive = function(loader, key, timeout)
+{
+    return function(callback)
+    {
+        var opts = {};
+        // up to 50000 tasks in the queue
+        opts.maxPending = 50000;
+        if (timeout) {
+            opts.timeout = timeout;
+        }
+
+        lock.acquire(key, function(releaseFn) {
+            loader(function(err, value) {
+                setTimeout(function() {
+                    releaseFn();
+                }, 0);
+                callback.call(this, err, value);
+            });
+        }, function(err, value) {
+            // lock was released
+            if (err) {
+                console.error("Failed to acquire lock for key: " + key + ", error: ", err, " return value: ", value);
+            }
+        }, opts);
+    };
+};
+
+var cachedExclusive = exports.cachedExclusive = function(loader, cache, key, timeout)
+{
+    var cachedLoader1 = cached(loader, cache, key);
+    var exclusiveLoader = exclusive(cachedLoader1, key, timeout);
+    var cachedLoader2 = cached(exclusiveLoader, cache, key);
+
+    return cachedLoader2;
+};

package/util/workqueue.js

@@ -0,0 +1,100 @@
+module.exports = function(maxSize)
+{
+    if (!maxSize) {
+        maxSize = 3;
+    }
+
+    var blockExecution = false;
+
+    var pendingWorkQueue = [];
+    var activeCount = 0;
+
+    var processWork = function () {
+
+        // if another "thread" is running the processor, don't bother
+        if (blockExecution)
+        {
+            return;
+        }
+
+        blockExecution = true;
+
+        // add as many pending work items as we can, loop until full or no more pending
+        var process = true;
+        do
+        {
+            // if nothing to work on, bail
+            if (pendingWorkQueue.length === 0)
+            {
+                process = false;
+            }
+
+            // if we're full, bail
+            if (activeCount >= maxSize)
+            {
+                process = false;
+            }
+
+            if (process)
+            {
+                // increment active count
+                activeCount++;
+
+                // console.log("Active work items: " + activeCount);
+
+                // define execution function and splice/bind to 0th element from pending list
+                var executionFn = function(work) {
+                    return function() {
+                        var workFn = work.workFn;
+                        var callbackFn = work.callbackFn;
+
+                        // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] start work");
+
+                        workFn(function(err, obj1, obj2) {
+
+                            // decrement active count
+                            activeCount--;
+
+                            // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] finish work");
+
+                            // fire optional callback
+                            if (callbackFn) {
+                                // console.log("[WORKQUEUE - queue: " + pendingWorkQueue.length + ", actives: " + activeCount + "] fire work callback");
+                                window.setTimeout(function() {
+                                    callbackFn(err, obj1, obj2);
+                                });
+                            }
+
+                            // process more work on timeout
+                            window.setTimeout(processWork);
+                        });
+
+                    };
+                }(pendingWorkQueue.splice(0, 1)[0]);
+
+                // execute on timeout
+                window.setTimeout(executionFn);
+            }
+
+        } while (process);
+
+        blockExecution = false;
+    };
+
+    return function(workFn, callbackFn) {
+
+        var pendingWork = {
+            "workFn": workFn
+        };
+
+        if (callbackFn) {
+            pendingWork.callbackFn = callbackFn;
+        }
+
+        pendingWorkQueue.push(pendingWork);
+
+        // execute on timeout
+        window.setTimeout(processWork);
+    };
+
+};

package/duster/helpers/core/cloudcms/associations.js

@@ -1,34 +0,0 @@
-/**
- * @associations
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../engine")(app, dust);
-
-    /**
-     * ASSOCIATIONS
-     *
-     * Finds associations around a node.
-     *
-     * Syntax:
-     *
-     *    {@associations node="<nodeId>" type="<association_type>" limit="" skip="" as=""}
-     *       {+templateIdentifier/}
-     *    {/associations}
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     */
-    dust.helpers.associations = function(chunk, context, bodies, params)
-    {
-        return engine.handleAssociations(chunk, context, bodies, params);
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/beta/markdown.js

@@ -1,46 +0,0 @@
-var marked = require("marked");
-marked.setOptions({
-    renderer: new marked.Renderer(),
-    gfm: true,
-    tables: true,
-    breaks: false,
-    pedantic: false,
-    sanitize: true,
-    smartLists: true,
-    smartypants: false
-});
-
-/**
- * @markdown
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    /**
-     * Renders markdown into the Dust template.
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     * @returns {*}
-     */
-    dust.helpers.markdown = function(chunk, context, bodies, params) {
-
-        params = params || {};
-
-        var text = context.resolve(params.text);
-        if (!text) {
-            return chunk;
-        }
-
-        text = marked(text);
-
-        return chunk.write(text);
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/beta/nodeAttachmentText.js

@@ -1,46 +0,0 @@
-/**
- * @nodeAttachmentText
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../../engine")(app, dust);
-
-    var map = engine.map;
-    var end = engine.end;
-
-    dust.helpers.nodeAttachmentText = dust.helpers.nodeAttachmentValue = function(chunk, context, bodies, params)
-    {
-        params = params || {};
-
-        var nodeId = context.resolve(params.node);
-        var attachmentId = context.resolve(params.attachment);
-        if (!attachmentId)
-        {
-            attachmentId = "default";
-        }
-
-        return map(chunk, function(chunk) {
-
-            var req = context.get("req");
-            req.branch(function(err, branch) {
-
-                if (err) {
-                    return end(chunk, context);
-                }
-
-                branch.readNode(nodeId).attachment(attachmentId).download(function(text) {
-
-                    chunk.write(text);
-
-                    end(chunk, context);
-                });
-            });
-        });
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/beta/params.js

@@ -1,33 +0,0 @@
-/**
- * @params
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    /**
-     * Allows parameters to be passed into blocks or partials
-     */
-    dust.helpers.params = function( chunk, context, bodies, params ){
-
-        var partial = {};
-        if( params)
-        {
-            for (var key in params)
-            {
-                partial[key] = params[key];
-            }
-        }
-
-        // render
-        var newContext = context.push(partial);
-
-        //return bodies.block(chunk, dust.makeBase(partial))
-        return bodies.block(chunk, newContext);
-    };
-    dust.helpers.parameters = dust.helpers.params;
-
-    callback();
-};

package/duster/helpers/core/cloudcms/beta/processTemplate.js

@@ -1,82 +0,0 @@
-/**
- * @processTemplate
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../../engine")(app, dust);
-
-    var map = engine.map;
-    var end = engine.end;
-
-    dust.helpers.processTemplate = function(chunk, context, bodies, params)
-    {
-        params = params || {};
-
-        var nodeId = context.resolve(params.node);
-        var attachmentId = context.resolve(params.attachment);
-        if (!attachmentId)
-        {
-            attachmentId = "default";
-        }
-        var propertyId = context.resolve(params.property);
-        var locale = context.resolve(params.locale);
-
-        return map(chunk, function(chunk) {
-
-            if (locale)
-            {
-                var gitana = context.get("gitana");
-                gitana.getDriver().setLocale(locale);
-            }
-
-            if (propertyId)
-            {
-                var req = context.get("req");
-                req.branch(function(err, branch) {
-
-                    if (err) {
-                        return end(chunk, context);
-                    }
-
-                    branch.readNode(nodeId).then(function() {
-
-                        resolveVariables([this[propertyId]], context, function (err, resolutions) {
-
-                            chunk.write(resolutions[0]);
-
-                            end(chunk, context);
-
-                        });
-                    });
-                });
-            }
-            else
-            {
-                var req = context.get("req");
-                req.branch(function(err, branch) {
-
-                    if (err) {
-                        return end(chunk, context);
-                    }
-
-                    branch.readNode(nodeId).attachment(attachmentId).download(function (text) {
-
-                        resolveVariables([text], context, function (err, resolutions) {
-
-                            chunk.write(resolutions[0]);
-
-                            end(chunk, context);
-
-                        });
-                    });
-                });
-            }
-        });
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/content.js

@@ -1,34 +0,0 @@
-/**
- * @content
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../engine")(app, dust);
-
-    /**
-     * CONTENT
-     *
-     * Selects a single content item.
-     *
-     * Syntax:
-     *
-     *    {@content id="GUID" path="/a/b/c" as=""}
-     *       {+templateIdentifier/}
-     *    {/content}
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     */
-    dust.helpers.content = function(chunk, context, bodies, params)
-    {
-        return engine.handleContent(chunk, context, bodies, params);
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/expand.js

@@ -1,38 +0,0 @@
-/**
- * @expand
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../engine")(app, dust);
-
-    /**
-     * EXPAND
-     *
-     * Query for a list of nodes by their id (._doc).
-     * The list of id values can be supplied as an array of id values in
-     * the 'list' arg or as an array of node records in the 'list' arg
-     * along with a the name of a common key within each node that holds
-     * the id.
-     *
-     * Syntax:
-     *   ex. 1
-	 *      {@expand list="components" key="editorialpage.components" as="components"}
-     *          {+templateIdentifier/}
-     *      {/expand}
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     */
-    dust.helpers.expand = function(chunk, context, bodies, params)
-    {
-        return engine.handleExpand(chunk, context, bodies, params);
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/form.js

@@ -1,34 +0,0 @@
-/**
- * @form
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../engine")(app, dust);
-
-    /**
-     * FORM
-     *
-     * Renders a form.
-     *
-     * Syntax:
-     *
-     *    {@form definition="custom:type" form="formKey" list="listKeyOrId" successUrl="" errorUrl=""}
-     *       {+templateIdentifier/}
-     *    {/form}
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     */
-    dust.helpers.form = function(chunk, context, bodies, params)
-    {
-        return engine.handleForm(chunk, context, bodies, params);
-    };
-
-    callback();
-};

package/duster/helpers/core/cloudcms/query.js

@@ -1,34 +0,0 @@
-/**
- * @query
- *
- * @param app
- * @param dust
- * @param callback
- */
-module.exports = function(app, dust, callback)
-{
-    var engine = require("../engine")(app, dust);
-
-    /**
-     * QUERY
-     *
-     * Queries for content from the content repository and renders.
-     *
-     * Syntax:
-     *
-     *    {@query sort="title" scope="page" type="custom:type" limit="" skip="" as=""}
-     *       {+templateIdentifier/}
-     *    {/query}
-     *
-     * @param chunk
-     * @param context
-     * @param bodies
-     * @param params
-     */
-    dust.helpers.query = function(chunk, context, bodies, params)
-    {
-        return engine.handleQuery(chunk, context, bodies, params, false);
-    };
-
-    callback();
-};