cloudcms-server 4.0.0-beta.2 → 4.0.0-beta.21

package/README.md

@@ -116,11 +116,6 @@ This looks like:
             "modules": "hosts_s3fs"
         }
     },
-    "duster": {
-        "fragments": {
-            "cache": true
-        }
-    },
     "virtualHost": {
         "enabled": false
     },

package/cloudcms-server.iml

@@ -3,6 +3,7 @@
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

package/index.js

@@ -29,12 +29,45 @@ process.logInfo = process.log = function(text, level)
     systemLogger.log(text, level);
 };
 
+// var debugLog = process.debugLog = function(req, message)
+// {
+//     var text = "[" + req.id + "] URL: " + req.url;
+//     // if (req.headers)
+//     // {
+//     //     text += ", HEADERS: " + JSON.stringify(req.headers);
+//     // }
+//     if (req.query)
+//     {
+//         text += ", QUERY: " + JSON.stringify(req.query);
+//     }
+//     text += ", MESSAGE: " + message;
+//
+//     console.log(text);
+// };
+//
+// var debugMiddleware = process.debugMiddleware = function(message)
+// {
+//     return function(req, res, next)
+//     {
+//         debugLog(req, message);
+//
+//         next();
+//     }
+// };
+
+
 
 // by default, set up Gitana driver so that it limits to five concurrent HTTP requests back to Cloud CMS API at at time
 var Gitana = require("gitana");
 
-// default http timeout
-process.defaultHttpTimeoutMs = 60000;
+// default keep alive (3 minutes)
+process.defaultKeepAliveMs = (3 * 60 * 1000);
+
+// default http timeout (2 minutes)
+process.defaultHttpTimeoutMs = 2 * 60 * 1000;
+
+// default exclusive lock timeout (2 minutes)
+process.defaultExclusiveLockTimeoutMs = 2 * 60 * 1000;
 
 if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
 {
@@ -51,27 +84,37 @@ if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
 // dns fix for Node 17 +
 // see: https://nodejs.org/api/dns.html#dnssetdefaultresultorderorder
 var dns = require("dns");
-dns.setDefaultResultOrder("ipv4first");
+if (typeof(dns.setDefaultResultOrder) !== "undefined") {
+    dns.setDefaultResultOrder("ipv4first");
+}
 
 // default agents
 var HttpKeepAliveAgent = require('agentkeepalive');
 var HttpsKeepAliveAgent = require('agentkeepalive').HttpsAgent;
 http.globalAgent = new HttpKeepAliveAgent({
     keepAlive: true,
-    keepAliveMsecs: 5000,
-    maxSockets: 16000,
+    keepAliveMsecs: process.defaultKeepAliveMs,
+    maxSockets: 1024,
     maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    freeSocketTimeout: 4000
+    freeSocketTimeout: 5000
 });
 https.globalAgent = new HttpsKeepAliveAgent({
     keepAlive: true,
-    keepAliveMsecs: 1000,
-    maxSockets: 16000,
+    keepAliveMsecs: process.defaultKeepAliveMs,
+    maxSockets: 1024,
     maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    freeSocketTimeout: 4000
+    freeSocketTimeout: 5000
+});
+
+// install dns cache
+const CacheableLookup = require("cacheable-lookup");
+const cacheable = new CacheableLookup({
+    // Set any custom options here
 });
+cacheable.install(http.globalAgent);
+cacheable.install(https.globalAgent);
 
 // disable for now
 /*
@@ -141,8 +184,8 @@ exports = module.exports = function()
     // not already specified
     var defaultGitanaProxyScheme = "https";
     var defaultGitanaProxyHost = "api.cloudcms.com";
-    var defaultGitanaProxyPort = 443;
     var defaultGitanaProxyPath = "";
+    var defaultGitanaProxyPort = 443;
 
     var gitanaJsonPath = path.join(process.env.CLOUDCMS_APPSERVER_BASE_PATH, "gitana.json");
     if (fs.existsSync(gitanaJsonPath))
@@ -153,8 +196,8 @@ exports = module.exports = function()
             var parsedUrl = url.parse(gitanaJson.baseURL);
 
             defaultGitanaProxyHost = parsedUrl.hostname;
-            defaultGitanaProxyScheme = parsedUrl.protocol.substring(0, parsedUrl.protocol.length - 1); // remove the :
             defaultGitanaProxyPath = parsedUrl.path;
+            defaultGitanaProxyScheme = parsedUrl.protocol.substring(0, parsedUrl.protocol.length - 1); // remove the :
 
             if (parsedUrl.port)
             {
@@ -178,13 +221,13 @@ exports = module.exports = function()
     if (!process.env.GITANA_PROXY_HOST) {
         process.env.GITANA_PROXY_HOST = defaultGitanaProxyHost;
     }
-    if (!process.env.GITANA_PROXY_PORT) {
-        process.env.GITANA_PROXY_PORT = defaultGitanaProxyPort;
-    }
     if (!process.env.GITANA_PROXY_PATH) {
         process.env.GITANA_PROXY_PATH = defaultGitanaProxyPath;
     }
-    
+    if (!process.env.GITANA_PROXY_PORT) {
+        process.env.GITANA_PROXY_PORT = defaultGitanaProxyPort;
+    }
+
     if (cluster.isMaster)
     {
         process.log("Gitana Proxy pointed to: " + util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
@@ -223,13 +266,11 @@ exports = module.exports = function()
     var registration = require("./middleware/registration/registration");
     var resources = require("./middleware/resources/resources");
     var runtime = require("./middleware/runtime/runtime");
-    var serverTags = require("./middleware/server-tags/server-tags");
     var storeService = require("./middleware/stores/stores");
     var templates = require("./middleware/templates/templates");
     var themes = require("./middleware/themes/themes");
     var virtualConfig = require("./middleware/virtual-config/virtual-config");
     var virtualFiles = require("./middleware/virtual-files/virtual-files");
-    var wcm = require("./middleware/wcm/wcm");
     var welcome = require("./middleware/welcome/welcome");
     var awareness = require("./middleware/awareness/awareness");
     var userAgent = require('express-useragent');
@@ -469,15 +510,6 @@ exports = module.exports = function()
 
         // supports user-configured dynamic configuration
         app.use(config.remoteConfigInterceptor());
-
-        // tag processing, injection of scripts, etc, kind of a catch all at the moment
-        app.use(serverTags.interceptor(configuration));
-
-        if (includeCloudCMS)
-        {
-            // handles retrieval of content from wcm
-            app.use(wcm.wcmInterceptor());
-        }
     };
 
     r.handlers = function(app, includeCloudCMS)
@@ -557,12 +589,6 @@ exports = module.exports = function()
         // add User-Agent device info to req
         app.use(userAgent.express());
 
-        if (includeCloudCMS)
-        {
-            // handles retrieval of content from wcm
-            app.use(wcm.wcmHandler());
-        }
-
         // handles 404
         app.use(final.finalHandler());
     };

package/middleware/authentication/authentication.js

@@ -315,16 +315,11 @@ exports = module.exports = function()
         registerAdapter("session", require("./adapters/session"));
 
         // providers
-        registerProvider("cas", require("./providers/cas"));
-        registerProvider("facebook", require("./providers/facebook"));
-        registerProvider("github", require("./providers/github"));
         registerProvider("google", require("./providers/google"));
         registerProvider("keycloak", require("./providers/keycloak"));
-        registerProvider("linkedin", require("./providers/linkedin"));
         registerProvider("local", require("./providers/local"));
         registerProvider("saml", require("./providers/saml"));
         registerProvider("trusted", require("./providers/trusted"));
-        registerProvider("twitter", require("./providers/twitter"));
 
         // authenticators
         registerAuthenticator("default", require("./authenticators/default"));
@@ -361,7 +356,7 @@ exports = module.exports = function()
                         };
                     }
 
-                    console.log("Auth Callback failed, err: " + JSON.stringify(err));
+                    console.log("Auth Callback failed, err: " + err + ", err json: " + JSON.stringify(err));
 
                     if (err.message)
                     {
@@ -390,13 +385,19 @@ exports = module.exports = function()
 
                 return function (err, profile, info) {
 
+                    if (err) {
+                        console.log("Caught error on auth callback function: ", err, JSON.stringify(err));
+                    }
+
                     if (err) {
                         return handleFailure(err, res);
                     }
 
                     if (!profile || !info)
                     {
-                        return handleFailure(null, res);
+                        return handleFailure({
+                            "message": "Authentication callback missing both profile and info"
+                        }, res);
                     }
 
                     // store these onto request
@@ -695,11 +696,11 @@ exports = module.exports = function()
         return function(req, res, next) {
 
             // record filter start time
-            var _auth_filter_start_ms = new Date().getTime();
+            var _auth_filter_start_ms = Date.now();
 
             fn(req, res, function(result, authenticator) {
 
-                var _auth_filter_end_ms = new Date().getTime() - _auth_filter_start_ms;
+                var _auth_filter_end_ms = Date.now() - _auth_filter_start_ms;
 
                 util.setHeader(res, "x-cloudcms-auth-filter-ms", _auth_filter_end_ms);
 
@@ -736,8 +737,11 @@ exports = module.exports = function()
                         };
                     }
 
-                    if (result.err && result.err.message) {
-                        req.log("Auth strategy: " + strategyId + " - filter error: " + result.err.message);
+                    if (!result.skip)
+                    {
+                        if (result.err && result.err.message) {
+                            req.log("Auth strategy: " + strategyId + " - filter error: " + result.err.message);
+                        }
                     }
 
                     var providerId = null;
@@ -810,7 +814,31 @@ exports = module.exports = function()
                             }
                             redirectUrl += "requested_url=" + requested_url;
 
-                            return res.redirect(redirectUrl);
+                            res.status(200);
+                            res.type("text/html");
+
+                            // serve back a redirect via html
+                            var html = " \
+                                <html> \
+                                    <head> \
+                                        <script> \
+                                            var _redirectUrl = '" + redirectUrl + "'; \
+                                            var hash = window.location.hash ? window.location.hash : ''; \
+                                            if (hash && hash.indexOf('#') === 0) { \
+                                                hash = hash.substring(1); \
+                                            } \
+                                            if (hash) { \
+                                                _redirectUrl += '&requested_hash=' + hash; \
+                                            } \
+                                            window.location.href = _redirectUrl; \
+                                </script> \
+                                    </head> \
+                                </html> \
+                            ";
+                            res.send(html);
+                            return;
+
+                            //return res.redirect(redirectUrl);
                         }
                         else if (loginHandler)
                         {

package/middleware/authentication/providers/saml.js

@@ -1,6 +1,6 @@
 //var auth = require("../../../util/auth");
 
-var SamlStrategy = require('passport-saml').Strategy;
+var SamlStrategy = require('@node-saml/passport-saml').Strategy;
 var AbstractProvider = require("./abstract");
 
 if (!process.configuration) {
@@ -57,10 +57,14 @@ class SAMLProvider extends AbstractProvider
             samlConfig.entryPoint = config.entryPoint;
         }
         if (config.cert) {
-            samlConfig.cert = config.cert;
+            samlConfig.idpCert = config.cert;
         }
-        if (config.callbackURL) {
-            samlConfig.callbackUrl = "http://localhost:5000" + config.callbackURL;
+        var callbackUrl = config.callbackURL;
+        if (!callbackUrl) {
+            callbackUrl = config.callbackUrl;
+        }
+        if (callbackUrl) {
+            samlConfig.callbackUrl = "http://localhost:5000" + callbackUrl;
         }
         if (config.issuer) {
             samlConfig.issuer = config.issuer;

package/middleware/awareness/awareness.js

@@ -16,7 +16,8 @@ exports = module.exports = function()
 
     var pluginPaths = [
         "./plugins/editorial",
-        "./plugins/resources"
+        "./plugins/resources",
+        "./plugins/api_event"
     ];
     var plugins = {};
 
@@ -149,21 +150,21 @@ exports = module.exports = function()
             var r = {};
 
             // allow plugins to bind on("connection") handlers
-            r.bindOnSocketConnection = function(socket, provider, callback)
+            r.bindOnSocketConnection = function(socket, provider, io, callback)
             {
                 var fns = [];
                 for (var pluginPath in plugins)
                 {
                     var plugin = plugins[pluginPath];
 
-                    var fn = function(pluginPath, plugin, socket) {
+                    var fn = function(pluginPath, plugin, socket, io) {
                         return function(done) {
-
-                            plugin.bindSocket(socket, provider);
+    
+                            plugin.bindSocket(socket, provider, io);
 
                             done();
                         }
-                    }(pluginPath, plugin, socket);
+                    }(pluginPath, plugin, socket, io);
                     fns.push(fn);
                 }
 
@@ -309,7 +310,7 @@ exports = module.exports = function()
             });
 
             // allow plugins to register more on() handlers if they wish
-            pluginProxy.bindOnSocketConnection(socket, provider, function() {
+            pluginProxy.bindOnSocketConnection(socket, provider, io, function() {
                 // done
             });
 

package/middleware/awareness/plugins/api_event.js

@@ -0,0 +1,105 @@
+/*
+
+this awareness plugin adds support for general purpose API events
+the API in 4.0 supports passing API events forward via the message queue to the app server
+the app server then converts these into notifications
+
+the UI can register and event listener via socket.io ("onApiEvent").
+a listener binds to an event type + an optional reference
+
+when an event is received by the API, it triggers to socket.io emits (one for the general event and the other for the
+event + the reference being acted upon)
+ */
+exports = module.exports = {};
+
+var util = require("../../../util/util");
+var socketUtil = require("../../../util/socket");
+
+var subscriptionsBound = false;
+
+var bindSubscriptions = function(io)
+{
+    if (subscriptionsBound) {
+        return;
+    }
+
+    subscriptionsBound = true;
+    
+    // LISTEN: "api_event"
+    process.broadcast.subscribe("api_event", function (message, channel, done) {
+        
+        if (!done) {
+            done = function () {};
+        }
+    
+        // the message
+        // {
+        //     "applicationId": applicationId,
+        //     "deploymentKey": deploymentKey,
+        //     "host": host,
+        //     "eventType": eventType,
+        //     "eventId": eventId,
+        //     "objectType": objectType,
+        //     "objectId": objectId,
+        //     "objectRef": objectRef,
+        //     "object": object.object
+        // };
+    
+        var apiEvent = {};
+        apiEvent.type = message.eventType;
+        apiEvent.id = message.eventId;
+        apiEvent.objectType = message.objectType;
+        apiEvent.objectId = message.objectId;
+        apiEvent.objectRef = message.objectRef;
+        apiEvent.object = message.object;
+        
+        // dispatch for event + reference
+        if (apiEvent.objectRef)
+        {
+            try {
+                apiEvent.channelId = "apiEvent-" + apiEvent.type + "_" + apiEvent.objectRef;
+                //console.log("api_event -> " + apiEvent.channelId);
+                io.to(apiEvent.channelId).emit("apiEvent", apiEvent);
+            } catch (e) {
+                console.log(e);
+            }
+        }
+    
+        // dispatch for event
+        try {
+            apiEvent.channelId = "apiEvent-" + apiEvent.type;
+            //console.log("apiEvent -> " + apiEvent.channelId);
+            io.to(apiEvent.channelId).emit("apiEvent", apiEvent);
+        } catch (e) {
+            console.log(e);
+        }
+        
+        done();
+    });
+};
+
+exports.bindSocket = function(socket, provider, io)
+{
+    bindSubscriptions(io);
+
+    socketUtil.bindGitana(socket, function() {
+
+        socket.on("onApiEvent", function(eventName, reference, callback) {
+
+            if (eventName)
+            {
+                var channelId = "apiEvent-" + eventName;
+                
+                if (reference)
+                {
+                    channelId = eventName + "-" + reference;
+                }
+                
+                // join room for this event
+                socket.join(channelId);
+            }
+
+            callback();
+        });
+    });
+};

package/middleware/awareness/plugins/editorial.js

@@ -5,7 +5,7 @@ var socketUtil = require("../../../util/socket");
 
 var request = require("../../../util/request");
 
-exports.bindSocket = function(socket, provider)
+exports.bindSocket = function(socket, provider, io)
 {
     socketUtil.bindGitana(socket, function() {
 
@@ -21,6 +21,10 @@ exports.bindSocket = function(socket, provider)
             commitEditorialWorkspace(socket, provider, sessionKey, repositoryId, branchId, callback);
         });
 
+        socket.on("touchEditorialWorkspace", function(sessionKey, repositoryId, branchId, callback) {
+            touchEditorialWorkspace(socket, provider, sessionKey, repositoryId, branchId, callback);
+        });
+
         socket.on("editorialWorkspaceInfo", function(sessionKey, repositoryId, branchId, callback) {
             editorialWorkspaceInfo(socket, provider, sessionKey, repositoryId, branchId, callback);
         });
@@ -219,6 +223,52 @@ exports.bindSocket = function(socket, provider)
         });
     };
 
+    /**
+     * Touches an editorial workspace.
+     *
+     * This is a "keep alive" call to prevent a workspace from being cleaned up while it is being
+     * worked on.
+     *
+     * @param socket
+     * @param provider
+     * @param sessionKey
+     * @param repositoryId
+     * @param branchId
+     * @param callback
+     */
+    var touchEditorialWorkspace = function(socket, provider, sessionKey, repositoryId, branchId, callback)
+    {
+        var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + "/oneteam/editorial/workspace/touch";
+
+        var json = {};
+        json.repositoryId = repositoryId;
+        json.branchId = branchId;
+        json.key = sessionKey;
+
+        var headers = {};
+        var gitanaTicket = extractTicket(socket);
+        if (gitanaTicket)
+        {
+            headers["GITANA_TICKET"] = gitanaTicket;
+        }
+
+        request({
+            "method": "POST",
+            "url": URL,
+            "qs": {},
+            "json": json,
+            "headers": headers,
+            "timeout": process.defaultHttpTimeoutMs
+        }, function(err, response, json) {
+
+            if (err || (json && json.error)) {
+                return callback(err);
+            }
+
+            callback(null, json);
+        });
+    };
+
     var extractTicket = function(socket)
     {
         var ticket = null;
@@ -231,10 +281,11 @@ exports.bindSocket = function(socket, provider)
                 var x1 = cookieValue.indexOf("GITANA_TICKET=");
                 if (x1 > -1)
                 {
-                    var x2 = cookieValue.indexOf(";", x1 + 14);
+                    ticket = cookieValue.substring(x1 + 14);
+                    var x2 = ticket.indexOf(";");
                     if (x2 > -1)
                     {
-                        ticket = cookieValue.substring(x1 + 14, x2);
+                        ticket = ticket.substring(0, x2);
                     }
                 }
             }

package/middleware/awareness/plugins/resources.js

@@ -5,7 +5,7 @@ var socketUtil = require("../../../util/socket");
 
 var subscriptionsBound = false;
 
-var bindSubscriptions = function()
+var bindSubscriptions = function(io)
 {
     if (subscriptionsBound) {
         return;
@@ -43,19 +43,27 @@ var bindSubscriptions = function()
             };
 
             // fire to reference
-            process.IO.to(reference).emit("watchResource", reference, watchObject);
+            try {
+                if (reference) {
+                    io.to(reference).emit("watchResource", reference, watchObject);
+                }
+            } catch (e) { }
 
             // fire to head reference
-            process.IO.to(headReference).emit("watchResource", headReference, watchObject);
+            try {
+                if (headReference) {
+                    io.to(headReference).emit("watchResource", headReference, watchObject);
+                }
+            } catch (e) { }
         }
 
         done();
     });
 };
 
-exports.bindSocket = function(socket, provider)
+exports.bindSocket = function(socket, provider, io)
 {
-    bindSubscriptions();
+    bindSubscriptions(io);
 
     socketUtil.bindGitana(socket, function() {
 

package/middleware/config/adapter.js

@@ -359,50 +359,6 @@ module.exports = function(configStore)
 
             // all done
             callback(null, registry);
-
-            var _watchLog = function(text) {
-                process.log("[Configuration Watch] " + text);
-            };
-
-            if (process.env.CLOUDCMS_APPSERVER_CONFIG_WATCH === "true" || process.env.CLOUDCMS_APPSERVER_CONFIG_WATCH === true)
-            {
-                _watchLog("Setting up live watch...");
-
-                // set a watch
-                // watch for changes and when they happen, reload context
-                (function (registry) {
-
-                    configStore.watchDirectory("/", function () {
-
-                        _watchLog("Detected changes on disk - reloading...");
-
-                        var t1 = new Date().getTime();
-
-                        // reload context
-                        loadContext(function (err, context) {
-
-                            if (err) {
-                                return _watchLog("Failed to load configuration context: " + err);
-                            }
-
-                            try
-                            {
-                                compileContextToRegistry(context);
-                                registry.reloadContext(context);
-
-                                var t2 = new Date().getTime();
-                                _watchLog("Reloaded context in: " + (t2 - t1) + " ms");
-                            }
-                            catch (e)
-                            {
-                                _watchLog("Caught error while compiling and reloading context: " + err);
-                            }
-                        });
-                    });
-
-                })(registry);
-            }
-
         });
     };