cloudcms-server 4.0.0-beta.1 → 4.0.0-beta.10

package/cloudcms-server.iml

@@ -3,6 +3,7 @@
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

package/middleware/authentication/providers/saml.js

@@ -1,6 +1,6 @@
 //var auth = require("../../../util/auth");
 
-var SamlStrategy = require('passport-saml').Strategy;
+var SamlStrategy = require('@node-saml/passport-saml').Strategy;
 var AbstractProvider = require("./abstract");
 
 if (!process.configuration) {

package/middleware/awareness/awareness.js

@@ -16,7 +16,8 @@ exports = module.exports = function()
 
     var pluginPaths = [
         "./plugins/editorial",
-        "./plugins/resources"
+        "./plugins/resources",
+        "./plugins/api_event"
     ];
     var plugins = {};
 
@@ -149,21 +150,21 @@ exports = module.exports = function()
             var r = {};
 
             // allow plugins to bind on("connection") handlers
-            r.bindOnSocketConnection = function(socket, provider, callback)
+            r.bindOnSocketConnection = function(socket, provider, io, callback)
             {
                 var fns = [];
                 for (var pluginPath in plugins)
                 {
                     var plugin = plugins[pluginPath];
 
-                    var fn = function(pluginPath, plugin, socket) {
+                    var fn = function(pluginPath, plugin, socket, io) {
                         return function(done) {
-
-                            plugin.bindSocket(socket, provider);
+    
+                            plugin.bindSocket(socket, provider, io);
 
                             done();
                         }
-                    }(pluginPath, plugin, socket);
+                    }(pluginPath, plugin, socket, io);
                     fns.push(fn);
                 }
 
@@ -309,7 +310,7 @@ exports = module.exports = function()
             });
 
             // allow plugins to register more on() handlers if they wish
-            pluginProxy.bindOnSocketConnection(socket, provider, function() {
+            pluginProxy.bindOnSocketConnection(socket, provider, io, function() {
                 // done
             });
 

package/middleware/awareness/plugins/api_event.js

@@ -0,0 +1,105 @@
+/*
+
+this awareness plugin adds support for general purpose API events
+the API in 4.0 supports passing API events forward via the message queue to the app server
+the app server then converts these into notifications
+
+the UI can register and event listener via socket.io ("onApiEvent").
+a listener binds to an event type + an optional reference
+
+when an event is received by the API, it triggers to socket.io emits (one for the general event and the other for the
+event + the reference being acted upon)
+ */
+exports = module.exports = {};
+
+var util = require("../../../util/util");
+var socketUtil = require("../../../util/socket");
+
+var subscriptionsBound = false;
+
+var bindSubscriptions = function(io)
+{
+    if (subscriptionsBound) {
+        return;
+    }
+
+    subscriptionsBound = true;
+    
+    // LISTEN: "api_event"
+    process.broadcast.subscribe("api_event", function (message, channel, done) {
+        
+        if (!done) {
+            done = function () {};
+        }
+    
+        // the message
+        // {
+        //     "applicationId": applicationId,
+        //     "deploymentKey": deploymentKey,
+        //     "host": host,
+        //     "eventType": eventType,
+        //     "eventId": eventId,
+        //     "objectType": objectType,
+        //     "objectId": objectId,
+        //     "objectRef": objectRef,
+        //     "object": object.object
+        // };
+    
+        var apiEvent = {};
+        apiEvent.type = message.eventType;
+        apiEvent.id = message.eventId;
+        apiEvent.objectType = message.objectType;
+        apiEvent.objectId = message.objectId;
+        apiEvent.objectRef = message.objectRef;
+        apiEvent.object = message.object;
+        
+        // dispatch for event + reference
+        if (apiEvent.objectRef)
+        {
+            try {
+                apiEvent.channelId = "apiEvent-" + apiEvent.type + "_" + apiEvent.objectRef;
+                //console.log("api_event -> " + apiEvent.channelId);
+                io.to(apiEvent.channelId).emit("apiEvent", apiEvent);
+            } catch (e) {
+                console.log(e);
+            }
+        }
+    
+        // dispatch for event
+        try {
+            apiEvent.channelId = "apiEvent-" + apiEvent.type;
+            //console.log("apiEvent -> " + apiEvent.channelId);
+            io.to(apiEvent.channelId).emit("apiEvent", apiEvent);
+        } catch (e) {
+            console.log(e);
+        }
+        
+        done();
+    });
+};
+
+exports.bindSocket = function(socket, provider, io)
+{
+    bindSubscriptions(io);
+
+    socketUtil.bindGitana(socket, function() {
+
+        socket.on("onApiEvent", function(eventName, reference, callback) {
+
+            if (eventName)
+            {
+                var channelId = "apiEvent-" + eventName;
+                
+                if (reference)
+                {
+                    channelId = eventName + "-" + reference;
+                }
+                
+                // join room for this event
+                socket.join(channelId);
+            }
+
+            callback();
+        });
+    });
+};

package/middleware/awareness/plugins/editorial.js

@@ -5,7 +5,7 @@ var socketUtil = require("../../../util/socket");
 
 var request = require("../../../util/request");
 
-exports.bindSocket = function(socket, provider)
+exports.bindSocket = function(socket, provider, io)
 {
     socketUtil.bindGitana(socket, function() {
 
@@ -231,10 +231,11 @@ exports.bindSocket = function(socket, provider)
                 var x1 = cookieValue.indexOf("GITANA_TICKET=");
                 if (x1 > -1)
                 {
-                    var x2 = cookieValue.indexOf(";", x1 + 14);
+                    ticket = cookieValue.substring(x1 + 14);
+                    var x2 = ticket.indexOf(";");
                     if (x2 > -1)
                     {
-                        ticket = cookieValue.substring(x1 + 14, x2);
+                        ticket = ticket.substring(0, x2);
                     }
                 }
             }

package/middleware/awareness/plugins/resources.js

@@ -5,7 +5,7 @@ var socketUtil = require("../../../util/socket");
 
 var subscriptionsBound = false;
 
-var bindSubscriptions = function()
+var bindSubscriptions = function(io)
 {
     if (subscriptionsBound) {
         return;
@@ -43,19 +43,27 @@ var bindSubscriptions = function()
             };
 
             // fire to reference
-            process.IO.to(reference).emit("watchResource", reference, watchObject);
+            try {
+                if (reference) {
+                    io.to(reference).emit("watchResource", reference, watchObject);
+                }
+            } catch (e) { }
 
             // fire to head reference
-            process.IO.to(headReference).emit("watchResource", headReference, watchObject);
+            try {
+                if (headReference) {
+                    io.to(headReference).emit("watchResource", headReference, watchObject);
+                }
+            } catch (e) { }
         }
 
         done();
     });
 };
 
-exports.bindSocket = function(socket, provider)
+exports.bindSocket = function(socket, provider, io)
 {
-    bindSubscriptions();
+    bindSubscriptions(io);
 
     socketUtil.bindGitana(socket, function() {
 

package/middleware/driver-config/driver-config.js

@@ -1,10 +1,4 @@
-var path = require('path');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
-var Gitana = require("gitana");
-
-var fs = require("fs");
 
 /**
  * Retrieves local driver configuration for hosts from Cloud CMS.

package/middleware/modules/modules.js

@@ -55,14 +55,20 @@ exports = module.exports = function()
             });
         }
 
-        if (!moduleConfig.source)
+        var source = moduleConfig.source40;
+        if (!source)
+        {
+            source = moduleConfig.source;
+        }
+
+        if (!source)
         {
             return callback({
                 "message": "Missing module config source settings"
             });
         }
 
-        var sourceType = moduleConfig.source.type;
+        var sourceType = source.type;
         if (!sourceType)
         {
             return callback({
@@ -70,7 +76,7 @@ exports = module.exports = function()
             });
         }
 
-        var sourceUrl = moduleConfig.source.uri;
+        var sourceUrl = source.uri;
         if (!sourceUrl)
         {
             return callback({
@@ -78,12 +84,12 @@ exports = module.exports = function()
             });
         }
 
-        var sourcePath = moduleConfig.source.path;
+        var sourcePath = source.path;
         if (!sourcePath) {
             sourcePath = "/";
         }
 
-        var sourceBranch = moduleConfig.source.branch;
+        var sourceBranch = source.branch;
         if (!sourceBranch) {
             sourceBranch = "master";
         }

package/middleware/perf/perf.js

@@ -194,8 +194,9 @@ exports = module.exports = function()
                             car = filename;
                         }
                         var regex1 = new RegExp("-[0-9a-f]{32}$"); // md5
-                        var regex2 = new RegExp("-[0-9]{13}$"); // timestamp
-                        if (regex1.test(car) || regex2.test(car))
+                        var regex2 = new RegExp("-[0-9]{13}$"); // timestamp?
+                        var regex3 = new RegExp("-[0-9]{10}$"); // epoch millis
+                        if (regex1.test(car) || regex2.test(car) || regex3.test(car))
                         {
                             var x = car.lastIndexOf("-");
 

package/middleware/stores/store.js

@@ -1,4 +1,3 @@
-var request = require('request');
 var path = require('path');
 
 var util = require("../../util/util");

package/middleware/stores/stores.js

@@ -417,7 +417,7 @@ exports = module.exports = function()
                         var moduleStoreType = moduleDescriptors[i].store;
                         var modulePath = moduleDescriptors[i].path;
 
-                        //console.log("Config Store - Module Path: " + modulePath + ", type: " + moduleStoreType);
+                        console.log("Config Store - Module Path: " + modulePath + ", type: " + moduleStoreType);
 
                         var storePath = path.join(modulePath, "config");
                         if (moduleStoreType === "modules")
@@ -520,7 +520,7 @@ exports = module.exports = function()
                         callback();
                     });
                 };
-    
+
                 process.cache.read("module-descriptors-" + host, function(err, moduleDescriptors) {
 
                     moduleDescriptors = null;

package/middleware/virtual-config/virtual-config.js

@@ -1,6 +1,3 @@
-//var path = require('path');
-//var http = require('http');
-//var request = require('request');
 var util = require("../../util/util");
 
 /**
@@ -33,12 +30,12 @@ exports = module.exports = function()
             // force key to "virtualdriver"
             configuration.virtualDriver.key = VIRTUAL_DRIVER_CACHE_KEY;
 
-            console.log("a1: " + JSON.stringify(configuration.virtualDriver, null, 2));
+            //console.log("a1: " + JSON.stringify(configuration.virtualDriver, null, 2));
             
             // either connect anew or re-use an existing connection to Cloud CMS for this application
             Gitana.connect(configuration.virtualDriver, function(err) {
     
-                console.log("a2:" + err);
+                //console.log("a2:" + err);
                 if (err)
                 {
                     return callback(err);

package/middleware/virtual-files/virtual-files.js

@@ -1,6 +1,3 @@
-var path = require('path');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
 
 /**

package/middleware/welcome/welcome.js

@@ -1,7 +1,4 @@
 var path = require('path');
-var fs = require('fs');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
 
 /**

package/notifications/notifications.js

@@ -72,11 +72,11 @@ var handleNotificationMessages = function(items, callback) {
     var fns = [];
     for (var i = 0; i < items.length; i++)
     {
-        var fn = function(item, i) {
+        var fn = function(item, index) {
             return function(done) {
 
                 //logFn("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
-                console.log("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
+                console.log("WORKING ON ITEM: " + index + ", item: " + JSON.stringify(item, null, "  "));
 
                 var operation = item.operation;
 
@@ -540,6 +540,8 @@ var handleNotificationMessages = function(items, callback) {
 
                     var host = determineHost(item);
 
+                    var _fns = [];
+                    
                     var deployments = item.deployments;
                     if (deployments && deployments.length > 0)
                     {
@@ -554,17 +556,77 @@ var handleNotificationMessages = function(items, callback) {
                                 "host": host,
                                 "deployment": deployment
                             };
-
-                            // broadcast event
-                            process.broadcast.publish("deployment_synced", message, function(err) {
-                                if (err) {
-                                    logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                            
+                            var _fn = function(message) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("deployment_synced", message, function(err) {
+                                        if (err) {
+                                            logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                                        }
+                                        return done(err);
+                                    });
                                 }
-                                return done(err);
-                            });
-
+                            }(message);
+                            _fns.push(_fn);
                         }
                     }
+                    
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
+                }
+                else if (operation === "api_event")
+                {
+                    var eventType = item.type;
+                    var eventId = item.id;
+                    var objects = item.objects;
+    
+                    var host = determineHost(item);
+                    
+                    var _fns = [];
+                    if (objects && objects.length > 0)
+                    {
+                        for (var z = 0; z < objects.length; z++)
+                        {
+                            var object = objects[z];
+    
+                            var applicationId = object.applicationId;
+                            var deploymentKey = item.deploymentKey;
+                            
+                            var objectType = object.type; // sidekickMessage
+                            var objectId = object.id;
+                            var objectRef = object.ref;
+    
+                            var publishMessage = {
+                                "applicationId": applicationId,
+                                "deploymentKey": deploymentKey,
+                                "host": host,
+                                "eventType": eventType,
+                                "eventId": eventId,
+                                "objectType": objectType,
+                                "objectId": objectId,
+                                "objectRef": objectRef,
+                                "object": object.object
+                            };
+                            
+                            var _fn = function(publishMessage) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("api_event", publishMessage, function(err) {
+                                        if (err) {
+                                            logInfo("published api_event message. err:" + err + "\nmessage: " + JSON.stringify(publishMessage,null,2));
+                                        }
+                                        return d(err);
+                                    });
+                                }
+                            }(publishMessage);
+                            _fns.push(_fn);
+                        }
+                    }
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
                 }
                 else
                 {

package/notifications/providers/kafka.js

@@ -0,0 +1,182 @@
+const { Kafka } = require("kafkajs");
+const util = require("../../util/util");
+
+var holder = {};
+
+module.exports = {};
+module.exports.start = function(configuration, callback)
+{
+    // clientId
+    if (!configuration.clientId)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID)
+        {
+            configuration.clientId = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID;
+        }
+    }
+    if (!configuration.clientId)
+    {
+        configuration.clientId = "cloudcms-ui-kafka-notifications-client";
+    }
+
+    // topic
+    if (!configuration.topic)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC)
+        {
+            configuration.topic = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC;
+        }
+    }
+    if (!configuration.topic)
+    {
+        configuration.topic = "cloudcms.ui.topic";
+    }
+    
+    // group
+    if (!configuration.group)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP)
+        {
+            configuration.group = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP;
+        }
+    }
+    if (!configuration.group) {
+        configuration.group = "cloudcms-ui-kafka-notifications-group";
+    }
+    
+    // brokers
+    if (!configuration.brokers)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS)
+        {
+            configuration.brokers = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS;
+        }
+    }
+    
+    var clientId = configuration.clientId;
+    var brokers = configuration.brokers.split(",");
+    var topic = configuration.topic;
+    var group = configuration.group;
+
+    process.log("Connecting to kafka, client ID: " + clientId + ", brokers: " + brokers + ", topic: " + topic + ", group: " + group);
+
+    if (holder.consumer)
+    {
+        return callback();
+    }
+    
+    // console.log("a1");
+    var kafka = new Kafka({
+        clientId: clientId,
+        brokers: brokers
+    });
+    // console.log("a2: " + kafka);
+    // console.log("a3: " + kafka.consumer);
+    
+    var consumer = holder.consumer = kafka.consumer({ groupId: group });
+ 
+    // connect
+    (async function() {
+        await consumer.connect()
+    }());
+    
+    (async function() {
+        await consumer.subscribe({topic: topic, fromBeginning: true})
+    }());
+    
+    callback();
+};
+
+module.exports.process = function(callback)
+{
+    var consumer = holder.consumer;
+    if (!consumer)
+    {
+        return callback();
+    }
+    
+    (async function() {
+        await consumer.run({
+            eachMessage: async ({ topic, partition, message, heartbeat, pause }) => {
+                handleMessage(topic, partition, message, heartbeat, pause, callback);
+            }
+        });
+    }());
+};
+
+var handleMessage = function(topic, partition, messageObject, heartbeat, pause, callback)
+{
+    // console.log("Topic: " + topic);
+    // console.log("Partition: " + partition);
+    // console.log("Message: " + JSON.stringify(messageObject, null, 2));
+    
+    if (!messageObject || !messageObject.value)
+    {
+        return;
+    }
+    
+    //var key = message.key.toString();
+    var valueString = messageObject.value.toString();
+    var headers = messageObject.headers || {};
+    
+    var json = null;
+    if (valueString) {
+        json = JSON.parse("" + valueString);
+    }
+    
+    // console.log("b1: " + valueString);
+    // console.log("b2: " + data);
+    
+    if (json)
+    {
+        console.log("JSON: " + JSON.stringify(json));
+        console.log("HEADERS: " + JSON.stringify(headers));
+    
+        /**
+         * kafka1-ui-1  | VAL: {"subject":"invalidate_objects:aea334cb8accb0bd698e","message":"","data":"{\"operation\":\"invalidate_objects\",\"invalidations\":[{\"applicationId\":\"1785c4b13f74b3aa4b31\",\"ref\":\"application://826a3ebefe4c1e006a60/1785c4b13f74b3aa4b31\",\"id\":\"1785c4b13f74b3aa4b31\",\"type\":\"application\",\"stackId\":\"a17879ca0923f6d9696b\",\"stackMembers\":{\"archives\":{\"typeId\":\"vault\",\"id\":\"385e37eb07bc3d16dca1\"},\"console\":{\"typeId\":\"application\",\"id\":\"1785c4b13f74b3aa4b31\"},\"content\":{\"typeId\":\"repository\",\"id\":\"9904550a6b2a2a71f015\"},\"hosting\":{\"typeId\":\"webhost\",\"id\":\"5d2db66ced26608dc355\"},\"oneteam\":{\"typeId\":\"application\",\"id\":\"8d5ba57420ce31a3172c\"},\"principals\":{\"typeId\":\"domain\",\"id\":\"32cbffafe9f2de4d1b9a\"},\"root\":{\"typeId\":\"directory\",\"id\":\"6db38348323d001c2c10\"}}}]}"}
+         * kafka1-ui-1  | HEADERS: [object Object]
+         * @type {*[]}
+         */
+        
+        var subject = json.subject;
+        var message = json.message;
+        
+        // build out the notification message items
+        var items = [];
+    
+        if (json.data)
+        {
+            var data = JSON.parse("" + json.data);
+
+            // timestamp from headers
+            //var timestamp = parseInt(headers.timestamp);
+            var timestamp = new Date().getTime();
+        
+            // copy data as item
+            var item = util.clone(data, true);
+            if (!item)
+            {
+                item = {};
+            }
+        
+            // other properties
+            item.timestamp = timestamp;
+            item.sentTimestamp = timestamp;
+            item.subject = subject;
+        
+            // raw message
+            item.rawMessage = json.data; // string
+        
+            items.push(item);
+        
+            //console.log("ITEM: " + JSON.stringify(item));
+        }
+    }
+    
+    // call back to notifications engine to process these items
+    // when they're done processing, our callback is fired so that we can handle deletes and things
+    // we call the done() method when we're finished
+    callback(null, items, function(err, items, done) {
+        done(err, items, items);
+    });
+};

package/notifications/providers/stomp.js

@@ -63,6 +63,7 @@ module.exports.start = function(configuration, callback)
         var MessageConsumer = function MessageConsumer() { };
         MessageConsumer.prototype.init = function init(done) {
             console.log("STOMP client initializing to host: " + host + ", port: " + port);
+            var self = this;
             var stompClient = new Stomp({
                 "host": host,
                 "port": port,
@@ -85,6 +86,9 @@ module.exports.start = function(configuration, callback)
                 });
 
                 done();
+            }, function(err) {
+                console.error("STOMP Error: " + err);
+                done(err);
             });
         };
         holder.consumer = new MessageConsumer();