cloudcms-server 4.0.0-beta.2 → 4.0.0-beta.21

package/notifications/notifications.js

@@ -72,11 +72,11 @@ var handleNotificationMessages = function(items, callback) {
     var fns = [];
     for (var i = 0; i < items.length; i++)
     {
-        var fn = function(item, i) {
+        var fn = function(item, index) {
             return function(done) {
 
                 //logFn("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
-                console.log("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
+                console.log("WORKING ON ITEM: " + index + ", item: " + JSON.stringify(item, null, "  "));
 
                 var operation = item.operation;
 
@@ -540,6 +540,8 @@ var handleNotificationMessages = function(items, callback) {
 
                     var host = determineHost(item);
 
+                    var _fns = [];
+                    
                     var deployments = item.deployments;
                     if (deployments && deployments.length > 0)
                     {
@@ -554,17 +556,77 @@ var handleNotificationMessages = function(items, callback) {
                                 "host": host,
                                 "deployment": deployment
                             };
-
-                            // broadcast event
-                            process.broadcast.publish("deployment_synced", message, function(err) {
-                                if (err) {
-                                    logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                            
+                            var _fn = function(message) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("deployment_synced", message, function(err) {
+                                        if (err) {
+                                            logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                                        }
+                                        return done(err);
+                                    });
                                 }
-                                return done(err);
-                            });
-
+                            }(message);
+                            _fns.push(_fn);
                         }
                     }
+                    
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
+                }
+                else if (operation === "api_event")
+                {
+                    var eventType = item.type;
+                    var eventId = item.id;
+                    var objects = item.objects;
+    
+                    var host = determineHost(item);
+                    
+                    var _fns = [];
+                    if (objects && objects.length > 0)
+                    {
+                        for (var z = 0; z < objects.length; z++)
+                        {
+                            var object = objects[z];
+    
+                            var applicationId = object.applicationId;
+                            var deploymentKey = item.deploymentKey;
+                            
+                            var objectType = object.type; // sidekickMessage
+                            var objectId = object.id;
+                            var objectRef = object.ref;
+    
+                            var publishMessage = {
+                                "applicationId": applicationId,
+                                "deploymentKey": deploymentKey,
+                                "host": host,
+                                "eventType": eventType,
+                                "eventId": eventId,
+                                "objectType": objectType,
+                                "objectId": objectId,
+                                "objectRef": objectRef,
+                                "object": object.object
+                            };
+                            
+                            var _fn = function(publishMessage) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("api_event", publishMessage, function(err) {
+                                        if (err) {
+                                            logInfo("published api_event message. err:" + err + "\nmessage: " + JSON.stringify(publishMessage,null,2));
+                                        }
+                                        return d(err);
+                                    });
+                                }
+                            }(publishMessage);
+                            _fns.push(_fn);
+                        }
+                    }
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
                 }
                 else
                 {

package/notifications/providers/kafka.js

@@ -0,0 +1,182 @@
+const { Kafka } = require("kafkajs");
+const util = require("../../util/util");
+
+var holder = {};
+
+module.exports = {};
+module.exports.start = function(configuration, callback)
+{
+    // clientId
+    if (!configuration.clientId)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID)
+        {
+            configuration.clientId = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID;
+        }
+    }
+    if (!configuration.clientId)
+    {
+        configuration.clientId = "cloudcms-ui-kafka-notifications-client";
+    }
+
+    // topic
+    if (!configuration.topic)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC)
+        {
+            configuration.topic = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC;
+        }
+    }
+    if (!configuration.topic)
+    {
+        configuration.topic = "cloudcms.ui.topic";
+    }
+    
+    // group
+    if (!configuration.group)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP)
+        {
+            configuration.group = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP;
+        }
+    }
+    if (!configuration.group) {
+        configuration.group = "cloudcms-ui-kafka-notifications-group";
+    }
+    
+    // brokers
+    if (!configuration.brokers)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS)
+        {
+            configuration.brokers = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS;
+        }
+    }
+    
+    var clientId = configuration.clientId;
+    var brokers = configuration.brokers.split(",");
+    var topic = configuration.topic;
+    var group = configuration.group;
+
+    process.log("Connecting to kafka, client ID: " + clientId + ", brokers: " + brokers + ", topic: " + topic + ", group: " + group);
+
+    if (holder.consumer)
+    {
+        return callback();
+    }
+    
+    // console.log("a1");
+    var kafka = new Kafka({
+        clientId: clientId,
+        brokers: brokers
+    });
+    // console.log("a2: " + kafka);
+    // console.log("a3: " + kafka.consumer);
+    
+    var consumer = holder.consumer = kafka.consumer({ groupId: group });
+ 
+    // connect
+    (async function() {
+        await consumer.connect()
+    }());
+    
+    (async function() {
+        await consumer.subscribe({topic: topic, fromBeginning: true})
+    }());
+    
+    callback();
+};
+
+module.exports.process = function(callback)
+{
+    var consumer = holder.consumer;
+    if (!consumer)
+    {
+        return callback();
+    }
+    
+    (async function() {
+        await consumer.run({
+            eachMessage: async ({ topic, partition, message, heartbeat, pause }) => {
+                handleMessage(topic, partition, message, heartbeat, pause, callback);
+            }
+        });
+    }());
+};
+
+var handleMessage = function(topic, partition, messageObject, heartbeat, pause, callback)
+{
+    // console.log("Topic: " + topic);
+    // console.log("Partition: " + partition);
+    // console.log("Message: " + JSON.stringify(messageObject, null, 2));
+    
+    if (!messageObject || !messageObject.value)
+    {
+        return;
+    }
+    
+    //var key = message.key.toString();
+    var valueString = messageObject.value.toString();
+    var headers = messageObject.headers || {};
+    
+    var json = null;
+    if (valueString) {
+        json = JSON.parse("" + valueString);
+    }
+    
+    // console.log("b1: " + valueString);
+    // console.log("b2: " + data);
+    
+    if (json)
+    {
+        console.log("JSON: " + JSON.stringify(json));
+        console.log("HEADERS: " + JSON.stringify(headers));
+    
+        /**
+         * kafka1-ui-1  | VAL: {"subject":"invalidate_objects:aea334cb8accb0bd698e","message":"","data":"{\"operation\":\"invalidate_objects\",\"invalidations\":[{\"applicationId\":\"1785c4b13f74b3aa4b31\",\"ref\":\"application://826a3ebefe4c1e006a60/1785c4b13f74b3aa4b31\",\"id\":\"1785c4b13f74b3aa4b31\",\"type\":\"application\",\"stackId\":\"a17879ca0923f6d9696b\",\"stackMembers\":{\"archives\":{\"typeId\":\"vault\",\"id\":\"385e37eb07bc3d16dca1\"},\"console\":{\"typeId\":\"application\",\"id\":\"1785c4b13f74b3aa4b31\"},\"content\":{\"typeId\":\"repository\",\"id\":\"9904550a6b2a2a71f015\"},\"hosting\":{\"typeId\":\"webhost\",\"id\":\"5d2db66ced26608dc355\"},\"oneteam\":{\"typeId\":\"application\",\"id\":\"8d5ba57420ce31a3172c\"},\"principals\":{\"typeId\":\"domain\",\"id\":\"32cbffafe9f2de4d1b9a\"},\"root\":{\"typeId\":\"directory\",\"id\":\"6db38348323d001c2c10\"}}}]}"}
+         * kafka1-ui-1  | HEADERS: [object Object]
+         * @type {*[]}
+         */
+        
+        var subject = json.subject;
+        var message = json.message;
+        
+        // build out the notification message items
+        var items = [];
+    
+        if (json.data)
+        {
+            var data = JSON.parse("" + json.data);
+
+            // timestamp from headers
+            //var timestamp = parseInt(headers.timestamp);
+            var timestamp = new Date().getTime();
+        
+            // copy data as item
+            var item = util.clone(data, true);
+            if (!item)
+            {
+                item = {};
+            }
+        
+            // other properties
+            item.timestamp = timestamp;
+            item.sentTimestamp = timestamp;
+            item.subject = subject;
+        
+            // raw message
+            item.rawMessage = json.data; // string
+        
+            items.push(item);
+        
+            //console.log("ITEM: " + JSON.stringify(item));
+        }
+    }
+    
+    // call back to notifications engine to process these items
+    // when they're done processing, our callback is fired so that we can handle deletes and things
+    // we call the done() method when we're finished
+    callback(null, items, function(err, items, done) {
+        done(err, items, items);
+    });
+};

package/notifications/providers/stomp.js

@@ -63,6 +63,7 @@ module.exports.start = function(configuration, callback)
         var MessageConsumer = function MessageConsumer() { };
         MessageConsumer.prototype.init = function init(done) {
             console.log("STOMP client initializing to host: " + host + ", port: " + port);
+            var self = this;
             var stompClient = new Stomp({
                 "host": host,
                 "port": port,
@@ -85,6 +86,9 @@ module.exports.start = function(configuration, callback)
                 });
 
                 done();
+            }, function(err) {
+                console.error("STOMP Error: " + err);
+                done(err);
             });
         };
         holder.consumer = new MessageConsumer();

package/package.json

@@ -1,107 +1,91 @@
 {
   "author": {
     "name": "Gitana Software, Inc.",
-    "email": "info@cloudcms.com",
-    "url": "https://www.cloudcms.com"
+    "email": "info@gitana.io",
+    "url": "https://gitana.io"
   },
   "name": "cloudcms-server",
   "description": "Cloud CMS Application Server Module",
-  "version": "4.0.0-beta.2",
+  "version": "4.0.0-beta.21",
   "repository": {
     "type": "git",
     "url": "git://github.com/gitana/cloudcms-server.git"
   },
   "dependencies": {
-    "@socket.io/cluster-adapter": "^0.2.0",
-    "@socket.io/redis-adapter": "^7.2.0",
-    "@socket.io/sticky": "^1.0.1",
+    "@socket.io/cluster-adapter": "^0.2.2",
+    "@socket.io/redis-adapter": "^8.3.0",
+    "@socket.io/sticky": "^1.0.4",
     "accepts": "^1.3.8",
-    "agentkeepalive": "^4.2.1",
+    "agentkeepalive": "^4.5.0",
     "alpaca": "^1.5.27",
-    "archiver": "^5.1.0",
-    "async": "^3.2.3",
-    "async-lock": "^1.3.2",
-    "aws-sdk": "^2.1208.0",
-    "axios": "^0.21.0",
+    "archiver": "^7.0.1",
+    "async": "^3.2.6",
+    "async-lock": "^1.4.1",
+    "aws-sdk": "^2.1692.0",
+    "axios": "^1.7.9",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.20.0",
-    "bytes": "^3.1.0",
+    "body-parser": "^1.20.3",
+    "bytes": "^3.1.2",
+    "cacheable-lookup": "^6.1.0",
     "canoe": "^0.3.3",
     "clone": "^2.1.2",
     "connect-flash": "^0.1.1",
     "connect-multiparty": "^2.2.0",
-    "connect-redis": "^6.1.3",
-    "consolidate": "^0.16.0",
+    "connect-redis": "^8.0.1",
     "cookie-parser": "^1.4.5",
     "debug": "^4.3.1",
-    "dustjs-helpers": "^1.7.4",
-    "dustjs-linkedin": "^3.0.1",
     "errorhandler": "^1.5.1",
-    "express": "^4.18.1",
-    "express-session": "^1.17.3",
+    "express": "^4.21.2",
+    "express-session": "^1.18.1",
     "express-useragent": "^1.0.15",
-    "extend-with-super": "^2.0.0",
-    "finalhandler": "^1.2.0",
-    "form-data": "^2.3.3",
-    "gitana": "^1.0.322",
-    "handlebars": "^4.7.6",
-    "hbs": "^4.1.1",
-    "helmet": "^4.6.0",
-    "http2-proxy": "^5.0.53",
-    "ioredis": "4.28.5",
-    "json5": "^2.1.3",
-    "jsonwebtoken": "^8.5.1",
-    "klaw": "^3.0.0",
-    "lru-cache": "^6.0.0",
-    "marked": "^4.0.14",
+    "fast-proxy": "^2.2.0",
+    "gitana": "^1.0.331",
+    "handlebars": "^4.7.8",
+    "hbs": "^4.2.0",
+    "helmet": "^8.0.0",
+    "ioredis": "5.4.1",
+    "json5": "^2.2.3",
+    "jsonwebtoken": "^9.0.2",
+    "kafkajs": "^2.2.4",
+    "klaw": "^4.1.0",
+    "lru-cache": "^7.14.0",
+    "marked": "^15.0.4",
     "memored": "^1.1.1",
     "memorystore": "^1.6.4",
     "mime": "^2.4.6",
-    "mkdirp": "^0.5.1",
-    "moment": "^2.29.1",
+    "moment": "^2.30.1",
     "morgan": "^1.10.0",
-    "object-hash": "^2.0.3",
+    "object-hash": "^3.0.0",
     "object-merge": "^2.5.1",
     "on-headers": "^1.0.2",
-    "passport": "^0.4.1",
-    "passport-cas": "^0.1.1",
-    "passport-facebook": "^3.0.0",
-    "passport-github": "^1.1.0",
+    "passport": "^0.7.0",
     "passport-google-oauth": "^2.0.0",
-    "passport-linkedin": "^1.0.0",
     "passport-local": "^1.0.0",
     "passport-oauth": "^1.0.0",
-    "passport-saml": "^2.2.0",
-    "passport-twitter": "^0.1.5",
+    "@node-saml/passport-saml": "5.0.0",
     "pkginfo": "^0.4.1",
     "random-js": "^2.1.0",
-    "recursive-readdir": "^2.2.2",
-    "redis": "^4.2.0",
+    "recursive-readdir": "^2.2.3",
+    "redis": "^4.7.0",
     "redlock": "4.2.0",
     "request-param": "^1.0.1",
-    "response-time": "^2.3.2",
-    "semver": "^7.3.7",
+    "response-time": "^2.3.3",
+    "semver": "^7.6.3",
     "serve-favicon": "^2.5.0",
     "session-file-store": "^1.5.0",
     "sha1": "^1.1.1",
-    "socket.io": "^4.5.1",
+    "socket.io": "^4.8.1",
     "ssl-root-cas": "^1.3.1",
     "stomp-client": "^0.9.0",
     "targz": "^1.0.1",
     "temp": "^0.9.4",
-    "uuid": "^8.3.2",
-    "vm2": "^3.9.2",
-    "watch": "^0.13.0",
+    "uuid": "^11.0.3",
     "winston": "^3.3.3"
   },
   "contributors": [
     {
       "name": "Michael Uzquiano",
       "email": "uzi@cloudcms.com"
-    },
-    {
-      "name": "Harry Moore",
-      "email": "harry.moore@cloudcms.com"
     }
   ],
   "keywords": [