cloudcms-server 3.3.1-beta.9 → 4.0.0-beta.10

package/middleware/wcm/wcm.js

@@ -309,7 +309,7 @@ exports = module.exports = function()
                     req.application(function (err, application) {
 
                         var loadingPagesCacheKey = application._doc + "-wcm-loading-pages";
-                        _LOCK(null, loadingPagesCacheKey, function (releaseLockFn) {
+                        _LOCK(null, loadingPagesCacheKey, function (err, releaseLockFn) {
                             
                             // check again inside lock in case another request preloaded this before we arrived
                             req.cache.read(WCM_PAGES, function (err, cachedPages) {
@@ -716,7 +716,7 @@ exports = module.exports = function()
         var cacheFilePath = cloudcms.toCacheFilePath(pageFilePath);
 
         // take out a lock so that only one "request" can write to cache at a time for this path
-        _LOCK(contentStore, _lock_identifier(pageBasePath), function(releaseLockFn) {
+        _LOCK(contentStore, _lock_identifier(pageBasePath), function(err, releaseLockFn) {
 
             // write page file
             contentStore.writeFile(pageFilePath, text, function (err) {
@@ -773,7 +773,7 @@ exports = module.exports = function()
         var cacheFilePath = cloudcms.toCacheFilePath(pageFilePath);
 
         // take out a lock so that only one "request" can read from cache at a time for this path
-        _LOCK(contentStore, _lock_identifier(pageBasePath), function(releaseLockFn) {
+        _LOCK(contentStore, _lock_identifier(pageBasePath), function(err, releaseLockFn) {
 
             contentStore.readFile(cacheFilePath, function(err, cacheInfoString) {
 
@@ -858,7 +858,7 @@ exports = module.exports = function()
 
     var _handleCachePageInvalidate = function(contentStore, pageBasePath, callback)
     {
-        _LOCK(contentStore, _lock_identifier(pageBasePath), function(releaseLockFn) {
+        _LOCK(contentStore, _lock_identifier(pageBasePath), function(err, releaseLockFn) {
 
             contentStore.existsDirectory(pageBasePath, function (exists) {
 

package/middleware/welcome/welcome.js

@@ -1,7 +1,4 @@
 var path = require('path');
-var fs = require('fs');
-var http = require('http');
-var request = require('request');
 var util = require("../../util/util");
 
 /**

package/notifications/notifications.js

@@ -72,11 +72,11 @@ var handleNotificationMessages = function(items, callback) {
     var fns = [];
     for (var i = 0; i < items.length; i++)
     {
-        var fn = function(item, i) {
+        var fn = function(item, index) {
             return function(done) {
 
                 //logFn("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
-                console.log("WORKING ON ITEM: " + i + ", item: " + JSON.stringify(item, null, "  "));
+                console.log("WORKING ON ITEM: " + index + ", item: " + JSON.stringify(item, null, "  "));
 
                 var operation = item.operation;
 
@@ -540,6 +540,8 @@ var handleNotificationMessages = function(items, callback) {
 
                     var host = determineHost(item);
 
+                    var _fns = [];
+                    
                     var deployments = item.deployments;
                     if (deployments && deployments.length > 0)
                     {
@@ -554,17 +556,77 @@ var handleNotificationMessages = function(items, callback) {
                                 "host": host,
                                 "deployment": deployment
                             };
-
-                            // broadcast event
-                            process.broadcast.publish("deployment_synced", message, function(err) {
-                                if (err) {
-                                    logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                            
+                            var _fn = function(message) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("deployment_synced", message, function(err) {
+                                        if (err) {
+                                            logInfo("published deployment_synced message. err:" + err + "\nmessage: " + JSON.stringify(message,null,2));
+                                        }
+                                        return done(err);
+                                    });
                                 }
-                                return done(err);
-                            });
-
+                            }(message);
+                            _fns.push(_fn);
+                        }
+                    }
+                    
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
+                }
+                else if (operation === "api_event")
+                {
+                    var eventType = item.type;
+                    var eventId = item.id;
+                    var objects = item.objects;
+    
+                    var host = determineHost(item);
+                    
+                    var _fns = [];
+                    if (objects && objects.length > 0)
+                    {
+                        for (var z = 0; z < objects.length; z++)
+                        {
+                            var object = objects[z];
+    
+                            var applicationId = object.applicationId;
+                            var deploymentKey = item.deploymentKey;
+                            
+                            var objectType = object.type; // sidekickMessage
+                            var objectId = object.id;
+                            var objectRef = object.ref;
+    
+                            var publishMessage = {
+                                "applicationId": applicationId,
+                                "deploymentKey": deploymentKey,
+                                "host": host,
+                                "eventType": eventType,
+                                "eventId": eventId,
+                                "objectType": objectType,
+                                "objectId": objectId,
+                                "objectRef": objectRef,
+                                "object": object.object
+                            };
+                            
+                            var _fn = function(publishMessage) {
+                                return function(d) {
+                                    // broadcast event
+                                    process.broadcast.publish("api_event", publishMessage, function(err) {
+                                        if (err) {
+                                            logInfo("published api_event message. err:" + err + "\nmessage: " + JSON.stringify(publishMessage,null,2));
+                                        }
+                                        return d(err);
+                                    });
+                                }
+                            }(publishMessage);
+                            _fns.push(_fn);
                         }
                     }
+                    async.parallelLimit(_fns, 4, function(err) {
+                        done(err);
+                    });
                 }
                 else
                 {
@@ -659,7 +721,7 @@ module.exports = function()
     var r = {};
 
     r.start = function(callback) {
-
+    
         var config = process.configuration;
         if (!config["notifications"])
         {
@@ -686,7 +748,7 @@ module.exports = function()
                 notifications.enabled = true;
             }
         }
-
+        
         if (notifications.enabled)
         {
             if (process.env.CLOUDCMS_NOTIFICATIONS_TYPE)

package/notifications/providers/kafka.js

@@ -0,0 +1,182 @@
+const { Kafka } = require("kafkajs");
+const util = require("../../util/util");
+
+var holder = {};
+
+module.exports = {};
+module.exports.start = function(configuration, callback)
+{
+    // clientId
+    if (!configuration.clientId)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID)
+        {
+            configuration.clientId = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_CLIENT_ID;
+        }
+    }
+    if (!configuration.clientId)
+    {
+        configuration.clientId = "cloudcms-ui-kafka-notifications-client";
+    }
+
+    // topic
+    if (!configuration.topic)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC)
+        {
+            configuration.topic = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_TOPIC;
+        }
+    }
+    if (!configuration.topic)
+    {
+        configuration.topic = "cloudcms.ui.topic";
+    }
+    
+    // group
+    if (!configuration.group)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP)
+        {
+            configuration.group = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_GROUP;
+        }
+    }
+    if (!configuration.group) {
+        configuration.group = "cloudcms-ui-kafka-notifications-group";
+    }
+    
+    // brokers
+    if (!configuration.brokers)
+    {
+        if (process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS)
+        {
+            configuration.brokers = process.env.CLOUDCMS_NOTIFICATIONS_KAFKA_BROKERS;
+        }
+    }
+    
+    var clientId = configuration.clientId;
+    var brokers = configuration.brokers.split(",");
+    var topic = configuration.topic;
+    var group = configuration.group;
+
+    process.log("Connecting to kafka, client ID: " + clientId + ", brokers: " + brokers + ", topic: " + topic + ", group: " + group);
+
+    if (holder.consumer)
+    {
+        return callback();
+    }
+    
+    // console.log("a1");
+    var kafka = new Kafka({
+        clientId: clientId,
+        brokers: brokers
+    });
+    // console.log("a2: " + kafka);
+    // console.log("a3: " + kafka.consumer);
+    
+    var consumer = holder.consumer = kafka.consumer({ groupId: group });
+ 
+    // connect
+    (async function() {
+        await consumer.connect()
+    }());
+    
+    (async function() {
+        await consumer.subscribe({topic: topic, fromBeginning: true})
+    }());
+    
+    callback();
+};
+
+module.exports.process = function(callback)
+{
+    var consumer = holder.consumer;
+    if (!consumer)
+    {
+        return callback();
+    }
+    
+    (async function() {
+        await consumer.run({
+            eachMessage: async ({ topic, partition, message, heartbeat, pause }) => {
+                handleMessage(topic, partition, message, heartbeat, pause, callback);
+            }
+        });
+    }());
+};
+
+var handleMessage = function(topic, partition, messageObject, heartbeat, pause, callback)
+{
+    // console.log("Topic: " + topic);
+    // console.log("Partition: " + partition);
+    // console.log("Message: " + JSON.stringify(messageObject, null, 2));
+    
+    if (!messageObject || !messageObject.value)
+    {
+        return;
+    }
+    
+    //var key = message.key.toString();
+    var valueString = messageObject.value.toString();
+    var headers = messageObject.headers || {};
+    
+    var json = null;
+    if (valueString) {
+        json = JSON.parse("" + valueString);
+    }
+    
+    // console.log("b1: " + valueString);
+    // console.log("b2: " + data);
+    
+    if (json)
+    {
+        console.log("JSON: " + JSON.stringify(json));
+        console.log("HEADERS: " + JSON.stringify(headers));
+    
+        /**
+         * kafka1-ui-1  | VAL: {"subject":"invalidate_objects:aea334cb8accb0bd698e","message":"","data":"{\"operation\":\"invalidate_objects\",\"invalidations\":[{\"applicationId\":\"1785c4b13f74b3aa4b31\",\"ref\":\"application://826a3ebefe4c1e006a60/1785c4b13f74b3aa4b31\",\"id\":\"1785c4b13f74b3aa4b31\",\"type\":\"application\",\"stackId\":\"a17879ca0923f6d9696b\",\"stackMembers\":{\"archives\":{\"typeId\":\"vault\",\"id\":\"385e37eb07bc3d16dca1\"},\"console\":{\"typeId\":\"application\",\"id\":\"1785c4b13f74b3aa4b31\"},\"content\":{\"typeId\":\"repository\",\"id\":\"9904550a6b2a2a71f015\"},\"hosting\":{\"typeId\":\"webhost\",\"id\":\"5d2db66ced26608dc355\"},\"oneteam\":{\"typeId\":\"application\",\"id\":\"8d5ba57420ce31a3172c\"},\"principals\":{\"typeId\":\"domain\",\"id\":\"32cbffafe9f2de4d1b9a\"},\"root\":{\"typeId\":\"directory\",\"id\":\"6db38348323d001c2c10\"}}}]}"}
+         * kafka1-ui-1  | HEADERS: [object Object]
+         * @type {*[]}
+         */
+        
+        var subject = json.subject;
+        var message = json.message;
+        
+        // build out the notification message items
+        var items = [];
+    
+        if (json.data)
+        {
+            var data = JSON.parse("" + json.data);
+
+            // timestamp from headers
+            //var timestamp = parseInt(headers.timestamp);
+            var timestamp = new Date().getTime();
+        
+            // copy data as item
+            var item = util.clone(data, true);
+            if (!item)
+            {
+                item = {};
+            }
+        
+            // other properties
+            item.timestamp = timestamp;
+            item.sentTimestamp = timestamp;
+            item.subject = subject;
+        
+            // raw message
+            item.rawMessage = json.data; // string
+        
+            items.push(item);
+        
+            //console.log("ITEM: " + JSON.stringify(item));
+        }
+    }
+    
+    // call back to notifications engine to process these items
+    // when they're done processing, our callback is fired so that we can handle deletes and things
+    // we call the done() method when we're finished
+    callback(null, items, function(err, items, done) {
+        done(err, items, items);
+    });
+};

package/notifications/providers/stomp.js

@@ -63,6 +63,7 @@ module.exports.start = function(configuration, callback)
         var MessageConsumer = function MessageConsumer() { };
         MessageConsumer.prototype.init = function init(done) {
             console.log("STOMP client initializing to host: " + host + ", port: " + port);
+            var self = this;
             var stompClient = new Stomp({
                 "host": host,
                 "port": port,
@@ -85,6 +86,9 @@ module.exports.start = function(configuration, callback)
                 });
 
                 done();
+            }, function(err) {
+                console.error("STOMP Error: " + err);
+                done(err);
             });
         };
         holder.consumer = new MessageConsumer();

package/package.json

@@ -6,56 +6,63 @@
   },
   "name": "cloudcms-server",
   "description": "Cloud CMS Application Server Module",
-  "version": "3.3.1-beta.9",
+  "version": "4.0.0-beta.10",
   "repository": {
     "type": "git",
     "url": "git://github.com/gitana/cloudcms-server.git"
   },
   "dependencies": {
-    "accepts": "^1.3.7",
-    "agentkeepalive": "^4.1.3",
+    "@socket.io/cluster-adapter": "^0.2.0",
+    "@socket.io/redis-adapter": "^7.2.0",
+    "@socket.io/sticky": "^1.0.1",
+    "accepts": "^1.3.8",
+    "agentkeepalive": "^4.5.0",
     "alpaca": "^1.5.27",
     "archiver": "^5.1.0",
-    "async": "^3.2.0",
-    "aws-sdk": "^2.807.0",
-    "axios": "^0.21.0",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.2",
+    "aws-sdk": "^2.1208.0",
+    "axios": "^1.5.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.19.0",
+    "body-parser": "^1.20.0",
     "bytes": "^3.1.0",
     "canoe": "^0.3.3",
     "clone": "^2.1.2",
     "connect-flash": "^0.1.1",
     "connect-multiparty": "^2.2.0",
-    "connect-redis": "^5.0.0",
+    "connect-redis": "^6.1.3",
     "consolidate": "^0.16.0",
     "cookie-parser": "^1.4.5",
     "debug": "^4.3.1",
     "dustjs-helpers": "^1.7.4",
-    "dustjs-linkedin": "^2.7.5",
+    "dustjs-linkedin": "^3.0.1",
     "errorhandler": "^1.5.1",
-    "express": "^4.17.1",
-    "express-session": "^1.17.1",
+    "express": "^4.18.2",
+    "express-session": "^1.17.3",
     "express-useragent": "^1.0.15",
     "extend-with-super": "^2.0.0",
-    "form-data": "^2.3.3",
-    "gitana": "^1.0.320",
+    "fast-proxy": "^2.2.0",
+    "gitana": "^1.0.325",
     "handlebars": "^4.7.6",
     "hbs": "^4.1.1",
-    "http-proxy": "^1.18.1",
+    "helmet": "^4.6.0",
+    "ioredis": "4.28.5",
     "json5": "^2.1.3",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.2",
+    "kafkajs": "^2.2.4",
     "klaw": "^3.0.0",
-    "lru-cache": "^6.0.0",
-    "marked": "^1.2.5",
+    "lru-cache": "^7.14.0",
+    "marked": "^4.0.14",
+    "memored": "^1.1.1",
     "memorystore": "^1.6.4",
     "mime": "^2.4.6",
+    "mkdirp": "^0.5.1",
     "moment": "^2.29.1",
     "morgan": "^1.10.0",
-    "node-redis-pubsub": "^4.0.0",
     "object-hash": "^2.0.3",
     "object-merge": "^2.5.1",
     "on-headers": "^1.0.2",
-    "passport": "^0.4.1",
+    "passport": "^0.7.0",
     "passport-cas": "^0.1.1",
     "passport-facebook": "^3.0.0",
     "passport-github": "^1.1.0",
@@ -63,31 +70,27 @@
     "passport-linkedin": "^1.0.0",
     "passport-local": "^1.0.0",
     "passport-oauth": "^1.0.0",
-    "passport-saml": "^2.0.2",
-    "passport-twitter": "^1.0.4",
+    "@node-saml/passport-saml": "^4.0.4",
+    "passport-twitter": "^0.1.5",
     "pkginfo": "^0.4.1",
-    "q": "^1.5.1",
     "random-js": "^2.1.0",
     "recursive-readdir": "^2.2.2",
-    "redis": "^3.0.2",
-    "redislock": "^1.3.0",
+    "redis": "^4.2.0",
+    "redlock": "4.2.0",
     "request-param": "^1.0.1",
     "response-time": "^2.3.2",
-    "rwlock": "^5.0.0",
-    "semver": "^7.3.4",
+    "semver": "^7.3.7",
     "serve-favicon": "^2.5.0",
     "session-file-store": "^1.5.0",
     "sha1": "^1.1.1",
-    "socket.io": "4.1.2",
+    "socket.io": "^4.5.1",
     "ssl-root-cas": "^1.3.1",
-    "sticky-session": "^1.1.2",
     "stomp-client": "^0.9.0",
     "targz": "^1.0.1",
     "temp": "^0.9.4",
-    "toobusy-js": "^0.5.1",
     "uuid": "^8.3.2",
     "vm2": "^3.9.2",
-    "watch": "^1.0.2",
+    "watch": "^0.13.0",
     "winston": "^3.3.3"
   },
   "contributors": [