cloudcms-server 3.3.1-beta.9 → 4.0.0-beta.1

package/d1/index.js.works

@@ -0,0 +1,203 @@
+var createProxyHandler = function(protocol, hostname, port, pathPrefix)
+{
+    const proxy = require("http2-proxy");
+    const finalhandler = require('finalhandler')
+    
+    const defaultWebHandler = function(err, req, res) {
+        if (err)
+        {
+            console.log("A web proxy error was caught, path: " + req.path + ", err: ", err);
+            try { res.status(500); } catch (e) { }
+            try { res.end('Something went wrong while proxying the request.'); } catch (e) { }
+        }
+        
+        finalhandler(req, res)(err);
+    };
+    
+    // web
+    var webConfig = {};
+    webConfig.hostname = hostname;
+    webConfig.port = port;
+    webConfig.protocol = protocol;
+    //webConfig.path = null;
+    webConfig.timeout = 120000;
+    webConfig.proxyTimeout = 120000;
+    webConfig.proxyName = "Cloud CMS UI Proxy";
+    webConfig.onReq = function(req, options) {
+        
+        if (!options.headers) {
+            options.headers = {};
+        }
+        var headers = options.headers;
+        
+        if (options.path && options.path.startsWith("/proxy")) {
+            options.path = options.path.substring(6);
+        }
+        
+        if (pathPrefix) {
+            options.path = path.join(pathPrefix, options.path);
+        }
+        
+        // used to auto-assign the client header for /oauth/token requests
+        //oauth2.autoProxy(req);
+        
+        // copy domain host into "x-cloudcms-domainhost"
+        if (req.domainHost) {
+            headers["x-cloudcms-domainhost"] = req.domainHost; // this could be "localhost"
+        }
+        
+        // copy virtual host into "x-cloudcms-virtualhost"
+        if (req.virtualHost) {
+            headers["x-cloudcms-virtualhost"] = req.virtualHost; // this could be "root.cloudcms.net" or "abc.cloudcms.net"
+        }
+        
+        // copy deployment descriptor info
+        if (req.descriptor)
+        {
+            if (req.descriptor.tenant)
+            {
+                if (req.descriptor.tenant.id)
+                {
+                    headers["x-cloudcms-tenant-id"] = req.descriptor.tenant.id;
+                }
+                
+                if (req.descriptor.tenant.title)
+                {
+                    headers["x-cloudcms-tenant-title"] = req.descriptor.tenant.title;
+                }
+            }
+            
+            if (req.descriptor.application)
+            {
+                if (req.descriptor.application.id)
+                {
+                    headers["x-cloudcms-application-id"] = req.descriptor.application.id;
+                }
+                
+                if (req.descriptor.application.title)
+                {
+                    headers["x-cloudcms-application-title"] = req.descriptor.application.title;
+                }
+            }
+        }
+        
+        // set optional "x-cloudcms-origin" header
+        var cloudcmsOrigin = null;
+        if (req.virtualHost)
+        {
+            cloudcmsOrigin = req.virtualHost;
+        }
+        if (cloudcmsOrigin)
+        {
+            headers["x-cloudcms-origin"] = cloudcmsOrigin;
+        }
+        
+        // set x-cloudcms-server-version header
+        //headers["x-cloudcms-server-version"] = process.env.CLOUDCMS_APPSERVER_PACKAGE_VERSION;
+        
+        // keep alive
+        //req.headers["connection"] = "keep-alive";
+        
+        // if the incoming request didn't have an "Authorization" header
+        // and we have a logged in Gitana User via Auth, then set authorization header to Bearer Access Token
+        if (!req.headers["authorization"])
+        {
+            if (req.gitana_user)
+            {
+                headers["authorization"] = "Bearer " + req.gitana_user.getDriver().http.accessToken();
+            }
+            else if (req.gitana_proxy_access_token)
+            {
+                headers["authorization"] = "Bearer " + req.gitana_proxy_access_token;
+            }
+        }
+    };
+    webConfig.onRes = function(req, res, proxyRes) {
+        
+        // if (req.gitana_user)
+        // {
+        //     var chunks = [];
+        //
+        //     // triggers on data receive
+        //     proxyRes.on('data', function(chunk) {
+        //         // add received chunk to chunks array
+        //         chunks.push(chunk);
+        //     });
+        //
+        //     proxyRes.on("end", function () {
+        //
+        //         if (proxyRes.statusCode === 401)
+        //         {
+        //             var text = "" + Buffer.concat(chunks);
+        //             if (text && (text.indexOf("invalid_token") > -1) || (text.indexOf("invalid_grant") > -1))
+        //             {
+        //                 var identifier = req.identity_properties.provider_id + "/" + req.identity_properties.user_identifier;
+        //
+        //                 _LOCK([identifier], function(err, releaseLockFn) {
+        //
+        //                     if (err)
+        //                     {
+        //                         // failed to acquire lock
+        //                         console.log("FAILED TO ACQUIRE LOCK", err);
+        //                         req.log("FAILED TO ACQUIRE LOCK", err);
+        //                         try { releaseLockFn(); } catch (e) { }
+        //                         return;
+        //                     }
+        //
+        //                     var cleanup = function (full)
+        //                     {
+        //                         delete Gitana.APPS[req.identity_properties.token];
+        //                         delete Gitana.PLATFORM_CACHE[req.identity_properties.token];
+        //
+        //                         if (full) {
+        //                             auth.removeUserCacheEntry(identifier);
+        //                         }
+        //                     };
+        //
+        //                     // null out the access token
+        //                     // this will force the refresh token to be used to get a new one on the next request
+        //                     req.gitana_user.getDriver().http.refresh(function (err) {
+        //
+        //                         if (err) {
+        //                             cleanup(true);
+        //                             req.log("Invalidated auth state for gitana user: " + req.identity_properties.token);
+        //                             return releaseLockFn();
+        //                         }
+        //
+        //                         req.gitana_user.getDriver().reloadAuthInfo(function () {
+        //                             cleanup(true);
+        //                             req.log("Refreshed token for gitana user: " + req.identity_properties.token);
+        //                             releaseLockFn();
+        //                         });
+        //                     });
+        //                 });
+        //             }
+        //
+        //         }
+        //     });
+        // }
+        
+        //res.setHeader('x-powered-by', 'cloudcms');
+        res.writeHead(proxyRes.statusCode, proxyRes.headers)
+        proxyRes.pipe(res)
+    };
+    
+    var proxyRequestHandler = function(req, res) {
+        proxy.web(req, res, webConfig, function(err, req, res) {
+            defaultWebHandler(err, req, res);
+        });
+    };
+    
+    return proxyRequestHandler;
+};
+
+const http = require("http");
+const server = http.createServer()
+server.listen(3000);
+
+var proxyRequestHandler = createProxyHandler("http", "api.default.svc.cluster.local", 80);
+server.on('request', (req, res) => {
+    req.virtualHost = "mt85.us1.cloudcms.net";
+    
+    proxyRequestHandler(req, res);
+});

package/d1/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "d1",
+  "description": "D1",
+  "version": "0.0.1",
+  "dependencies": {
+    "@socket.io/cluster-adapter": "^0.2.0",
+    "@socket.io/redis-adapter": "^7.2.0",
+    "@socket.io/sticky": "^1.0.1",
+    "accepts": "^1.3.8",
+    "agentkeepalive": "^4.2.1",
+    "alpaca": "^1.5.27",
+    "archiver": "^1.3.0",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.2",
+    "aws-sdk": "^2.1208.0",
+    "basic-auth": "^2.0.1",
+    "body-parser": "^1.20.0",
+    "bytes": "^2.5.0",
+    "canoe": "^0.3.3",
+    "clone": "^2.1.2",
+    "connect-flash": "^0.1.1",
+    "connect-multiparty": "^2.2.0",
+    "connect-redis": "^6.1.3",
+    "consolidate": "^0.14.5",
+    "cookie-parser": "^1.4.4",
+    "debug": "^2.6.9",
+    "dustjs-helpers": "1.7.4",
+    "dustjs-linkedin": "3.0.1",
+    "errorhandler": "^1.5.1",
+    "express": "^4.18.1",
+    "express-session": "^1.17.3",
+    "express-useragent": "^1.0.15",
+    "extend-with-super": "^2.0.0",
+    "finalhandler": "^1.2.0",
+    "gitana": "^1.0.322",
+    "handlebars": "^4.4.2",
+    "hbs": "^4.0.5",
+    "helmet": "^4.6.0",
+    "http2-proxy": "^5.0.53",
+    "ioredis": "4.28.5",
+    "json5": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
+    "klaw": "^1.3.1",
+    "lru-cache": "^4.1.5",
+    "marked": "^4.0.14",
+    "memorystore": "^1.6.1",
+    "mime": "^1.6.0",
+    "mkdirp": "^0.5.1",
+    "moment": "^2.24.0",
+    "morgan": "^1.9.1",
+    "object-hash": "^1.3.1",
+    "object-merge": "^2.5.1",
+    "on-headers": "^1.0.2",
+    "passport": "^0.4.0",
+    "passport-cas": "^0.0.3",
+    "passport-facebook": "^2.1.1",
+    "passport-github": "^1.1.0",
+    "passport-google-oauth": "^1.0.0",
+    "passport-linkedin": "^1.0.0",
+    "passport-local": "^1.0.0",
+    "passport-oauth": "^1.0.0",
+    "passport-saml": "^2.2.0",
+    "passport-twitter": "^0.1.5",
+    "pkginfo": "^0.4.1",
+    "random-js": "^1.0.8",
+    "recursive-readdir": "^2.2.2",
+    "redis": "^4.2.0",
+    "redlock": "4.2.0",
+    "request": "^2.88.0",
+    "request-param": "^1.0.1",
+    "response-time": "^2.3.2",
+    "semver": "^7.3.7",
+    "serve-favicon": "^2.5.0",
+    "session-file-store": "^0.2.2",
+    "sha1": "^1.1.1",
+    "socket.io": "^4.5.1",
+    "ssl-root-cas": "^1.3.1",
+    "stomp-client": "^0.9.0",
+    "targz": "^1.0.1",
+    "temp": "^0.8.3",
+    "uuid": "^3.3.2",
+    "vm2": "^3.8.4",
+    "watch": "^0.13.0",
+    "winston": "^3.3.3"
+  }
+}

package/d1/package.json.works

@@ -0,0 +1,14 @@
+{
+  "author": {
+    "name": "Gitana Software, Inc.",
+    "email": "info@cloudcms.com",
+    "url": "https://www.cloudcms.com"
+  },
+  "name": "d1",
+  "description": "D1",
+  "version": "0.0.1",
+  "dependencies": {
+    "http2-proxy": "^5.0.53",
+    "finalhandler": "^1.2.0"
+  }
+}

package/duster/helpers/sample/nyt.js

@@ -1,3 +1,5 @@
+var request = require("../../util/request");
+
 /**
  * Sample New York Times Events tag for Dust.
  *
@@ -6,7 +8,6 @@
 exports = module.exports = function(app, dust, callback)
 {
     var support = require("../support")(dust);
-    var request = require("../../util/request");
 
     // helper functions
     var isDefined = support.isDefined;

package/framework/controllers.js

@@ -162,10 +162,10 @@ var handleInvalidate = function(req, res)
     // new timestamp
     process.env.CLOUDCMS_APPSERVER_TIMESTAMP = new Date().getTime();
 
-    // update all socket clients
-    process.IO.sockets.emit("timestamp", {
-        "timestamp": process.env.CLOUDCMS_APPSERVER_TIMESTAMP
-    });
+    // // update all socket clients
+    // process.IO.sockets.emit("timestamp", {
+    //     "timestamp": process.env.CLOUDCMS_APPSERVER_TIMESTAMP
+    // });
 
     console.log("Server timestamp regenerated");
 

package/index.js

@@ -34,7 +34,7 @@ process.logInfo = process.log = function(text, level)
 var Gitana = require("gitana");
 
 // default http timeout
-process.defaultHttpTimeoutMs = 120000; // 2 minutes
+process.defaultHttpTimeoutMs = 60000;
 
 if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
 {
@@ -48,27 +48,29 @@ if (process.env.DEFAULT_HTTP_TIMEOUT_MS)
     }
 }
 
+// dns fix for Node 17 +
+// see: https://nodejs.org/api/dns.html#dnssetdefaultresultorderorder
+var dns = require("dns");
+dns.setDefaultResultOrder("ipv4first");
+
 // default agents
 var HttpKeepAliveAgent = require('agentkeepalive');
-const templates = require("./middleware/templates/templates");
 var HttpsKeepAliveAgent = require('agentkeepalive').HttpsAgent;
 http.globalAgent = new HttpKeepAliveAgent({
     keepAlive: true,
-    keepAliveMsecs: 1000,
-    freeSocketTimeout: 30000,
+    keepAliveMsecs: 5000,
+    maxSockets: 16000,
+    maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    maxSockets: 200,
-    maxFreeSockets: 40,
-    rejectUnauthorized: false
+    freeSocketTimeout: 4000
 });
 https.globalAgent = new HttpsKeepAliveAgent({
     keepAlive: true,
     keepAliveMsecs: 1000,
-    freeSocketTimeout: 30000,
+    maxSockets: 16000,
+    maxFreeSockets: 256,
     timeout: process.defaultHttpTimeoutMs,
-    maxSockets: 200,
-    maxFreeSockets: 40,
-    rejectUnauthorized: false
+    freeSocketTimeout: 4000
 });
 
 // disable for now
@@ -127,7 +129,7 @@ exports = module.exports = function()
     // TODO: this is to disable really annoying Express 3.0 deprecated's for multipart() which should hopefully
     // TODO: be resolved soon
     console.warn = function() {};
-
+    
     process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
 
     // assume app-server base path if none provided
@@ -140,6 +142,7 @@ exports = module.exports = function()
     var defaultGitanaProxyScheme = "https";
     var defaultGitanaProxyHost = "api.cloudcms.com";
     var defaultGitanaProxyPort = 443;
+    var defaultGitanaProxyPath = "";
 
     var gitanaJsonPath = path.join(process.env.CLOUDCMS_APPSERVER_BASE_PATH, "gitana.json");
     if (fs.existsSync(gitanaJsonPath))
@@ -151,6 +154,7 @@ exports = module.exports = function()
 
             defaultGitanaProxyHost = parsedUrl.hostname;
             defaultGitanaProxyScheme = parsedUrl.protocol.substring(0, parsedUrl.protocol.length - 1); // remove the :
+            defaultGitanaProxyPath = parsedUrl.path;
 
             if (parsedUrl.port)
             {
@@ -177,10 +181,13 @@ exports = module.exports = function()
     if (!process.env.GITANA_PROXY_PORT) {
         process.env.GITANA_PROXY_PORT = defaultGitanaProxyPort;
     }
-
+    if (!process.env.GITANA_PROXY_PATH) {
+        process.env.GITANA_PROXY_PATH = defaultGitanaProxyPath;
+    }
+    
     if (cluster.isMaster)
     {
-        process.log("Gitana Proxy pointed to: " + util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT));
+        process.log("Gitana Proxy pointed to: " + util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
     }
 
     // all web modules are included by default

package/insight/insight.js

@@ -128,7 +128,7 @@ var doSend = function(callback)
     }
 
     // url over to cloud cms
-    var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT) + "/warehouses/" + warehouseId + "/interactions/_create";
+    var URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH) + "/warehouses/" + warehouseId + "/interactions/_create";
     var requestConfig = {
         "url": URL,
         "qs": {},

package/launchpad/index.js

@@ -1,17 +1,209 @@
-module.exports = function(options)
+const cluster = require("cluster");
+
+// Handle uncaught exceptions...
+process.on('uncaughtException', function(err, source) {
+    // if (err === "read ECONNRESET")
+    // {
+    //     // skip
+    // }
+    // else
+    // {
+        console.log(`Launchpad - process received event 'uncaughtException': ${err}, source: ${source}`);
+        console.log(err.stack);
+    // }
+});
+
+module.exports = function(type, config, options)
 {
-    // allow for override
-    if (process.env.CLOUDCMS_LAUNCHPAD_SETUP) {
-        options.setup = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
+    if (!type) {
+        type = process.env.CLOUDCMS_LAUNCHPAD_SETUP;
     }
-
-    // default to single mode
-    if (!options.setup) {
-        options.setup = "single";
+    if (!type) {
+        type = "single";
+    }
+    
+    process.env.CLOUDCMS_LAUNCHPAD_SETUP = type
+    
+    var launcherFactory = require("./launchers/" + type);
+    if (!launcherFactory)
+    {
+        throw new Error("Cannot find launcher factory: " + type);
+    }
+    
+    var launcher = launcherFactory(config);
+    
+    var reportFn = options.report;
+    if (!reportFn) {
+        options.report = reportFn = function () {};
+    }
+    
+    var completionFn = options.complete;
+    if (!completionFn) {
+        options.complete = completionFn = function(err) {
+            throw new Error(err);
+        };
+    }
+    
+    var fork = true;
+    if (type === "single") {
+        fork = false;
     }
+    
+    var bindToListeningPort = function(app, httpServer)
+    {
+        var httpServerPort = -1;
+        if (app) {
+            httpServerPort = app.get("port");
+        }
+        if (httpServerPort === -1) {
+            httpServerPort = process.env.PORT;
+        }
+        if (httpServerPort === -1) {
+            httpServerPort = 3000;
+        }
+    
+        httpServer.listen(httpServerPort);
+    }
+    
+    var bindSignalHandler = function()
+    {
+        var signal = false;
+        process.on('SIGINT', function() {
+            if (!signal) {
+                signal = true;
+                console.log("-------");
+                console.log("Heard SIGINT - shutting down in 10 seconds...");
+                console.log("-------");
+                setTimeout(function() { process.exit(0); }, 10000);
+            }
+        });
+        process.on('SIGTERM', function() {
+            if (!signal) {
+                signal = true;
+                console.log("-------");
+                console.log("Heard SIGTERM - shutting down in 10 seconds...");
+                console.log("-------");
+                setTimeout(function() { process.exit(0); }, 10000);
+            }
+        });
+    };
+    
+    if (!fork)
+    {
+        bindSignalHandler();
+        
+        return launchWorker(launcher, config, options, function(err, app, httpServer) {
+            
+            if (err) {
+                return completionFn(config, err);
+            }
+    
+            // bind to listening port
+            bindToListeningPort(app, httpServer);
+    
+            reportFn(config);
+            completionFn(config);
+        });
+    }
+    else
+    {
+        // in cluster mode, we have a single master listening to the port which distributes work to the workers
+        
+        if (cluster.isMaster)
+        {
+            bindSignalHandler();
+            
+            return launchMaster(launcher, config, options, function(err, workers, httpServer) {
+    
+                if (err) {
+                    return completionFn(config, err);
+                }
+                
+                //reportFn(config);
+                completionFn(config);
+            });
+        }
+        else
+        {
+            return launchWorker(launcher, config, options, function(err, app, httpServer) {
+    
+                // bind to listening port
+                bindToListeningPort(app, httpServer);
+    
+                completionFn(config, err);
+            });
+        }
+    }
+};
 
-    process.env.CLOUDCMS_LAUNCHPAD_SETUP = options.setup;
+var launchMaster = function(launcher, config, options, done)
+{
+    var createHttpServer = options.createHttpServer;
+    
+    createHttpServer(null, function(err, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+    
+        launcher.startCluster(httpServer, function(err) {
+    
+            if (err) {
+                return done(err);
+            }
+            
+            launcher.afterStartCluster(httpServer, function(err, workers) {
+                console.log("LaunchPad started Master: " + process.pid);
+                done(err, workers, httpServer);
+            });
+        });
+    });
+};
+
+var launchWorker = function(launcher, config, options, done)
+{
+    var startServer = options.startServer;
+    
+    var configureServer = options.configureServer;
+    if (!configureServer) {
+        options.configureServer = configureServer = function(config, app, httpServer, done) {
+            done();
+        }
+    }
+    
+    startServer(config, function(err, app, httpServer) {
+        
+        if (err) {
+            return done(err);
+        }
+        
+        launcher.afterStartServer(app, httpServer, function(err) {
+            
+            if (err) {
+                return done(err);
+            }
+    
+            configureServer(config, app, httpServer, function(err) {
+                
+                if (err) {
+                    return done(err);
+                }
+    
+                // if we are on a worker process, then inform the master that we completed
+                if (process.send) {
+                    process.send("worker-startup");
+                }
+    
+                if (process.env["CLUSTER_REPORT"])
+                {
+                    var reportFn = options.report;
+                    reportFn(config);
+                }
 
-    var launch = require("./launchers/" + options.setup);
-    launch(options);
+                console.log("LaunchPad started Worker: " + process.pid);
+                
+                done(null, app, httpServer);
+            });
+        });
+    });
 };