cob-cli 2.53.0-beta-3 → 2.54.0-beta-1

package/lib/commands/updateFromServer.js

@@ -27,7 +27,7 @@ async function updateFromServer(args) {
 
             console.log("\nOk to proceed. Getting files from server's live directories...");
             await cmdEnv.applyCurrentCommandEnvironmentChanges()
-            let changes = await copyFiles(cmdEnv, "serverLive", "localCopy", args)
+            let changes = (await copyFiles(cmdEnv, "serverLive", "localCopy", args)).changes
             await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
 
             if (changes.length == 0) {
@@ -91,7 +91,7 @@ async function updateFromServer(args) {
                 const dataPath = `others/solutionsData/${solution.name}`
                 fs.mkdir(dataPath, { recursive: true })
 
-                if (solution.permissions) {
+                if (solution.groups) {
                     await getGroupsFromServer(
                         cmdEnv.servername,
                         solution.groups,

package/lib/task_lists/common_releaseManager.js

@@ -49,7 +49,7 @@ async function registerRelease(cmdEnv) {
         // Add deploy tags to repo
         await git().fetch(["--tags","-f"]) // Apenas para ter certeza que temos todas as tags
         try {
-            await git().silent(true).tag([deploySignature, "-d"])
+            await git().tag([deploySignature, "-d"])
         } catch {}
 
         await git().addAnnotatedTag(deploySignature, newDeployText )
@@ -86,3 +86,11 @@ function getCurrentBranch() {
     return git().revparse(["--abbrev-ref", "HEAD"]);
 }
 exports.getCurrentBranch = getCurrentBranch;
+
+/* ************************************ */
+async function appendToDeployLog(server, output) {
+    await execa('ssh', [...getSshArgs(server),
+        `{ echo $'${output.replace(/'/g, "\\'")}' | sed "s/^/\\t /"; } >> ${SERVER_CHANGELOG}`
+    ]);
+}
+exports.appendToDeployLog = appendToDeployLog;

package/lib/task_lists/common_syncFiles.js

@@ -2,7 +2,7 @@ const execa = require('execa');
 const path = require('path');
 const fs = require('fs-extra');
 const { getKeypress } = require("../task_lists/common_helpers");
-const { getRsyncSsh, getServerAddress } = require("./common_helpers");
+const { getRsyncSsh, getServerAddress, getSshArgs } = require("./common_helpers");
 
 /* ************************************ */
 const COB_PRODUCTS = ["recordm", "integrationm", "recordm-importer", "reportm", "userm", "confm", "others", "authm", "public"];
@@ -19,13 +19,14 @@ exports.copyFiles = copyFiles;
 
 /* ************************************ */
 async function testEquality(cmdEnv, from, to, args) {
-    return (await _syncFiles(_syncFiles.TEST, cmdEnv, from, to, [], args)).filter( f => !f.endsWith("/"))
+    return (await _syncFiles(_syncFiles.TEST, cmdEnv, from, to, [], args)).changes.filter( f => !f.endsWith("/"))
 }
 exports.testEquality = testEquality;
 
 /* ************************************ */
 async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], args = []) {
     let changes = [];
+    let encFiles = [];
     let products = cmdEnv.products.length ? cmdEnv.products : COB_PRODUCTS;
     // we need to recheck, because it can exist in head but not in the commit we're testing
     let hasRsyncFilter = cmdEnv.rsyncFilter && fs.lstatSync(cmdEnv.rsyncFilter, {throwIfNoEntry: false})?.isFile();
@@ -46,6 +47,7 @@ async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], ar
             while (count < RSYNC_RETRIES) {
                 if (args.verbose > 2)
                     console.log("rsync".bold +" '" + (product).blue + "': attempt " + (count + 1));
+                const protectFilters = _getDecryptedEncFilePaths(product).map(p => `--filter='P ${p}'`);
                 const rsyncArgs = [
                     fromPath,
                     toPath,
@@ -58,6 +60,7 @@ async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], ar
                     "--filter='merge " + path.resolve(__dirname,"rsyncFilter-pre.txt") + "'",
                     hasRsyncFilter ? "--filter='merge " + cmdEnv.rsyncFilter + "'" : "",
                     "--filter='merge " + path.resolve(__dirname,"rsyncFilter-post.txt") + "'",
+                    ...protectFilters,
                     executionType == _syncFiles.COPY ? "-v" : "--dry-run"
                 ].concat(productExtraOptions)
                 if (args.verbose > 3) {
@@ -71,12 +74,17 @@ async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], ar
                         if (args.verbose > 3)
                             console.log("rsync".bold +" '" + (product).blue + "': " + "success".green + " " + value.stdout);
 
-                        let result = _formatRsyncOutput(product, value.stdout);
+                        let rsyncOutput = value.stdout;
+                        if (executionType == _syncFiles.TEST || executionType == _syncFiles.DIFF)
+                            rsyncOutput = _filterDecryptedEncFiles(product, rsyncOutput);
+                        let result = _formatRsyncOutput(product, rsyncOutput);
                         if (args.verbose > 2)
                             console.log("rsync '" + product + "': success " + result);
                         else if (args.verbose > 1)
                             console.log("rsync '" + product + "': success ");
                         changes = changes.concat(result);
+                        if (executionType == _syncFiles.COPY)
+                            encFiles = encFiles.concat(_extractEncFilePaths(product, value.stdout));
                         count = RSYNC_RETRIES;
 
                         resolve();
@@ -120,7 +128,41 @@ async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], ar
         });
     });
     await Promise.all(requests).catch(err => { throw new Error(err.message); });
-    return changes;
+    return { changes, encFiles };
+}
+
+/* ************************************ */
+function _filterDecryptedEncFiles(product, rsyncOutput) {
+    return rsyncOutput
+        .split("\n")
+        .filter(line => {
+            // ignore decrypted files:
+            // * marked as deleted on local -> server
+            // * marked as added on server -> local
+            if (line.startsWith("*deleting ") || /^>f\+/.test(line)) {
+                const relativePath = line.split(/\s+/)[1];
+                if (!relativePath) return true;
+                return !fs.existsSync("./" + product + "/" + relativePath + ".enc");
+            }
+            return true;
+        })
+        .join("\n");
+}
+
+/* ************************************ */
+function _getDecryptedEncFilePaths(product) {
+    const productDir = "./" + product;
+    if (!fs.existsSync(productDir)) return [];
+    const paths = [];
+    const walk = (dir, rel) => {
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+            const entryRel = rel ? rel + "/" + entry.name : entry.name;
+            if (entry.isDirectory()) walk(path.join(dir, entry.name), entryRel);
+            else if (entry.name.endsWith(".enc")) paths.push("/" + entryRel.slice(0, -4));
+        }
+    };
+    walk(productDir, "");
+    return paths;
 }
 
 /* ************************************ */
@@ -147,7 +189,7 @@ function _formatRsyncOutput(product, rsyncOutput) {
 /* ************************************ */
 async function confirmExecutionOfChanges(cmdEnv, extraOptions = []) {
     console.log("\nChecking changes... ")
-    let changes = (await _syncFiles(_syncFiles.DIFF, cmdEnv,  "localCopy", "serverLive", extraOptions)).filter( f => !f.endsWith("/") )
+    let changes = (await _syncFiles(_syncFiles.DIFF, cmdEnv,  "localCopy", "serverLive", extraOptions)).changes.filter( f => !f.endsWith("/") )
     if (changes.length) {
         console.log("\nChanges that will be done in " + cmdEnv.serverStr + ":");
         console.log(" " + changes.join("\n "));
@@ -163,6 +205,19 @@ async function confirmExecutionOfChanges(cmdEnv, extraOptions = []) {
 }
 exports.confirmExecutionOfChanges = confirmExecutionOfChanges;
 
+/* ************************************ */
+function _getServerLiveBasePath(product) {
+    switch (product) {
+        case "public":
+            return "/usr/share/cob/"
+        case "recordm-importer":
+        case "others":
+            return "/opt/" + product + "/";
+        default:
+            return "/etc/" + product + "/";
+    }
+}
+
 /* ************************************ */
 function resolveCobPath(server, type, product) {
     switch (type) {
@@ -173,18 +228,47 @@ function resolveCobPath(server, type, product) {
             return getServerAddress(server) + ":/opt/" + product + "/etc.default/";
 
         case "serverLive":
-            switch (product) {
-                case "public":
-                    return server + ":/usr/share/cob/"
-                case "recordm-importer":
-                case "others":
-                    return getServerAddress(server) + ':' + "/opt/" + product + "/";
-                default:
-                    return getServerAddress(server) + ':' + "/etc/" + product + "/";
-            }
+            return getServerAddress(server) + ':' + _getServerLiveBasePath(product);
 
         case "staging":
             return ".staging/" + product + "/";
     }
 }
 exports.resolveCobPath = resolveCobPath;
+
+/* ************************************ */
+function _extractEncFilePaths(product, rsyncOutput) {
+    const basePath = _getServerLiveBasePath(product);
+    return rsyncOutput
+        .split("\n")
+        .slice(1, -3)
+        .filter(line => /^[<>]f/.test(line))
+        .map(line => line.split(/\s+/)[1])
+        .filter(filePath => filePath && filePath.endsWith(".enc"))
+        .map(filePath => basePath + filePath);
+}
+
+/* ************************************ */
+async function decryptEncFile(server, serverFilePath) {
+    await execa('ssh', [...getSshArgs(server), `sudo -u cob-secrets /usr/local/bin/cob-decrypt '${serverFilePath}'`]);
+}
+exports.decryptEncFile = decryptEncFile;
+
+/* ************************************ */
+async function decryptEncFiles(cmdEnv, encFiles) {
+    for (const f of encFiles) {
+        await decryptEncFile(cmdEnv.server, f);
+    }
+}
+exports.decryptEncFiles = decryptEncFiles;
+
+/* ************************************ */
+async function deploySolutions(server) {
+    const result = await execa('ssh', [...getSshArgs(server),
+        '/usr/local/bin/cc-install solutions'
+    ], { reject: false, all: true });
+
+    return { output: result.all, exitCode: result.exitCode };
+}
+exports.deploySolutions = deploySolutions;
+

package/lib/task_lists/deploy_execute.js

@@ -1,21 +1,30 @@
 const Listr = require('listr');
 const UpdaterRenderer = require('listr-update-renderer');
 const verboseRenderer = require('listr-verbose-renderer');
+const fs = require('fs');
 const git = require('simple-git');
-const { registerRelease } = require("./common_releaseManager");
-const { copyFiles } = require("./common_syncFiles");
+const { registerRelease, appendToDeployLog } = require("./common_releaseManager");
+const { copyFiles, decryptEncFiles, deploySolutions } = require("./common_syncFiles");
 
 
 function executeTasks(cmdEnv, args) {
     console.log("\nDeploying ...");
+    let encFiles = [];
+    let ccInstallResult = {};
 
     return new Listr([
         {title: "Register release in DEPLOYLOG.md",           task: () => registerRelease(cmdEnv)},
         {title: `git push (branch: ${cmdEnv.currentBranch})`, task: () => git().push("origin", cmdEnv.currentBranch, {"--set-upstream": null})},
-        {title: "Apply new enviroment specifics",             task: () => cmdEnv.applyCurrentCommandEnvironmentChanges() },        
-        {title: "Deploy files to server's live directories",  task: () => copyFiles(cmdEnv, "localCopy", "serverLive")},
+        {title: "Apply new enviroment specifics",             task: () => cmdEnv.applyCurrentCommandEnvironmentChanges() },
+        {title: "Deploy files to server's live directories",  task: async () => { const result = await copyFiles(cmdEnv, "localCopy", "serverLive"); encFiles = result.encFiles; }},
+        {title: "Decrypt .enc files",                         skip: () => !encFiles.length, task: () => decryptEncFiles(cmdEnv, encFiles)},
+        {title: "Deploy data",                                skip: () => !fs.existsSync('solutions.yml'), task: async () => {
+            ccInstallResult = await deploySolutions(cmdEnv.server);
+            if (ccInstallResult.exitCode !== 0) throw new Error(`Deploying data failed:\n${ccInstallResult.output}`);
+        }},
+        {title: "Register solutions deploy output",           skip: () => !fs.existsSync('solutions.yml'), task: () => appendToDeployLog(cmdEnv.server, ccInstallResult.output)},
         {title: "Set last environment deployed",              task: () => cmdEnv.setLastEnvironmentDeployed()},
-        {title: "Undo new enviroment specifics",              task: () => cmdEnv.unApplyCurrentCommandEnvironmentChanges() },        
+        {title: "Undo new enviroment specifics",              task: () => cmdEnv.unApplyCurrentCommandEnvironmentChanges() },
     ], {
         renderer: args.verbose ? verboseRenderer : UpdaterRenderer,
         collapse: false

package/lib/task_lists/deploy_validate.js

@@ -12,11 +12,11 @@ const { checkNoTestsRunningOnServer } = require("./test_otherFilesContiousReload
 function validateDeployConditions(cmdEnv, args) {
     return new Listr([
         {   title: "Check connectivity and permissions".bold,                                                           task: ()     => checkConnectivity(cmdEnv.server) },
-        {   title: "Check git status".bold,                                                                             task: ()     => checkWorkingCopyCleanliness(false) }, 
+        {   title: "Check git status".bold,                                                                             task: ()     => checkWorkingCopyCleanliness(false) },
         {   title: "Check there's no 'cob-cli test' running on server".bold,                                            task: ()     => checkNoTestsRunningOnServer(cmdEnv.server) },
-        {   title: "find out branch-for-HEAD",                                 skip: ()  => args.force,                 task: (ctx)  => getCurrentBranch().then( currentBranch => ctx.currentBranch = currentBranch ) }, 
-        {   title: "find out SHA for last-deploy on specified server",         skip: ()  => args.force,                 task: (ctx)  => getLastDeployedSha(cmdEnv.server).then(lastSha => _handleGetLastDeployedSha(ctx, cmdEnv, lastSha)) }, 
-        {   title: "git checkout SHA-for-last-deploy",                         skip: ctx => args.force || !ctx.lastSha, task: (ctx)  => git().checkout(ctx.lastSha) }, 
+        {   title: "find out branch-for-HEAD",                                 skip: ()  => args.force,                 task: (ctx)  => getCurrentBranch().then( currentBranch => ctx.currentBranch = currentBranch ) },
+        {   title: "find out SHA for last-deploy on specified server",         skip: ()  => args.force,                 task: (ctx)  => getLastDeployedSha(cmdEnv.server).then(lastSha => _handleGetLastDeployedSha(ctx, cmdEnv, lastSha)) },
+        {   title: "git checkout SHA-for-last-deploy",                         skip: ctx => args.force || !ctx.lastSha, task: (ctx)  => git().checkout(ctx.lastSha) },
         {   title: "Apply last enviroment specifics".bold,                     skip: ctx => args.force || !ctx.lastSha, task: ()     => cmdEnv.applyLastEnvironmentDeployedToServerChanges() },
         {   title: "Check last-deploy == serverLive".bold,                     skip: ctx => args.force || !ctx.lastSha, task: (ctx)  => testEquality(cmdEnv, "localCopy", "serverLive", args).then( changes => _handleTestEquality(ctx, changes)) },
         {   title: "Undo last enviroment specifics".bold,                      skip: ctx => args.force || !ctx.lastSha, task: ()     => cmdEnv.unApplyLastEnvironmentDeployedToServerChanges() },
@@ -32,18 +32,18 @@ exports.validateDeployConditions = validateDeployConditions;
 
 /* ************************************ */
 function _handleGetLastDeployedSha(ctx, cmdEnv, lastSha) {
-    if (lastSha) { 
-        ctx.lastSha = lastSha 
-    } else { 
-        ctx.err = 
-            "No previous deploy to " 
-            + cmdEnv.server.bold 
-            + " found.\n If it's a new server do " 
-            + ("cob-cli deploy -s " + cmdEnv.servername + " --force").yellow 
-            + " first, to initialize the server\n\n " 
-            + "Aborted:".bgRed + " no previous deploy detected on this server." 
+    if (lastSha) {
+        ctx.lastSha = lastSha
+    } else {
+        ctx.err =
+            "No previous deploy to "
+            + cmdEnv.server.bold
+            + " found.\n If it's a new server do "
+            + ("cob-cli deploy -s " + cmdEnv.servername + " --force").yellow
+            + " first, to initialize the server\n\n "
+            + "Aborted:".bgRed + " no previous deploy detected on this server."
             + "\n"
-    } 
+    }
 }
 
 /* ************************************ */

package/lib/task_lists/rsyncFilter-post.txt

@@ -1,11 +1,11 @@
-+ /services/com.cultofbits.web.integration.properties
-+ /services/com.cultofbits.genesis.comm.properties
-+ /services/com.cultofbits.confm.interact.properties
-+ /services/com.cultofbits.integrationm.service.properties
++ /services/com.cultofbits.web.integration.properties*
++ /services/com.cultofbits.genesis.comm.properties*
++ /services/com.cultofbits.confm.interact.properties*
++ /services/com.cultofbits.integrationm.service.properties*
 - /services/**
 
 - /security
 - /hornetq
 - /elasticsearch
 - /db
-- /reports
+- /reports

package/lib/task_lists/test_otherFilesContiousReload.js

@@ -36,7 +36,7 @@ async function otherFilesContiousReload(cmdEnv) {
     }
     /* ************************************ */
 
-    fs.writeFileSync('.'+IN_PROGRESS_TEST_FILE, "in progress")
+    fs.writeFileSync('.'+IN_PROGRESS_TEST_FILE, "starting")
 
     changes.length && console.log("\n Making the changes...".yellow);
     for(let change of changes) {
@@ -44,6 +44,9 @@ async function otherFilesContiousReload(cmdEnv) {
         const changedFile = change.split(" ")[1];
         await addFileToCurrentTest(cmdEnv.server, changedFile, changedFiles, changeType, restoreChanges);
     }
+
+    fs.writeFileSync('.'+IN_PROGRESS_TEST_FILE, "in progress")
+
     watcher.process = chokidar.watch('.', {
         //ignored: /(^|[\/\\])\../, // ignore dotfiles
         persistent: true,

package/lib/task_lists/test_syncFile.js

@@ -1,7 +1,7 @@
 const execa = require('execa');
 const fs = require('fs-extra');
 const path = require('path');
-const { resolveCobPath } = require("./common_syncFiles");
+const { resolveCobPath, decryptEncFile } = require("./common_syncFiles");
 const { getSshArgs, getRsyncSsh } = require("./common_helpers");
 
 const DEBUG = false;
@@ -17,6 +17,7 @@ async function syncFile(server, localFile) {
     let remoteProductPath = resolveCobPath(server, "serverLive", product);
     let remoteFileDir = remoteProductPath.split(":")[1] + productScopeFilePath.substring(0, productScopeFilePath.lastIndexOf("/"));
 
+    let fileWasSynced = true;
     let count = 0;
     while (count < RSYNC_RETRIES) {
         await execa('ssh', [...getSshArgs(server), "mkdir -p " + remoteFileDir]).catch({});
@@ -34,7 +35,15 @@ async function syncFile(server, localFile) {
             .catch(async (err) => {
                 // exitCode 23 indicam que o ficheiro não existe na origem. Temod de apagar no destino.
                 if (err.exitCode == 23) {
-                    await execa('ssh', [...getSshArgs(server), "rm -fd " + remoteProductPath.split(":")[1] + productScopeFilePath]).catch({});
+                    fileWasSynced = false;
+                    await execa('ssh', [...getSshArgs(server),
+                      "rm -fd " + remoteProductPath.split(":")[1] + productScopeFilePath
+                    ]).catch({});
+                    if (localFile.endsWith(".enc")) {
+                       await execa('ssh', [ ...getSshArgs(server),
+                          "rm -fd " + remoteProductPath.split(":")[1] + productScopeFilePath.slice(0, -4)
+                        ]).catch({});
+                    }
 
                     let localFileDir = localFile.substring(0, localFile.lastIndexOf("/"));
                     if (!fs.existsSync(localFileDir)) {
@@ -51,5 +60,9 @@ async function syncFile(server, localFile) {
             });
         count++;
     }
+
+    if (fileWasSynced && localFile.endsWith(".enc")) {
+        await decryptEncFile(server, remoteProductPath.split(":")[1] + productScopeFilePath);
+    }
 }
 exports.syncFile = syncFile;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cob-cli",
-  "version": "2.53.0-beta-3",
+  "version": "2.54.0-beta-1",
   "description": "A command line utility to help Cult of Bits partners develop with higher speed and reusing common code and best practices.",
   "preferGlobal": true,
   "repository": {