cob-cli 2.51.0 → 2.52.0-beta-2

package/bin/cob-cli.js

@@ -10,7 +10,10 @@ const init             = require("../lib/commands/init");
 const customize        = require("../lib/commands/customize");
 const test             = require("../lib/commands/test");
 const deploy           = require("../lib/commands/deploy");
+const cleanup          = require("../lib/commands/cleanup");
+const package          = require("../lib/commands/package");
 const updateFromServer = require("../lib/commands/updateFromServer");
+const tagDefs          = require("../lib/commands/tagDefs.js");
 const getDefs          = require("../lib/commands/getDefs");
 const generateMermaid  = require("../lib/commands/generateMermaid");
 const { upgradeRepo }  = require("../lib/commands/upgradeRepo");
@@ -73,12 +76,30 @@ program
     .description('Deploy customization to the server')
     .action( deploy );
 
+program
+    .command('cleanup')
+    .option('-e --environment <name>', 'environment to use')
+    .option('-s --servername <servername>', 'use <servername>.cultofbits.pt (i.e. name without the FQDN)')
+    .option('-l --localOnly', 'only clean local state, skip server cleanup')
+    .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .description('Clean up repository state after an interrupted test or deploy')
+    .action( cleanup );
+
+program
+    .command('package')
+    .option('-e --environment <name>', 'environment to use')
+    .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .description('Package customization for out-of-band deploying')
+    .action( package );
+
 program
     .command('updateFromServer')
     .description('Updates local copy with current files on server, in case of changes made out of standard process.')
     .option('-e --environment <name>', 'environment to use')
     .option('-s --servername <servername>', 'use <servername>.cultofbits.pt (i.e. name without the FQDN)')
     .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .option('-c --code', 'By adding this flag you indicate that you want to bring the code and not the data')
+    .option('--cookie <path>', '')
     .action( updateFromServer );
 
 program
@@ -91,6 +112,16 @@ program
     .option('-e --environment <name>', 'environment to use')
     .description('Updates local copy with definitions on server')
     .action( getDefs );
+
+program
+    .command('tagDefs')
+    .description('Allows the user to tag un-tagged Changes in Definitions configured on exported solutions.')
+    .option('-e --environment <name>', 'environment to use')
+    .option('-s --servername <servername>', 'use <servername>.cultofbits.pt (i.e. name without the FQDN)')
+    .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .option('--all', 'tag all Definitions of all exported solutions. Use with care!')
+    .option('--cookie <path>', '')
+    .action( tagDefs );
     
 program
     .command('generateMermaid')

package/lib/commands/cleanup.js

@@ -0,0 +1,116 @@
+require('colors');
+const git = require('simple-git');
+const fs = require('fs-extra');
+const fg = require('fast-glob');
+const execa = require('execa');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const verboseRenderer = require('listr-verbose-renderer');
+const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
+const { SERVER_COB_CLI_DIRECTORY } = require("../task_lists/common_helpers");
+const { restoreServerFilesToLastDeployed } = require("../task_lists/test_otherFilesContiousReload");
+const { loadOperationState, clearOperationState } = require("../task_lists/common_operationState");
+const { checkRepoVersion } = require("../commands/upgradeRepo");
+const { getSshArgs } = require("../task_lists/common_helpers");
+
+const LOCAL_TEST_LOCK_FILE = ".test.in.execution";
+const SERVER_TEST_LOCK_FILE = SERVER_COB_CLI_DIRECTORY + "test.in.execution";
+
+async function cleanup(args) {
+    try {
+        checkRepoVersion()
+
+        const savedState = loadOperationState();
+        if (!savedState && !args.environment) {
+            console.log("No saved state and no explicit environment, aborting".bold);
+            return;
+        }
+        if (savedState) {
+            if (!args.environment) args.environment = savedState.environment;
+            if (!args.servername)  args.servername  = savedState.servername;
+        }
+
+        console.log("Cleaning up interrupted test/deploy state...".bold)
+
+        const cmdEnv = await getCurrentCommandEnviroment(args)
+        console.log(`  environment: ${cmdEnv.name.bold}  server: ${cmdEnv.serverStr}`)
+
+        await new Listr([
+            {
+                title: "Undo environment-specific file changes".bold,
+                skip: async () => {
+                    const backupFiles = await fg(
+                        ['**/*.ENV__ORIGINAL_BACKUP__.*', '**/*.ENV__DELETE__.*'],
+                        { onlyFiles: false, dot: true }
+                    );
+                    return backupFiles.length === 0 ? "No environment-specific changes found" : false;
+                },
+                task: () => cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+            },
+            {
+                // Runs after the local env undo so the working tree is clean before
+                // stashing and checking out the last-deployed SHA.
+                title: "Restore server files to last deployed version".bold,
+                skip: () => args.localOnly ? "Skipping server restore (--localOnly)" : false,
+                task: async () => {
+                    let changedFiles = [];
+                    try {
+                        const result = await execa('ssh', [...getSshArgs(cmdEnv.server), "cat " + SERVER_TEST_LOCK_FILE]);
+                        changedFiles = result.stdout.split("\n")
+                            .map(l => l.split(" ")[0].trim())
+                            .filter(f => f);
+                    } catch (error) {
+                        if (error.exitCode === 255) {
+                            throw new Error(`SSH connection to '${cmdEnv.server}' failed: ${error.shortMessage}`);
+                        }
+                        /* no test.in.execution on server = nothing to restore */
+                    }
+
+                    if (changedFiles.length === 0) return;
+
+                    await restoreServerFilesToLastDeployed(cmdEnv, changedFiles);
+                }
+            },
+            {
+                title: "Restore git branch from detached HEAD".bold,
+                skip: async () => {
+                    const branch = await git().revparse(["--abbrev-ref", "HEAD"]);
+                    return branch.trim() !== "HEAD" ? "Not in detached HEAD state" : false;
+                },
+                task: () => git().checkout("-")
+            },
+            {
+                title: "Remove local test lock file".bold,
+                skip: () => !fs.existsSync(LOCAL_TEST_LOCK_FILE) ? "No local test lock file found" : false,
+                task: () => fs.unlinkSync(LOCAL_TEST_LOCK_FILE)
+            },
+            {
+                title: "Remove server test lock file".bold,
+                skip: () => args.localOnly ? "Skipping server cleanup (--localOnly)" : false,
+                task: () => execa('ssh', [...getSshArgs(cmdEnv.server), "rm -f " + SERVER_TEST_LOCK_FILE])
+                    .catch(() => {}) // non-critical if server unreachable
+            },
+        ], {
+            renderer: args.verbose ? verboseRenderer : UpdaterRenderer,
+            collapse: false
+        }).run()
+
+        clearOperationState()
+
+        // Inform about stashes — don't auto-pop since user may have unrelated stashes
+        const stashList = await git().stash(["list"]);
+        if (stashList) {
+            console.log("\n" + "Note:".yellow.bold
+                + " Stashed changes detected (may be from the interrupted operation):")
+            console.log(stashList.split("\n").slice(0, 3).map(l => "  " + l).join("\n"))
+            console.log("Run " + "git stash pop".yellow + " if you want to restore your uncommitted changes.\n")
+        }
+
+        console.log("\nDone!".green, "\nYou can now run 'cob-cli test' or 'cob-cli deploy' again.\n")
+
+    } catch(err) {
+        console.error("\n", err.message)
+    }
+}
+
+module.exports = cleanup;

package/lib/commands/deploy.js

@@ -4,40 +4,47 @@ const { confirmExecutionOfChanges } = require("../task_lists/common_syncFiles");
 const { validateDeployConditions } = require("../task_lists/deploy_validate");
 const { executeTasks } = require("../task_lists/deploy_execute");
 const { checkRepoVersion } = require("../commands/upgradeRepo");
+const { saveOperationState, clearOperationState, checkNoInterruptedRun } = require("../task_lists/common_operationState");
 
 async function deploy(args) {
-    try {
-        checkRepoVersion()
-        const cmdEnv = await getCurrentCommandEnviroment(args)
-        if(args.resync) args.force = true;
+    checkRepoVersion()
+    const cmdEnv = await getCurrentCommandEnviroment(args)
+    await checkNoInterruptedRun()
+    saveOperationState(cmdEnv.name, cmdEnv.servername)
+    if(args.resync) args.force = true;
 
-        console.log(`Checking conditions to deploy ${cmdEnv.branchStr} to ${cmdEnv.serverStr}...` );
+    console.log(`Checking conditions to deploy ${cmdEnv.branchStr} to ${cmdEnv.serverStr}...` );
 
+    try {
         await validateDeployConditions(cmdEnv,args).run()
+    } catch (err) {
+        // we should clear the operation state
+        clearOperationState();
+        throw err
+    }
 
-        await cmdEnv.applyCurrentCommandEnvironmentChanges()
-        let changes = [];
-        try {
-            changes = await confirmExecutionOfChanges(cmdEnv)
-        } catch (err) {
-            await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
-            throw err
-        }
+    await cmdEnv.applyCurrentCommandEnvironmentChanges()
+    let changes = [];
+    try {
+        changes = await confirmExecutionOfChanges(cmdEnv)
+    } catch (err) {
+        await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+        clearOperationState()
+        throw err
+    }
 
-        if(changes.length == 0) {
-            if(!args.force) {
-                await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
-                throw new Error("Canceled:".yellow + " nothing todo\n")
-            }
-            console.log(" Just updating deploy information.")
+    if(changes.length == 0) {
+        if(!args.force) {
+            await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+            clearOperationState()
+            throw new Error("Canceled:".yellow + " nothing todo\n")
         }
+        console.log(" Just updating deploy information.")
+    }
 
-        await executeTasks(cmdEnv, args).run();
-
-        console.log("\nDone!".green, "\nEnjoy!")
+    await executeTasks(cmdEnv, args).run();
 
-    } catch(err) {
-        console.error("\n",err.message);
-    }
+    clearOperationState()
+    console.log("\nDone!".green, "\nEnjoy!")
 }
-module.exports = deploy;
+module.exports = deploy;

package/lib/commands/package.js

@@ -0,0 +1,20 @@
+require('colors');
+const { checkRepoVersion } = require("../commands/upgradeRepo");
+const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
+const { executeTasks } = require("../task_lists/package_execute");
+
+async function package(args) {
+    try {
+        checkRepoVersion()
+        const cmdEnv = await getCurrentCommandEnviroment(args)
+        if(args.resync) args.force = true;
+
+        await executeTasks(cmdEnv, args).run();
+
+        console.log("\nDone!".green, "\nEnjoy!")
+
+    } catch(err) {
+        console.error("\n",err.message);
+    }
+}
+module.exports = package;

package/lib/commands/tagDefs.js

@@ -0,0 +1,67 @@
+require('colors');
+const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
+const { promptCredentials, readYamlFile } = require("../task_lists/common_helpers.js");
+const { tagDefinitions } = require("../task_lists/tagDefs_execute.js");
+const { checkRepoVersion } = require("../commands/upgradeRepo");
+const axios = require('axios');
+const tough = require('tough-cookie');
+const axiosCookieJarSupport = require('axios-cookiejar-support')
+const path = require('path')
+const FileCookieStore = require('file-cookie-store');
+
+async function tagDefs(args) {
+    try {
+        checkRepoVersion()
+        const cmdEnv = await getCurrentCommandEnviroment(args)
+
+        axiosCookieJarSupport.wrapper(axios)
+        axios.defaults.ignoreCookieErrors = true
+        axios.defaults.withCredentials = true
+        const cookieJar = new tough.CookieJar()
+        axios.defaults.jar = cookieJar
+
+        if( args.cookie ) {
+            const p = path.resolve(args.cookie);
+            const cookieFile = new FileCookieStore(p, {auto_sync: false, force_parse: true});
+            try{
+                let c = await new Promise((resolve, reject) => {
+                    cookieFile.findCookie(`${cmdEnv.servername}.cultofbits.com`, '/', 'cobtoken', (err, result) => {
+                        if(err != null) reject(err);
+                        else resolve(result)
+                    })
+                });
+                if(!c){
+                    throw Error( `couldn't find a cookie for ${cmdEnv.servername} in ${p}`)
+                }
+                cookieJar.setCookie(c.toString(), `https://${cmdEnv.servername}.cultofbits.com`)
+            } catch(e){
+                console.error(e)
+                return
+            }
+
+        } else {
+
+            const credentials = await promptCredentials()
+
+            await axios.post(`https://${cmdEnv.servername}.cultofbits.com/recordm/security/auth`, credentials, {
+                headers: {
+                    "Content-Type": "application/json",
+                },
+            })
+        }
+
+        const solutions = await readYamlFile("solutions.yml")
+
+        for (const solution of solutions) {
+            console.log("\nStarting to tag Defs of solution " + solution.name, solution.definitions)
+            if (solution.definitions) {
+                await tagDefinitions(cmdEnv.servername, solution.name, solution.definitions, args)
+            }
+        }
+
+    } catch (err) {
+        console.error("\n", err.message);
+    }
+}
+
+module.exports = tagDefs;

package/lib/commands/test.js

@@ -6,19 +6,33 @@ const { otherFilesContiousReload } = require("../task_lists/test_otherFilesConti
 const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
 const { getKeypress } = require("../task_lists/common_helpers");
 const { checkRepoVersion } = require("../commands/upgradeRepo");
+const { saveOperationState, clearOperationState, checkNoInterruptedRun } = require("../task_lists/common_operationState");
 
-async function test (args) {    
+async function test (args) {
     let restoreChanges, error = "";
     let cmdEnv
     let spawned
+    let stateSaved = false
+    let envChangesApplied = false
+
+    let rejectOnSignal;
+    const signalPromise = new Promise((_, reject) => { rejectOnSignal = reject; });
+    const handleSignal = () => rejectOnSignal(new Error('signal'));
+    process.once('SIGTERM', handleSignal);
+    process.once('SIGHUP', handleSignal);
+
     try {
         checkRepoVersion()
         console.log("Start testing… ")
         cmdEnv = await getCurrentCommandEnviroment(args)
+        await checkNoInterruptedRun()
+        saveOperationState(cmdEnv.name, cmdEnv.servername)
+        stateSaved = true
 
         if(!args.localOnly) {
-            await validateTestingConditions(cmdEnv, args) 
+            await validateTestingConditions(cmdEnv, args)
             await cmdEnv.applyCurrentCommandEnvironmentChanges()
+            envChangesApplied = true
             restoreChanges = await otherFilesContiousReload(cmdEnv)
         } else {
             await cmdEnv.applyCurrentCommandEnvironmentChanges()
@@ -27,16 +41,22 @@ async function test (args) {
 
         let key;
         do {
-            key = await getKeypress()
-            if(key == "o") opn("http://localhost:8040/recordm/index.html")
-        } while( key != "q" && key != "ctrl+c" ) 
+            key = await Promise.race([getKeypress(), signalPromise, spawned.failed]);
+            if(key == "o") opn(`http://localhost:${spawned.port}/recordm/index.html`)
+        } while( key != "q" && key != "ctrl+c" )
 
-    } catch (err) { 
-        error = err.message
-        console.log("\n",error)
+    } catch (err) {
+        if(err.message !== 'signal' && err.message !== 'processes-failed') {
+            error = err.message
+            console.log("\n",error)
+        }
     } finally {
+        process.off('SIGTERM', handleSignal);
+        process.off('SIGHUP', handleSignal);
+        try { process.stdin.setRawMode(false); } catch {}
         restoreChanges && await restoreChanges()
-        cmdEnv && await cmdEnv.unApplyCurrentCommandEnvironmentChanges() // Repõe as configurações
+        envChangesApplied && cmdEnv && await cmdEnv.unApplyCurrentCommandEnvironmentChanges() // Repõe as configurações
+        if (stateSaved) clearOperationState()
         spawned && await spawned.stop();
         // Dá tempo aos subprocessos para morrer (acho que já não é preciso)
         setTimeout(() => {

package/lib/commands/updateFromServer.js

@@ -1,31 +1,167 @@
 require('colors');
 const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
 const { copyFiles } = require("../task_lists/common_syncFiles");
+const { promptCredentials, readYamlFile } = require("../task_lists/common_helpers.js");
 const { validateUpdateFromServerConditions } = require("../task_lists/updateFromServer_validate");
+const { getGroupsFromServer } = require("../task_lists/updateFromServer_groups");
+const { getDefinitionsFromServer } = require("../task_lists/updateFromServer_definitions");
+const { getInstancesFromServer } = require("../task_lists/updateFromServer_instances");
+const { getKibanaFromServer } = require("../task_lists/updateFromServer_kibana");
+const { getDomainsFromServer } = require("../task_lists/updateFromServer_domains");
 const { checkRepoVersion } = require("../commands/upgradeRepo");
+const axios = require('axios');
+const tough = require('tough-cookie');
+const fs = require("fs/promises");
+const axiosCookieJarSupport = require('axios-cookiejar-support')
+const path = require('path')
+const FileCookieStore = require('file-cookie-store');
 
 async function updateFromServer(args) {
     try {
         checkRepoVersion()
         const cmdEnv = await getCurrentCommandEnviroment(args)
 
-        await validateUpdateFromServerConditions(cmdEnv).run()
+        if (args.code) {
 
-        console.log("\nOk to proceed. Getting files from server's live directories...");
-        await cmdEnv.applyCurrentCommandEnvironmentChanges()
-        let changes = await copyFiles(cmdEnv, "serverLive", "localCopy", args) 
-        await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+            await validateUpdateFromServerConditions(cmdEnv).run()
 
-        if(changes.length == 0) {
-            console.log("\nFinished.".green,"Nothing todo, no changes detected.");
+            console.log("\nOk to proceed. Getting files from server's live directories...");
+            await cmdEnv.applyCurrentCommandEnvironmentChanges()
+            let changes = await copyFiles(cmdEnv, "serverLive", "localCopy", args)
+            await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+
+            if (changes.length == 0) {
+                console.log("\nFinished.".green, "Nothing todo, no changes detected.");
+            } else {
+                console.log("\n " + changes.join("\n "));
+                console.log("\nUpdate done!".yellow, "Check", "git status".bold.blue, "and", "git diff".bold.blue, "to see the resulting differences.");
+                console.log("Notice that", "any changes since last deploy migth be lost.".underline)
+                console.log("Notice also that you will need to do a", "cob-cli deploy --force".bold.blue, "on next deploy.")
+            }
         } else {
-            console.log("\n " + changes.join("\n "));
-            console.log("\nUpdate done!".yellow,"Check","git status".bold.blue,"and","git diff".bold.blue,"to see the resulting differences.");
-            console.log("Notice that","any changes since last deploy migth be lost.".underline)
-            console.log("Notice also that you will need to do a","cob-cli deploy --force".bold.blue,"on next deploy.")
+
+            axiosCookieJarSupport.wrapper(axios)
+            axios.defaults.ignoreCookieErrors = true
+            axios.defaults.withCredentials = true
+            const cookieJar = new tough.CookieJar()
+            axios.defaults.jar = cookieJar
+
+            if( args.cookie ) {
+                const p = path.resolve(args.cookie);
+                const cookieFile = new FileCookieStore(p, {auto_sync: false, force_parse: true});
+                try{
+                    let c = await new Promise((resolve, reject) => {
+                        cookieFile.findCookie(`${cmdEnv.servername}.cultofbits.com`, '/', 'cobtoken', (err, result) => {
+                            if(err != null) reject(err);
+                            else resolve(result)
+                        })
+                    });
+                    if(!c){
+                        throw Error( `couldn't find a cookie for ${cmdEnv.servername} in ${p}`)
+                    }
+                    cookieJar.setCookie(c.toString(), `https://${cmdEnv.servername}.cultofbits.com`)
+                } catch(e){
+                    console.error(e)
+                    return
+                }
+
+            } else {
+                const credentials = await promptCredentials()
+                await axios.post(`https://${cmdEnv.servername}.cultofbits.com/recordm/security/auth`, credentials, {
+                    headers: {
+                        "Content-Type": "application/json",
+                    },
+                })
+            }
+
+            console.log("\n Removing existing Data")
+            try{ await fs.access("others/solutionsData") } catch { await fs.mkdir("others/solutionsData") }
+            const dataFiles = await fs.readdir("others/solutionsData");
+            for (const file of dataFiles) {
+                const filePath = path.join("others/solutionsData", file);
+                await fs.rm(filePath, {recursive: true, force: true} )
+            }
+
+            const solutions = await readYamlFile("solutions.yml")
+
+            for (const solution of solutions) {
+
+                console.log("\nStarting updateFromServer of solution " + solution.name)
+
+                const dataPath = `others/solutionsData/${solution.name}`
+                fs.mkdir(dataPath, { recursive: true })
+
+                if (solution.permissions) {
+                    await getGroupsFromServer(
+                        cmdEnv.servername,
+                        solution.groups,
+                        dataPath + "/permissions.json",
+                        solution.name,
+                        args
+                    )
+                }
+
+                if (solution.definitions) {
+                    const definitionPath = dataPath + '/definitions/'
+                    fs.mkdir(definitionPath, { recursive: true })
+                    const viewsPath = dataPath + '/views/'
+                    fs.mkdir(viewsPath, { recursive: true })
+                    await getDefinitionsFromServer(
+                        cmdEnv.servername,
+                        solution.definitions,
+                        definitionPath,
+                        viewsPath,
+                        solution.name,
+                        args
+                    ).run()
+                }
+
+                if (solution.instances) {
+                    const instancesPath = dataPath + '/instances/'
+                    fs.mkdir(instancesPath, { recursive: true })
+                    await getInstancesFromServer(
+                        cmdEnv.servername,
+                        solution.instances,
+                        instancesPath,
+                        solution.name,
+                        args
+                    )
+                }
+
+                if(solution.kibana){
+                    const kibanaPath = dataPath + '/kibana/'
+                    fs.mkdir(kibanaPath, { recursive: true })
+                    await getKibanaFromServer(
+                        cmdEnv.servername,
+                        solution.kibana.spaces,
+                        kibanaPath,
+                        solution.name,
+                        args
+                    )
+                }
+
+                if(solution.domains){
+                    const domainsPath = dataPath + '/domains/'
+                    fs.mkdir(domainsPath, { recursive: true })
+                    await getDomainsFromServer(
+                        cmdEnv.servername,
+                        solution.domains.filter,
+                        domainsPath,
+                        solution.name,
+                        args
+                    )
+                }
+
+            }
+
+            await fs.writeFile('others/solutionsData/_index', solutions.map(c => c.name).join('\n'), 'utf-8')
         }
-    } catch(err) {
-        console.error("\n",err.message);
+
+
+    } catch (err) {
+        console.error("\n", err.message);
     }
 }
+
+
 module.exports = updateFromServer;

package/lib/task_lists/common_definitions.js

@@ -0,0 +1,26 @@
+const axios = require("axios");
+
+async function getFilteredDefinitions(servername, defConfigs) {
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions`
+    const response = await axios.get( uri );
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    let result = defConfigs.flatMap(c => {
+        return response.data.filter(def => match(c.filter, def))
+            .map(m => ({...m, ...c}))
+    });
+
+    return result
+}
+
+function match(query, definition) {
+    // simple matching for now
+    return definition.name.includes(query) || definition.description?.includes(query)
+}
+
+module.exports = {
+    getFilteredDefinitions: getFilteredDefinitions
+}