cob-cli 2.46.0-beta.10 → 2.46.0-beta.11

package/bin/cob-cli.js

@@ -10,6 +10,7 @@ const init             = require("../lib/commands/init");
 const customize        = require("../lib/commands/customize");
 const test             = require("../lib/commands/test");
 const deploy           = require("../lib/commands/deploy");
+const package          = require("../lib/commands/package");
 const updateFromServer = require("../lib/commands/updateFromServer");
 const getDefs          = require("../lib/commands/getDefs");
 const generateMermaid  = require("../lib/commands/generateMermaid");
@@ -73,12 +74,20 @@ program
     .description('Deploy customization to the server')
     .action( deploy );
 
+program
+    .command('package')
+    .option('-e --environment <name>', 'environment to use')
+    .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .description('Package customization for out-of-band deploying')
+    .action( package );
+
 program
     .command('updateFromServer')
     .description('Updates local copy with current files on server, in case of changes made out of standard process.')
     .option('-e --environment <name>', 'environment to use')
     .option('-s --servername <servername>', 'use <servername>.cultofbits.pt (i.e. name without the FQDN)')
     .option('-V --verbose', 'verbose execution of tasks', increaseVerbosity, 0)
+    .option('-c --code', 'By adding this flag you indicate that you want to bring the code and not the data')
     .action( updateFromServer );
 
 program

package/lib/commands/package.js

@@ -0,0 +1,20 @@
+require('colors');
+const { checkRepoVersion } = require("../commands/upgradeRepo");
+const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
+const { executeTasks } = require("../task_lists/package_execute");
+
+async function package(args) {
+    try {
+        checkRepoVersion()
+        const cmdEnv = await getCurrentCommandEnviroment(args)
+        if(args.resync) args.force = true;
+
+        await executeTasks(cmdEnv, args).run();
+
+        console.log("\nDone!".green, "\nEnjoy!")
+
+    } catch(err) {
+        console.error("\n",err.message);
+    }
+}
+module.exports = package;

package/lib/commands/updateFromServer.js

@@ -2,30 +2,130 @@ require('colors');
 const { getCurrentCommandEnviroment } = require("../task_lists/common_enviromentHandler");
 const { copyFiles } = require("../task_lists/common_syncFiles");
 const { validateUpdateFromServerConditions } = require("../task_lists/updateFromServer_validate");
+const { getGroupsFromServer } = require("../task_lists/updateFromServer_groups");
+const { getDefinitionsFromServer } = require("../task_lists/updateFromServer_definitions");
+const { getInstancesFromServer } = require("../task_lists/updateFromServer_instances");
+const { getKibanaFromServer } = require("../task_lists/updateFromServer_kibana");
 const { checkRepoVersion } = require("../commands/upgradeRepo");
+const inquirer = require("inquirer")
+const axios = require('axios');
+const tough = require('tough-cookie');
+const fs = require("fs/promises");
+const yaml = require('js-yaml');
+const axiosCookieJarSupport = require('axios-cookiejar-support')
 
 async function updateFromServer(args) {
     try {
         checkRepoVersion()
         const cmdEnv = await getCurrentCommandEnviroment(args)
 
-        await validateUpdateFromServerConditions(cmdEnv).run()
+        if (args.code) {
 
-        console.log("\nOk to proceed. Getting files from server's live directories...");
-        await cmdEnv.applyCurrentCommandEnvironmentChanges()
-        let changes = await copyFiles(cmdEnv, "serverLive", "localCopy", args) 
-        await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+            await validateUpdateFromServerConditions(cmdEnv).run()
 
-        if(changes.length == 0) {
-            console.log("\nFinished.".green,"Nothing todo, no changes detected.");
+            console.log("\nOk to proceed. Getting files from server's live directories...");
+            await cmdEnv.applyCurrentCommandEnvironmentChanges()
+            let changes = await copyFiles(cmdEnv, "serverLive", "localCopy", args)
+            await cmdEnv.unApplyCurrentCommandEnvironmentChanges()
+
+            if (changes.length == 0) {
+                console.log("\nFinished.".green, "Nothing todo, no changes detected.");
+            } else {
+                console.log("\n " + changes.join("\n "));
+                console.log("\nUpdate done!".yellow, "Check", "git status".bold.blue, "and", "git diff".bold.blue, "to see the resulting differences.");
+                console.log("Notice that", "any changes since last deploy migth be lost.".underline)
+                console.log("Notice also that you will need to do a", "cob-cli deploy --force".bold.blue, "on next deploy.")
+            }
         } else {
-            console.log("\n " + changes.join("\n "));
-            console.log("\nUpdate done!".yellow,"Check","git status".bold.blue,"and","git diff".bold.blue,"to see the resulting differences.");
-            console.log("Notice that","any changes since last deploy migth be lost.".underline)
-            console.log("Notice also that you will need to do a","cob-cli deploy --force".bold.blue,"on next deploy.")
+
+            const cookieJar = new tough.CookieJar()
+            axiosCookieJarSupport.wrapper(axios)
+            axios.defaults.jar = cookieJar
+            axios.defaults.ignoreCookieErrors = true
+            axios.defaults.withCredentials = true
+
+            const credentials = await getCredentials()
+
+            await axios.post(`https://${cmdEnv.servername}.cultofbits.com/recordm/security/auth`, credentials, {
+                headers: {
+                    "Content-Type": "application/json",
+                },
+            })
+
+            const customizations = await readConfig()
+
+            for (const customization of customizations) {
+
+                console.log("\nStarting updateFromServer of customization " + customization.name)
+
+                const dataPath = 'others/customizationData/' + customization.name
+                fs.mkdir(dataPath, { recursive: true })
+
+
+                if (customization.permissions) {
+                    await getGroupsFromServer(cmdEnv.servername, customization.permissions.query, dataPath + "/permissions.json", customization.name)
+                }
+
+                if (customization.definitions) {
+                    const definitionPath = dataPath + '/definitions/'
+                    fs.mkdir(definitionPath, { recursive: true })
+                    const viewsPath = dataPath + '/views/'
+                    fs.mkdir(viewsPath, { recursive: true })
+                    await getDefinitionsFromServer(cmdEnv.servername, customization.definitions.filter, definitionPath, viewsPath, customization.name)
+                }
+
+                if (customization.instances) {
+                    const instancesPath = dataPath + '/instances/'
+                    fs.mkdir(instancesPath, { recursive: true })
+                    await getInstancesFromServer(cmdEnv.servername, customization.instances, instancesPath, customization.name)
+                }
+
+                if(customization.kibana){
+                    const kibanaPath = dataPath + '/kibana/'
+                    fs.mkdir(kibanaPath, { recursive: true })
+                    await getKibanaFromServer(cmdEnv.servername, customization.kibana.spaces, kibanaPath, customization.name)
+                }
+                
+            }
         }
-    } catch(err) {
-        console.error("\n",err.message);
+
+
+    } catch (err) {
+        console.error("\n", err.message);
     }
 }
+
+async function readConfig() {
+    try {
+        const data = await fs.readFile("customizationsDef.yml", 'utf-8');
+        const config = yaml.load(data);
+        return config
+    } catch (error) {
+        console.error('Error reading YAML file:', error.message);
+    }
+}
+
+
+const getCredentials = async () => {
+
+    const answers = await inquirer
+        .prompt([
+            {
+                type: 'text',
+                message: 'Enter username:',
+                name: 'username',
+            },
+            {
+                type: 'password',
+                message: 'Enter password:',
+                name: 'password',
+                mask: '*',
+            },
+        ])
+
+    return answers
+};
+
+
+
 module.exports = updateFromServer;

package/lib/task_lists/common_syncFiles.js

@@ -32,6 +32,7 @@ async function _syncFiles(executionType, cmdEnv, from, to, extraOptions = [], ar
         return new Promise(async (resolve, reject) => {
             let fromPath = resolveCobPath(cmdEnv.server, from, product);
             let toPath = resolveCobPath(cmdEnv.server, to, product);
+            if (toPath.startsWith('.') && !fs.existsSync(toPath)) fs.mkdirSync(toPath)
             let productExtraOptions = extraOptions.map( option => option.replace(/=.*\|/,"="))
             productExtraOptions = productExtraOptions.filter( option => option.indexOf("|") < 0 )
 
@@ -178,6 +179,9 @@ function resolveCobPath(server, type, product) {
                 default:
                     return server + ':' + "/etc/" + product + "/";
             }
+
+        case "staging":
+            return ".staging/" + product + "/";
     }
 }
 exports.resolveCobPath = resolveCobPath;

package/lib/task_lists/package_execute.js

@@ -0,0 +1,42 @@
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const verboseRenderer = require('listr-verbose-renderer');
+const { copyFiles } = require("./common_syncFiles");
+const fs = require("fs");
+const tar = require('tar');
+const { format } = require('date-fns');
+
+
+function executeTasks(cmdEnv, args) {
+    console.log("\nPackaging ...");
+
+    return new Listr([
+        {title: "Apply new enviroment specifics", task: () => cmdEnv.applyCurrentCommandEnvironmentChanges() },
+        {title: "Create staging dir",             task: () => fs.existsSync('.staging') || fs.mkdirSync('.staging') },
+        {title: "Copy files to staging dir",      task: () => copyFiles(cmdEnv, "localCopy", "staging")},
+        {title: "Create package",                 task: async () => { const filename = await createPackage(cmdEnv, args); console.log("created " + filename) }},
+        {title: "Sign package",                   task: () => console.log("lololo")},
+        {title: "Delete staging dir",             task: () => fs.rmSync('.staging', {recursive: true, force: true})},
+        {title: "Undo new enviroment specifics",  task: () => cmdEnv.unApplyCurrentCommandEnvironmentChanges() },
+    ], {
+        renderer: args.verbose ? verboseRenderer : UpdaterRenderer,
+        collapse: false
+    });
+}
+exports.executeTasks = executeTasks;
+
+const createPackage = async function(cmdEnv, args){
+   const filename = `${cmdEnv.servername}-${cmdEnv.name}-${format(new Date(), 'yyyy-MM-dd_HH:mm')}.tar.gz`; 
+   const files = fs.readdirSync('.staging')
+   if (args.verbose > 0){
+      console.log("packaging " + files)
+   }
+
+   await tar.create({
+      gzip: true,
+      file: filename,
+      cwd: '.staging'
+   }, files)
+
+   return filename;
+}

package/lib/task_lists/updateFromServer_definitions.js

@@ -0,0 +1,81 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+
+async function getDefinitions(servername, queries) {
+
+	const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions`
+	const response = await axios.get( uri );
+
+	if(response.status != 200){
+		throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	 }
+
+	const matchesAny = (def) => queries.filter(q => match(q, def)).length > 0;
+	 
+	return response.data
+		.filter( matchesAny )
+		.map( def => ({id: def.id, name: def.name}) )
+}
+
+function match(query, definition) {
+	// simple matching for now
+	return definition.name.includes(query) || definition.description.includes(query) 
+}
+
+async function exportDefinitionChanges(servername, id, dataPath) {
+
+	const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${id}/changes`
+	const response = await axios.get(uri);
+
+	if(response.status != 200){
+		throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	 }
+
+	const jsonLines = response.data.map(change => JSON.stringify(change)).join('\n');
+
+	await fs.writeFile(dataPath, jsonLines, 'utf-8');
+}
+
+async function getViewsForDef(servername, definition, dataPath, user) {
+	const definitionId = definition.id
+	const uri = `https://${servername}.cultofbits.com/recordm/user/settings/definitions-${definitionId}/views?`
+
+	const response = await axios.get(uri);
+
+	 if(response.status != 200){
+		throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	 }
+
+	 const userViews = response.data.filter(v => v.user == user)
+
+	 if(userViews.length == 0)
+		return
+
+	 const jsonLines = userViews.map(change => JSON.stringify(change))
+								.join('\n');
+
+	await fs.writeFile(dataPath, jsonLines)
+
+ }
+
+async function getDefinitionsFromServer(servername, queries, definitionsPath, viewsPath, customization, user) {
+
+		await new Listr(
+		[
+			{   title: `(${customization}) Definitions: `.bold + "searching for definitions matching queries",   task: async ctx => ctx.defs = await getDefinitions(servername, queries) },
+			{   title: `(${customization}) Definitions: `.bold + "exporting changes for definitions", task: ctx => ctx.defs.forEach( d => exportDefinitionChanges(servername, d.id, definitionsPath + `${d.name}.jsonl`) ) },
+			{   title: `(${customization}) Definitions: `.bold + "exporting views for definitions", task: ctx => ctx.defs.forEach( d => getViewsForDef(servername, d, viewsPath + `${d.name}.jsonl`, user) ) },
+		],{
+			renderer: UpdaterRenderer,
+			collapse: false
+		})
+		.run()
+	
+}
+
+
+exports.getDefinitionsFromServer = getDefinitionsFromServer

package/lib/task_lists/updateFromServer_groups.js

@@ -0,0 +1,68 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function getGroups(query, servername) {
+
+	const uri = `https://${servername}.cultofbits.com/userm/userm/search/userm/group?`
+	const response = await axios.get( uri + new URLSearchParams({
+	   q: query,
+	   size: "100" // probably excessive
+	}));
+	
+	if(response.status != 200){
+	   throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	}
+	const groupIds = response.data.hits.hits.map((h) => h._id)
+	return groupIds;
+ };
+
+ const sortByName = (a, b) => a.name?.localeCompare(b.name);
+
+ const exportGroups = async function(ids, servername, exportPath){
+	
+	const uri = `https://${servername}.cultofbits.com/userm/userm/group/export/${ids.join(',')}`
+ 
+	const response = await axios.get( uri );
+	if(response.status != 200 ){
+	   throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	}
+	const data = response.data;
+ 
+	// sort everything to be easier to diff
+	data.groups?.sort(sortByName);
+	data.groups?.forEach(group => group.roles?.sort())
+	data.roles?.sort(sortByName);
+	data.roles?.forEach(role => role.perms?.sort())
+	data.perms?.sort(sortByName);
+ 
+	fs.writeFile(exportPath, JSON.stringify(data, null, 2))
+
+ };
+
+async function groupsFromServer(servername, groupsQueries, exportPath, customization) {
+	const searchTasks = groupsQueries.map( gp => 
+		({ title: `(${customization}) Permissions: `.bold + "searching for groups with query: " + gp,
+			task: async ctx => ctx.groupIds = ctx.groupIds 
+											? ctx.groupIds.concat(await getGroups(gp, servername))
+										    :  await getGroups(gp, servername) }))
+	
+
+
+	await new Listr(
+	[
+		...searchTasks,
+		{   title: `(${customization}) Permissions: `.bold + "exporting groups and subsequent roles/permissions", task: ctx => exportGroups(ctx.groupIds, servername, exportPath)},
+	],{
+		renderer: UpdaterRenderer,
+		collapse: false
+	})
+	.run()
+}
+
+
+exports.getGroupsFromServer = groupsFromServer
+
+

package/lib/task_lists/updateFromServer_instances.js

@@ -0,0 +1,62 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+
+async function getInstancesFromServer(servername, definitionsForInstances, rootPath, customization) {
+
+	const taskList = definitionsForInstances.map( def => ({
+		title: `(${customization}) Instances: `.bold + `Exporting instances from definition ${def.definition}`,
+		task: async ctx => getInstancesOfDefinition(servername, await getDefinition(servername, def.definition), def.query, rootPath) 
+	}))
+
+	await new Listr(taskList , {
+		renderer: UpdaterRenderer,
+		collapse: false
+	}).run()
+
+
+
+}
+
+function getFieldList(definition) {
+	return Array.from(
+		new Set(flattenFields(definition.fieldDefinitions))
+	).join(',')
+}
+function flattenFields(fieldArray) {
+	if (!fieldArray || fieldArray.length == 0)
+		return []
+
+	return fieldArray.flatMap(field => flattenFields(field.fields).concat([field.name]))
+}
+
+
+async function getInstancesOfDefinition(servername, definition, query, rootPath) {
+
+	const definitionId = definition.id
+
+	const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/search/download/${definitionId}?`
+	const response = await axios.get(uri + new URLSearchParams({
+		q: query,
+		vcn: getFieldList(definition)
+	}), {
+		responseType: 'arraybuffer',
+	});
+
+	fs.writeFile(rootPath + `${definition.name}.xlsx`, response.data)
+
+}
+
+async function getDefinition(servername, definitionName) {
+
+	const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/name/${definitionName}`
+	const response = await axios.get(uri);
+
+	return response.data
+}
+
+
+exports.getInstancesFromServer = getInstancesFromServer

package/lib/task_lists/updateFromServer_kibana.js

@@ -0,0 +1,57 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function exportSavedObjectsOfSpace(servername, space, filePath) {
+
+	const uri = `https://${servername}.cultofbits.com/kibana/s/${space.toLowerCase()}/api/kibana/management/saved_objects/_find?perPage=1000&type=config&type=url&type=index-pattern&type=query&type=tag&type=canvas-element&type=canvas-workpad&type=action&type=alert&type=visualization&type=dashboard&type=map&type=lens&type=cases&type=search&sortField=type`
+	const response = await axios.get( uri );
+
+	if(response.status != 200){
+		throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	 }
+
+	await Promise.all(response.data.saved_objects.map(async element => { if(element.type === "index-pattern") await replaceIndexPatternsIds(servername, element) }))
+
+	fs.writeFile(filePath, JSON.stringify(response.data.saved_objects, null, 2))
+}
+
+// replaces in place
+async function replaceIndexPatternsIds(servername, indexPattern) {
+	if(indexPattern.type !== "index-pattern"){
+		throw new Error("Not an index pattern")
+	}
+
+	const definitionId = indexPattern.meta.title.substring('recordm-'.length)
+	const name = await getDefinitionName(servername, definitionId)	
+
+	indexPattern.meta.title = `recordm-§§${name}§§`
+}
+
+async function getDefinitionName(servername, id) {
+	const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${id}`
+	const response = await axios.get( uri );
+
+	if(response.status != 200){
+		throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+	 }
+
+	return response.data.name 
+}
+
+async function getKibanaFromServer(servername, spaces, roothpath, customization) {
+
+	const taskList = spaces.map( space => (
+		{   title: `(${customization}) Kibana: `.bold + `Exporting ${space}`,   task: async ctx => ctx.groupIds = await exportSavedObjectsOfSpace(servername, space, roothpath + `/${space}.json`) }
+	))
+
+	await new Listr( taskList, {
+			renderer: UpdaterRenderer,
+			collapse: false
+		})
+		.run()	
+}
+
+exports.getKibanaFromServer = getKibanaFromServer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cob-cli",
-  "version": "2.46.0-beta.10",
+  "version": "2.46.0-beta.11",
   "description": "A command line utility to help Cult of Bits partners develop with higher speed and reusing common code and best practices.",
   "preferGlobal": true,
   "repository": {
@@ -18,6 +18,7 @@
   "dependencies": {
     "@webpack-cli/serve": "^2.0.1",
     "axios": "^0.21.1",
+    "axios-cookiejar-support": "^5.0.5",
     "chokidar": "^3.5.3",
     "cmd-line-importer": "^1.0.0",
     "colors": "^1.4.0",
@@ -28,11 +29,14 @@
     "file-cookie-store": "^0.2.1",
     "fs-extra": "^9.0.1",
     "inquirer": "^7.3.3",
+    "js-yaml": "^4.1.0",
     "listr": "^0.14.3",
     "ncp": "^2.0.0",
     "omelette": "^0.4.15-1",
     "open": "^8.4.0",
     "simple-git": "^3.15.1",
+    "tar": "^7.4.3",
+    "tough-cookie": "^5.1.2",
     "webpack": "^5.75.0",
     "webpack-cli": "^5.0.1",
     "webpack-dev-server": "^4.11.1",

package/schema.json

@@ -0,0 +1,33 @@
+[
+	{
+		"name" : "BB032",
+		"definitions": {
+			"query" : ["BB032", "Dashboard", "BPO"]
+		},
+		"permissions" : {
+			"query" : "@BB032"
+		}, 
+		"instances" : [
+			{
+				"definition" : "Dashboard_v1",
+				"query" : "solution_reference:BB032"
+			},
+			{
+				"definition" : "Dashboard-Files",
+				"query" : "*"
+			},
+			{
+				"definition" : "Dashboard-Solutions",
+				"query" : "BB032"
+			},
+			{
+				"definition" : "Business Process",
+				"query" : "BB032"
+			},
+			{
+				"definition" : "Work Queues",
+				"query" : "BB032"
+			}
+		]
+	}
+]