cob-cli 2.49.0 → 2.50.0

package/lib/task_lists/updateFromServer_groups.js

@@ -0,0 +1,111 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function getGroups(query, servername) {
+
+    const uri = `https://${servername}.cultofbits.com/userm/userm/search/userm/group?`
+    const response = await axios.get( uri + new URLSearchParams({
+        q: query,
+        size: "100" // probably excessive
+    }));
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+    const groupIds = response.data.hits.hits.map((h) => h._id)
+    return groupIds;
+};
+
+const sortByName = (a, b) => a.name?.localeCompare(b.name);
+
+const exportGroups = async function(ids, servername, exportPath){
+
+    const uri = `https://${servername}.cultofbits.com/userm/userm/group/export/${ids.join(',')}`
+
+    const response = await axios.get( uri );
+    if(response.status != 200 ){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+    const data = response.data;
+
+    // sort everything to be easier to diff
+    data.groups?.sort(sortByName);
+    data.groups?.forEach(group => group.roles?.sort())
+    data.roles?.sort(sortByName);
+    data.roles?.forEach(role => role.perms?.sort())
+    data.perms?.sort(sortByName)
+    const re = /:(\d+)/g
+    defnameCache.servername = servername;
+
+    for(r of data.roles) {
+        for(let i = 0; i < r.perms.length; i++){
+            r.perms[i] = await replaceAsync(r.perms[i], re, replaceIdsWithName)
+        }
+    }
+    for(p of data.perms){
+        if( p.product == 'recordm') {
+            p.name = await replaceAsync(p.name, re, replaceIdsWithName)
+        }
+    }
+
+    fs.writeFile(exportPath, JSON.stringify(data, null, 2))
+
+};
+
+async function replaceIdsWithName(_m, g1){
+    return ":" + ( "§§" + await defnameCache.get(g1) + "§§")
+}
+
+async function getGroupsFromServer(servername, groupsQueries, exportPath, customization, args) {
+    const searchTasks = groupsQueries.map( gp =>
+        ({ title: `(${customization}) Permissions: `.bold + "searching for groups with query: " + gp,
+            task: async ctx => ctx.groupIds = ctx.groupIds
+            ? ctx.groupIds.concat(await getGroups(gp, servername))
+            :  await getGroups(gp, servername) }))
+
+    await new Listr(
+        [
+            ...searchTasks,
+            {
+                title: `(${customization}) Permissions: `.bold + "exporting groups and subsequent roles/permissions",
+                task: ctx => exportGroups(ctx.groupIds, servername, exportPath)
+            },
+        ],{
+            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+            collapse: false
+        })
+        .run()
+}
+
+const defnameCache = {
+    servername: '',
+    resolved: new Map(),
+    get: async function(id) {
+        if(!this.resolved.has(id)){
+            const uri = `https://${this.servername}.cultofbits.com/recordm/recordm/definitions/${id}`
+            const response = await axios.get( uri );
+
+            if(response.status != 200){
+                throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+            }
+
+            this.resolved.set(id, response.data.name)
+        }
+
+        return this.resolved.get(id)
+    }
+}
+
+async function replaceAsync(string, regexp, replacerFunction) {
+    const replacements = await Promise.all(
+        Array.from(string.matchAll(regexp),
+            match => replacerFunction(...match)));
+    let i = 0;
+    return string.replace(regexp, () => replacements[i++]);
+}
+
+exports.getGroupsFromServer = getGroupsFromServer

package/lib/task_lists/updateFromServer_instances.js

@@ -0,0 +1,76 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+
+async function getInstancesFromServer(servername, definitionsForInstances, rootPath, customization, args) {
+
+    const taskList = definitionsForInstances.map( def => ({
+        title: `(${customization}) Instances: `.bold + `Exporting instances from definition ${def.definition}`,
+        task: async () => getInstancesOfDefinition(servername, await getDefinition(servername, def.definition), def.query, def.crop, rootPath)
+    }))
+
+    taskList.push({
+        title: `(${customization}) Instances: `.bold + 'Exporting _index',
+        task: async () => await exportIndex(definitionsForInstances, rootPath + "/_index")
+    })
+
+    await new Listr(taskList , {
+        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+        collapse: false
+    }).run()
+}
+
+function flattenFields(fieldArray) {
+    if (!fieldArray || fieldArray.length == 0)
+        return []
+
+    return fieldArray.flatMap(field => flattenFields(field.fields).concat([field]))
+}
+function getFieldList(fields) {
+    return Array.from(new Set(fields.map(f => f.name))).join(',')
+}
+
+
+async function getInstancesOfDefinition(servername, definition, query, crop, rootPath) {
+
+    const definitionId = definition.id
+    const fields = flattenFields(definition.fieldDefinitions)
+
+    if(!fields.some(f => (f.description || '').includes('$instanceLabel'))){
+        throw new Error("Should not export instances of a Definition without an $instanceLabel field, it would generate duplicates on import")
+    }
+    if(crop && !fields.some(f => (f.description || '').includes('$importer.last'))){
+        throw new Error("Can not crop instances of a Definition without an $importer.last field")
+    }
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/search/download/${definitionId}?`
+    const response = await axios.get(uri + new URLSearchParams({
+        q: query,
+        vcn: "id," + getFieldList(fields)
+    }), {
+        responseType: 'arraybuffer',
+        headers: {'export-files': 'true'}
+    });
+
+    fs.writeFile(rootPath + `${definition.name}.xlsx`, response.data)
+
+}
+
+async function getDefinition(servername, definitionName) {
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/name/${encodeURIComponent(definitionName)}`
+    const response = await axios.get(uri);
+
+    return response.data
+}
+
+async function exportIndex(defs, dataPath) {
+   await fs.writeFile(dataPath, defs.map(d => d.definition + ":" + d.crop).join('\n'), 'utf-8')
+}
+
+
+exports.getInstancesFromServer = getInstancesFromServer

package/lib/task_lists/updateFromServer_kibana.js

@@ -0,0 +1,88 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+const KIBANA_TYPES = ['config','url','index-pattern','query','tag','canvas-element','canvas-workpad','action','alert','visualization','dashboard','map','lens','cases','search'];
+
+async function exportSavedObjectsOfSpace(servername, space, filePath, args) {
+    const uri = `https://${servername}.cultofbits.com/kibana/s/${space.identifier}/api/saved_objects/_export`
+    try {
+        const response = await axios.post(uri, {
+            type: KIBANA_TYPES,
+            excludeExportDetails: true,
+        }, {
+            headers: { 'kbn-xsrf': 'true' },
+            responseType: 'text',
+        });
+
+        const allObjects = response.data
+            .split('\n')
+            .filter(line => line.trim())
+            .map(line => JSON.parse(line));
+
+        await Promise.all(allObjects
+            .filter(e => e.type === 'index-pattern')
+            .map(async element => await replaceIndexPatternsIds(servername, element))
+        )
+
+        await fs.writeFile(filePath, allObjects.map(o => JSON.stringify(o)).join('\n'))
+    } catch(err) {
+        if(args.verbose > 0) {
+            console.error("error downloading from kibana {status:", err.response?.status, ", uri:", uri)
+        }
+        throw err;
+    }
+}
+
+async function exportIndex(spaces, dataPath) {
+   await fs.writeFile(dataPath, spaces.map(s => s.identifier + ":" + s.name).join('\n'), 'utf-8')
+}
+
+
+// replaces in place
+async function replaceIndexPatternsIds(servername, indexPattern) {
+    if(indexPattern.type !== "index-pattern"){
+        throw new Error("Not an index pattern")
+    }
+
+    const definitionId = indexPattern.attributes.title.substring('recordm-'.length)
+    const name = await getDefinitionName(servername, definitionId)
+
+    indexPattern.attributes.title = `recordm-§§${name}§§`
+}
+
+async function getDefinitionName(servername, id) {
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${id}`
+    const response = await axios.get( uri );
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    return response.data.name
+}
+
+async function getKibanaFromServer(servername, spaces, roothpath, customization, args) {
+
+    const taskList = spaces.map( space => (
+        {
+           title: `(${customization}) Kibana: `.bold + `Exporting ${space.name}`,
+           task: async () => await exportSavedObjectsOfSpace(servername, space, roothpath + `/${space.name}.ndjson`, args)
+        }
+    ))
+
+   taskList.push({
+      title: `(${customization}) Kibana: `.bold + 'Exporting _index',
+      task: async () => await exportIndex(spaces, roothpath + "/_index")
+   })
+
+    await new Listr( taskList, {
+        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+        collapse: false
+    }).run()
+}
+
+exports.getKibanaFromServer = getKibanaFromServer

package/lib/task_lists/updateFromServer_validate.js

@@ -8,7 +8,7 @@ function validateUpdateFromServerConditions(cmdEnv) {
     return new Listr([
         {   title: "Check connectivity and permissions".bold,      task: () => checkConnectivity(cmdEnv.server) },
         {   title: "Check there's no cob-cli test' running".bold,  task: () => checkNoTestsRunningOnServer(cmdEnv.server) },
-        {   title: "Check git status".bold,                        task: () => checkWorkingCopyCleanliness() }, 
-   ])
+        {   title: "Check git status".bold,                        task: () => checkWorkingCopyCleanliness() },
+    ])
 }
-exports.validateUpdateFromServerConditions = validateUpdateFromServerConditions
+exports.validateUpdateFromServerConditions = validateUpdateFromServerConditions

package/lib/webpack/webpack.config.js

@@ -36,7 +36,7 @@ module.exports = env => {
       },
       proxy: {
         [ "**/" + process.env.dash_dir + "/**"]: {
-          target: "http://localhost:8041",
+          target: "http://localhost:" + (process.env.dash_port || 8041),
           secure: false
         },
         [ "/docs/**"]: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cob-cli",
-  "version": "2.49.0",
+  "version": "2.50.0",
   "description": "A command line utility to help Cult of Bits partners develop with higher speed and reusing common code and best practices.",
   "preferGlobal": true,
   "repository": {
@@ -18,6 +18,7 @@
   "dependencies": {
     "@webpack-cli/serve": "^2.0.1",
     "axios": "^0.21.1",
+    "axios-cookiejar-support": "^5.0.5",
     "chokidar": "^3.5.3",
     "cmd-line-importer": "^1.0.0",
     "colors": "^1.4.0",
@@ -28,11 +29,14 @@
     "file-cookie-store": "^0.2.1",
     "fs-extra": "^9.0.1",
     "inquirer": "^7.3.3",
+    "js-yaml": "^4.1.0",
     "listr": "^0.14.3",
     "ncp": "^2.0.0",
     "omelette": "^0.4.15-1",
     "open": "^8.4.0",
     "simple-git": "^3.15.1",
+    "tar": "^7.4.3",
+    "tough-cookie": "^5.1.2",
     "webpack": "^5.75.0",
     "webpack-cli": "^5.0.1",
     "webpack-dev-server": "^4.11.1",