cob-cli 2.52.0-beta-1 → 2.52.0

package/lib/task_lists/package_execute.js

@@ -1,45 +0,0 @@
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const verboseRenderer = require('listr-verbose-renderer');
-const { copyFiles } = require("./common_syncFiles");
-const fs = require("fs");
-const tar = require('tar');
-const { format } = require('date-fns');
-
-
-function executeTasks(cmdEnv, args) {
-    console.log("\nPackaging ...");
-
-    return new Listr([
-        {title: "Apply new enviroment specifics", task: () => cmdEnv.applyCurrentCommandEnvironmentChanges() },
-        {title: "Create staging dir",             task: () => fs.existsSync('.staging') || fs.mkdirSync('.staging') },
-        {title: "Copy files to staging dir",      task: () => copyFiles(cmdEnv, "localCopy", "staging")},
-        {title: "Create package",                 task: async () => { const filename = await createPackage(cmdEnv, args); console.log("created " + filename) }},
-        {title: "Sign package",                   task: () => console.log("lololo")},
-        {title: "Delete staging dir",             task: () => fs.rmSync('.staging', {recursive: true, force: true})},
-        {title: "Undo new enviroment specifics",  task: () => cmdEnv.unApplyCurrentCommandEnvironmentChanges() },
-    ], {
-        renderer: args.verbose ? verboseRenderer : UpdaterRenderer,
-        collapse: false
-    });
-}
-exports.executeTasks = executeTasks;
-
-const createPackage = async function(cmdEnv, args){
-   const filename = `${cmdEnv.servername}-${cmdEnv.name}-${format(new Date(), 'yyyy-MM-dd_HH_mm')}.tar.gz`; 
-   const files = fs.readdirSync('.staging')
-   if (args.verbose > 0){
-      console.log("packaging " + files)
-   }
-
-    await tar.create({
-        gzip: true,
-        file: filename,
-        cwd: '.staging',
-        onWriteEntry(entry) {
-            if (args.verbose > 2) console.log('adding', entry.path, entry.stat.mode.toString(8))
-        }
-    }, files)
-
-   return filename;
-}

package/lib/task_lists/ssh.js

@@ -1,30 +0,0 @@
-function getSshArgs(server) {
-    const args = [
-        "-o", "StrictHostKeyChecking=no",
-        "-o", "PreferredAuthentications=publickey",
-        "-o", "BatchMode=yes",
-    ];
-
-    if (process.env.COB_SSH_PORT)     args.push("-p", process.env.COB_SSH_PORT);
-    if (process.env.COB_SSH_IDENTITY) args.push("-i", process.env.COB_SSH_IDENTITY);
-
-    const target = process.env.COB_SSH_USER ? `${process.env.COB_SSH_USER}@${server}` : server;
-    args.push(target);
-
-    return args;
-}
-
-function getServerAddress(server) {
-    return process.env.COB_SSH_USER ? `${process.env.COB_SSH_USER}@${server}` : server;
-}
-
-function getRsyncSsh() {
-    let cmd = "ssh -o ConnectTimeout=30 -o ServerAliveInterval=30 -o ServerAliveCountMax=30 -o StrictHostKeyChecking=no";
-
-    if (process.env.COB_SSH_IDENTITY) cmd += ` -i '${process.env.COB_SSH_IDENTITY}'`;
-    if (process.env.COB_SSH_PORT)     cmd += ` -p ${process.env.COB_SSH_PORT}`;
-
-    return cmd;
-}
-
-module.exports = { getSshArgs, getServerAddress, getRsyncSsh };

package/lib/task_lists/tagDefs_execute.js

@@ -1,121 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const { getFilteredDefinitions } = require('./common_definitions.js')
-const axios = require("axios");
-const inquirer = require("inquirer")
-
-async function getUntaggedChanges(servername, defId) {
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${defId}/changes?tagged=false`
-
-    const response = await axios.get(uri, {params: {tagged: false}});
-
-    if(response.status % 200 > 100){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    return ( response.data || []).filter(c => !c.tag)
-}
-
-async function tagDefinition(servername, defId, until) {
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${defId}/tag`
-
-    const params = {}
-    if(until) {
-        params.until = until
-    }
-   
-    const response = await axios.put(uri, "", {headers: {'Content-Type': 'application/json'}, params: params});
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    return response.data
-}
-
-async function tagDefinitions(servername, customization, defs, args) {
-    return new Listr([
-        {
-            title: `(${customization}) Definitions: `.bold + "searching for definitions matching queries",
-            task: async ctx => ctx.defs = (await getFilteredDefinitions(servername, defs))
-            .map(d => ({id: d.id, name: d.name}))
-        },
-        {
-            title: `(${customization}) Definitions: `.bold + "enriching metadata",
-            task: async ctx => {
-                for (const def of ctx.defs){
-                    def.changes = await getUntaggedChanges(servername, def.id)
-                    if(args.verbose) console.log(`definition ${def.name}: ${def.changes.length} untagged changes`)
-                }
-            }
-        },
-        {
-            title: `(${customization}) Definitions: `.bold + "tagging un-tagged changes",
-            task: async ctx => {
-                const subtasks = ctx.defs.map(d => ({
-                    title: "Tagging changes to " + d.name.bold,
-                    task: async (_, task) => {
-                        if (d.changes.length == 0) {
-                            task.skip("no untagged changes")
-                            return
-                        }
-
-                        if(args.all) {
-                            await tagDefinition(servername, d.id)
-                            task.output = 'Tagged by all'
-                            return
-                        }
-
-                        d.changes.forEach(c => console.log(JSON.stringify(c)))
-
-                        const answer = await inquirer.prompt([
-                            {
-                                type: 'rawlist',
-                                name: 'action',
-                                message: `Do you wish to tag the ${d.changes.length} changes? (^c to abort)`,
-                                choices: [
-                                    'Tag Changes',
-                                    'Choose Changes',
-                                    'Skip',
-                                ]
-                            }
-                        ])
-                        if (answer.action == 'Skip') {
-                            task.skip()
-
-                        } else if (answer.action == 'Tag Changes') {
-                            await tagDefinition(servername, d.id)
-                            task.output = 'Tagged'
-
-                        } else if (answer.action == 'Choose Changes') {
-                           const dates = d.changes.reduce((acc, cur) => {
-                              if(!acc.includes(cur.date)) acc.push(cur.date)
-                              return acc
-                           }, [])
-                           const answerDate = await inquirer.prompt([
-                              {
-                                type: 'rawlist',
-                                name: 'date',
-                                message: `Choose the oldest date to tag`,
-                                choices: dates
-                              }
-                           ])
-                           await tagDefinition(servername, d.id, answerDate.date)
-                           task.output = 'Tagged until ' + answerDate.date
-                        }
-                    }
-                }))
-                return new Listr(subtasks)
-            }
-        },
-    ],{
-        // we need to list changes and read user input, the VerboseRenderer is needed
-        renderer: VerboseRenderer,
-        collapse: false
-    }).run()
-}
-
-
-exports.tagDefinitions = tagDefinitions

package/lib/task_lists/updateFromServer_definitions.js

@@ -1,135 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const { getFilteredDefinitions } = require('./common_definitions.js')
-const axios = require("axios");
-const fs = require("fs/promises");
-
-async function getRefs(servername, def) {
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${def.id}`
-    const response = await axios.get(uri);
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    const fields = response.data.fieldDefinitions;
-    return [...new Set(get$Refs(fields))];
-}
-function get$Refs(fields){
-    let refs = [];
-    for(f of fields){
-        if(f.configuration.keys.Reference){
-            refs.push(f.configuration.keys.Reference.args.definition);
-        }
-        if(f.fields) {
-            const childRefs = get$Refs(f.fields);
-            if(childRefs.length > 0) refs.push(...childRefs)
-        }
-    }
-    return refs
-}
-
-async function exportDefinitionChanges(servername, def, dataPath) {
-
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${def.id}/changes`
-    const response = await axios.get(uri);
-
-    if(response.status == 204){
-        // this is probably an error
-        throw new Error(`Definition "${def.name}" [${def.id}] has no tagged Changes.`);
-
-    } else if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    // write every change on a single line
-    const stringified = "[\n" + response.data.map(change => "  " + JSON.stringify(change)).join(',\n') + "\n]\n";
-
-    await fs.writeFile(dataPath, stringified, 'utf-8');
-}
-
-async function exportDefinitionIdx(defs, dataPath) {
-   await fs.writeFile(dataPath, defs.map(d => d.name).join('\n'), 'utf-8')
-}
-
-async function getViewsForDef(servername, definition, dataPath) {
-    const definitionId = definition.id
-    const uri = `https://${servername}.cultofbits.com/recordm/user/settings/${definition.views}/definitions-${definitionId}/views?`
-
-    const response = await axios.get(uri);
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    let userViews = response.data.filter(v => v.isShared)
-
-    if(response.data.length == 0)
-        return
-
-    userViews = response.data.map(v => {
-        v.value = JSON.stringify( JSON.parse(v.value).filter(f => f.visible == true) )
-        return v
-    })
-
-    const jsonLines = userViews.map(v => JSON.stringify({
-        key: v.key,
-        value: v.value,
-        user: v.user,
-    })) .join('\n');
-
-    await fs.writeFile(dataPath, jsonLines)
-}
-
-function getDefinitionsFromServer(servername, defs, definitionsPath, viewsPath, customization, args) {
-
-    return new Listr(
-        [
-            {
-                title: `(${customization}) Definitions: `.bold + "searching for definitions matching queries",
-                task: async ctx =>  ctx.defs = await getFilteredDefinitions(servername, defs)
-            },
-            {
-                title: `(${customization}) Definitions: `.bold + "enriching metadata",
-                task: async ctx => {
-                    const definitions = await Promise.all(ctx.defs
-                        .map(async d => ({id: d.id, name: d.name, views: d.views, refs: await getRefs(servername, d)}))
-                    )
-
-                    // we'll sort by refs, Defs that reference others must come after them
-                    definitions.sort((a, b) => {
-                        if (a.refs.includes(b.name)){
-                            return 1;
-                        }
-
-                        if(b.refs.includes(a.name)){
-                            return -1;
-                        }
-
-                        return a.refs.length - b.refs.length;
-                    })
-
-                    ctx.defs = definitions
-                }
-            },
-            {
-                title: `(${customization}) Definitions: `.bold + "exporting changes",
-                task: async ctx => {
-                    for( d of ctx.defs ){ await exportDefinitionChanges(servername, d, definitionsPath + `${d.name}.json`)}
-                    await exportDefinitionIdx(ctx.defs, definitionsPath + "_index")
-                }
-            },
-            {
-                title: `(${customization}) Definitions: `.bold + "exporting views",
-                task: async ctx => { for (d of ctx.defs.filter(d => d.views )){ await getViewsForDef(servername, d, viewsPath + `${d.name}.jsonl`)} }
-            },
-        ],{
-            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
-            collapse: false
-        })
-}
-
-
-exports.getDefinitionsFromServer = getDefinitionsFromServer

package/lib/task_lists/updateFromServer_domains.js

@@ -1,50 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const axios = require("axios");
-const fs = require("fs/promises");
-
-async function domainsFromServer(servername, domainsFilter, exportPath, customization, args) {
-    await new Listr(
-        [
-            { title: `(${customization}) Domains: `.bold + "searching for domains matching the queries" ,
-                task: async ctx => ctx.domains = await getDomains(servername, domainsFilter, exportPath) },
-            { title: `(${customization}) Domains: `.bold + "exporting domains " ,
-                task: ctx => ctx.domains.forEach( d => exportDomain(d, exportPath))  }
-        ],{
-            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
-            collapse: false
-        })
-        .run()
-}
-
-function match(query, domain) {
-    // simple matching for now
-    return (domain.name && domain.name.includes(query)) ||
-        (domain.description && domain.description.includes(query) )
-}
-
-async function getDomains(servername, filters, exportPath) {
-
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/domains`
-    const response = await axios.get( uri );
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    const matchesAny = (domain) => filters.filter(q => match(q, domain)).length > 0;
-
-    return response.data.filter( matchesAny )
-}
-
-async function exportDomain(domain, rootPath) {
-    const d = {
-        name: domain.name,
-        definitions: domain.definitions.map(d => d.name).sort()
-    }
-    await fs.writeFile(rootPath + domain.name + ".json", JSON.stringify(d, null, 2) )
-}
-
-exports.getDomainsFromServer = domainsFromServer

package/lib/task_lists/updateFromServer_groups.js

@@ -1,111 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const axios = require("axios");
-const fs = require("fs/promises");
-
-async function getGroups(query, servername) {
-
-    const uri = `https://${servername}.cultofbits.com/userm/userm/search/userm/group?`
-    const response = await axios.get( uri + new URLSearchParams({
-        q: query,
-        size: "100" // probably excessive
-    }));
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-    const groupIds = response.data.hits.hits.map((h) => h._id)
-    return groupIds;
-};
-
-const sortByName = (a, b) => a.name?.localeCompare(b.name);
-
-const exportGroups = async function(ids, servername, exportPath){
-
-    const uri = `https://${servername}.cultofbits.com/userm/userm/group/export/${ids.join(',')}`
-
-    const response = await axios.get( uri );
-    if(response.status != 200 ){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-    const data = response.data;
-
-    // sort everything to be easier to diff
-    data.groups?.sort(sortByName);
-    data.groups?.forEach(group => group.roles?.sort())
-    data.roles?.sort(sortByName);
-    data.roles?.forEach(role => role.perms?.sort())
-    data.perms?.sort(sortByName)
-    const re = /:(\d+)/g
-    defnameCache.servername = servername;
-
-    for(r of data.roles) {
-        for(let i = 0; i < r.perms.length; i++){
-            r.perms[i] = await replaceAsync(r.perms[i], re, replaceIdsWithName)
-        }
-    }
-    for(p of data.perms){
-        if( p.product == 'recordm') {
-            p.name = await replaceAsync(p.name, re, replaceIdsWithName)
-        }
-    }
-
-    fs.writeFile(exportPath, JSON.stringify(data, null, 2))
-
-};
-
-async function replaceIdsWithName(_m, g1){
-    return ":" + ( "§§" + await defnameCache.get(g1) + "§§")
-}
-
-async function getGroupsFromServer(servername, groupsQueries, exportPath, customization, args) {
-    const searchTasks = groupsQueries.map( gp =>
-        ({ title: `(${customization}) Permissions: `.bold + "searching for groups with query: " + gp,
-            task: async ctx => ctx.groupIds = ctx.groupIds
-            ? ctx.groupIds.concat(await getGroups(gp, servername))
-            :  await getGroups(gp, servername) }))
-
-    await new Listr(
-        [
-            ...searchTasks,
-            {
-                title: `(${customization}) Permissions: `.bold + "exporting groups and subsequent roles/permissions",
-                task: ctx => exportGroups(ctx.groupIds, servername, exportPath)
-            },
-        ],{
-            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
-            collapse: false
-        })
-        .run()
-}
-
-const defnameCache = {
-    servername: '',
-    resolved: new Map(),
-    get: async function(id) {
-        if(!this.resolved.has(id)){
-            const uri = `https://${this.servername}.cultofbits.com/recordm/recordm/definitions/${id}`
-            const response = await axios.get( uri );
-
-            if(response.status != 200){
-                throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-            }
-
-            this.resolved.set(id, response.data.name)
-        }
-
-        return this.resolved.get(id)
-    }
-}
-
-async function replaceAsync(string, regexp, replacerFunction) {
-    const replacements = await Promise.all(
-        Array.from(string.matchAll(regexp),
-            match => replacerFunction(...match)));
-    let i = 0;
-    return string.replace(regexp, () => replacements[i++]);
-}
-
-exports.getGroupsFromServer = getGroupsFromServer

package/lib/task_lists/updateFromServer_instances.js

@@ -1,76 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const axios = require("axios");
-const fs = require("fs/promises");
-
-
-async function getInstancesFromServer(servername, definitionsForInstances, rootPath, customization, args) {
-
-    const taskList = definitionsForInstances.map( def => ({
-        title: `(${customization}) Instances: `.bold + `Exporting instances from definition ${def.definition}`,
-        task: async () => getInstancesOfDefinition(servername, await getDefinition(servername, def.definition), def.query, def.crop, rootPath)
-    }))
-
-    taskList.push({
-        title: `(${customization}) Instances: `.bold + 'Exporting _index',
-        task: async () => await exportIndex(definitionsForInstances, rootPath + "/_index")
-    })
-
-    await new Listr(taskList , {
-        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
-        collapse: false
-    }).run()
-}
-
-function flattenFields(fieldArray) {
-    if (!fieldArray || fieldArray.length == 0)
-        return []
-
-    return fieldArray.flatMap(field => flattenFields(field.fields).concat([field]))
-}
-function getFieldList(fields) {
-    return Array.from(new Set(fields.map(f => f.name))).join(',')
-}
-
-
-async function getInstancesOfDefinition(servername, definition, query, crop, rootPath) {
-
-    const definitionId = definition.id
-    const fields = flattenFields(definition.fieldDefinitions)
-
-    if(!fields.some(f => (f.description || '').includes('$instanceLabel'))){
-        throw new Error("Should not export instances of a Definition without an $instanceLabel field, it would generate duplicates on import")
-    }
-    if(crop && !fields.some(f => (f.description || '').includes('$importer.last'))){
-        throw new Error("Can not crop instances of a Definition without an $importer.last field")
-    }
-
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/search/download/${definitionId}?`
-    const response = await axios.get(uri + new URLSearchParams({
-        q: query,
-        vcn: "id," + getFieldList(fields)
-    }), {
-        responseType: 'arraybuffer',
-        headers: {'export-files': 'true'}
-    });
-
-    fs.writeFile(rootPath + `${definition.name}.xlsx`, response.data)
-
-}
-
-async function getDefinition(servername, definitionName) {
-
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/name/${encodeURIComponent(definitionName)}`
-    const response = await axios.get(uri);
-
-    return response.data
-}
-
-async function exportIndex(defs, dataPath) {
-   await fs.writeFile(dataPath, defs.map(d => d.definition + ":" + d.crop).join('\n'), 'utf-8')
-}
-
-
-exports.getInstancesFromServer = getInstancesFromServer

package/lib/task_lists/updateFromServer_kibana.js

@@ -1,88 +0,0 @@
-require('colors');
-const Listr = require('listr');
-const UpdaterRenderer = require('listr-update-renderer');
-const VerboseRenderer = require('listr-verbose-renderer');
-const axios = require("axios");
-const fs = require("fs/promises");
-
-const KIBANA_TYPES = ['config','url','index-pattern','query','tag','canvas-element','canvas-workpad','action','alert','visualization','dashboard','map','lens','cases','search'];
-
-async function exportSavedObjectsOfSpace(servername, space, filePath, args) {
-    const uri = `https://${servername}.cultofbits.com/kibana/s/${space.identifier}/api/saved_objects/_export`
-    try {
-        const response = await axios.post(uri, {
-            type: KIBANA_TYPES,
-            excludeExportDetails: true,
-        }, {
-            headers: { 'kbn-xsrf': 'true' },
-            responseType: 'text',
-        });
-
-        const allObjects = response.data
-            .split('\n')
-            .filter(line => line.trim())
-            .map(line => JSON.parse(line));
-
-        await Promise.all(allObjects
-            .filter(e => e.type === 'index-pattern')
-            .map(async element => await replaceIndexPatternsIds(servername, element))
-        )
-
-        await fs.writeFile(filePath, allObjects.map(o => JSON.stringify(o)).join('\n'))
-    } catch(err) {
-        if(args.verbose > 0) {
-            console.error("error downloading from kibana {status:", err.response?.status, ", uri:", uri)
-        }
-        throw err;
-    }
-}
-
-async function exportIndex(spaces, dataPath) {
-   await fs.writeFile(dataPath, spaces.map(s => s.identifier + ":" + s.name).join('\n'), 'utf-8')
-}
-
-
-// replaces in place
-async function replaceIndexPatternsIds(servername, indexPattern) {
-    if(indexPattern.type !== "index-pattern"){
-        throw new Error("Not an index pattern")
-    }
-
-    const definitionId = indexPattern.attributes.title.substring('recordm-'.length)
-    const name = await getDefinitionName(servername, definitionId)
-
-    indexPattern.attributes.title = `recordm-§§${name}§§`
-}
-
-async function getDefinitionName(servername, id) {
-    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${id}`
-    const response = await axios.get( uri );
-
-    if(response.status != 200){
-        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
-    }
-
-    return response.data.name
-}
-
-async function getKibanaFromServer(servername, spaces, roothpath, customization, args) {
-
-    const taskList = spaces.map( space => (
-        {
-           title: `(${customization}) Kibana: `.bold + `Exporting ${space.name}`,
-           task: async () => await exportSavedObjectsOfSpace(servername, space, roothpath + `/${space.name}.ndjson`, args)
-        }
-    ))
-
-   taskList.push({
-      title: `(${customization}) Kibana: `.bold + 'Exporting _index',
-      task: async () => await exportIndex(spaces, roothpath + "/_index")
-   })
-
-    await new Listr( taskList, {
-        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
-        collapse: false
-    }).run()
-}
-
-exports.getKibanaFromServer = getKibanaFromServer