cob-cli 2.51.0 → 2.52.0-beta-2

package/lib/task_lists/test_otherFilesContiousReload.js

@@ -7,6 +7,7 @@ const { SERVER_COB_CLI_DIRECTORY } = require("./common_helpers");
 const { confirmExecutionOfChanges, resolveCobPath } = require("./common_syncFiles");
 const { syncFile } = require("./test_syncFile");
 const { getLastDeployedSha } = require("./common_releaseManager");
+const { getSshArgs } = require("./common_helpers");
 
 const IN_PROGRESS_TEST_FILE = "test.in.execution"
 const SERVER_IN_PROGRESS_TEST_FILE = SERVER_COB_CLI_DIRECTORY + IN_PROGRESS_TEST_FILE
@@ -18,7 +19,7 @@ async function otherFilesContiousReload(cmdEnv) {
     let changedFiles = new Set()
 
     let currentTestsFilesList = []
-    await execa('ssh', [cmdEnv.server, "cat " + SERVER_IN_PROGRESS_TEST_FILE])
+    await execa('ssh', [...getSshArgs(cmdEnv.server), "cat " + SERVER_IN_PROGRESS_TEST_FILE])
     .then( value => currentTestsFilesList = value.stdout.split("\n").map(l => l.substring(0,l.indexOf(" "))) )
     .catch( () => { } ) /* grep returns error if no match and execa throws an excepcions when return is error */
     
@@ -29,22 +30,7 @@ async function otherFilesContiousReload(cmdEnv) {
     let restoreChanges = async () => {
         if(watcher.process) watcher.process.close();
         if(changedFiles.size > 0) {
-            console.log("\nRestoring changed files...".yellow);
-            let stash = true;
-            await git().env('LC_ALL', 'C').stash(["--include-untracked"]).then( value => value.indexOf("Saved") == 0 || (stash = false))
-            let lastSha = await getLastDeployedSha(cmdEnv.server)
-            await git().checkout(lastSha)
-            
-            await cmdEnv.applyLastEnvironmentDeployedToServerChanges()
-            for(let changedFile of changedFiles) {
-                await removeFileFromCurrentTest(cmdEnv.server, changedFile);
-            }
-            await cmdEnv.unApplyLastEnvironmentDeployedToServerChanges()
-
-            await execa('ssh', [cmdEnv.server,"find " + SERVER_IN_PROGRESS_TEST_FILE + " -size 0 -delete "]) //Remove file if empty
-
-            await git().checkout("-")
-            if (stash) await git().stash(["pop"])
+            await restoreServerFilesToLastDeployed(cmdEnv, changedFiles);
         }
         fs.unlinkSync("."+IN_PROGRESS_TEST_FILE)
     }
@@ -71,7 +57,7 @@ async function otherFilesContiousReload(cmdEnv) {
         let productDir = resolveCobPath(cmdEnv.server, "serverLive", product).split(":")[1];
         let productScopeFilePath = changedFile.substring(changedFile.indexOf("/") + 1); // remove o directorio inicial do producto
 
-        let isRemoteFile = () => execa('ssh', [cmdEnv.server, "[[ -f " + productDir + productScopeFilePath + " ]] && echo 'file' || echo '' " ])
+        let isRemoteFile = () => execa('ssh', [...getSshArgs(cmdEnv.server), "[[ -f " + productDir + productScopeFilePath + " ]] && echo 'file' || echo '' " ])
             .then( r => {
                 return r.stdout == "file"
             }).catch( () => {
@@ -114,10 +100,45 @@ async function otherFilesContiousReload(cmdEnv) {
 }
 exports.otherFilesContiousReload = otherFilesContiousReload
 
+/* ************************************ */
+async function restoreServerFilesToLastDeployed(cmdEnv, changedFiles) {
+    console.log("\nRestoring changed files...".yellow);
+
+    // Clean up any env-swap markers left by applyCurrentCommandEnvironmentChanges before
+    // stashing. These files are gitignored, so git stash --include-untracked ignores them.
+    // If they survive into _undoSpecific(lastEnv) they create ghost env files that are
+    // untracked at lastSha, causing "git checkout -" to abort.
+    await cmdEnv.unApplyCurrentCommandEnvironmentChanges();
+
+    let stashed = false;
+    await git().env('LC_ALL', 'C').stash(["--include-untracked"])
+        .then(value => { stashed = value.indexOf("Saved") === 0; })
+        .catch(() => {});
+
+    const lastSha = await getLastDeployedSha(cmdEnv.server);
+    if (!lastSha) throw new Error("Cannot restore server files: no previous deploy found on server.");
+    await git().checkout(lastSha);
+
+    await cmdEnv.applyLastEnvironmentDeployedToServerChanges();
+
+    for (const changedFile of changedFiles) {
+        await removeFileFromCurrentTest(cmdEnv.server, changedFile);
+    }
+
+    await cmdEnv.unApplyLastEnvironmentDeployedToServerChanges();
+
+    await execa('ssh', [...getSshArgs(cmdEnv.server), "find " + SERVER_IN_PROGRESS_TEST_FILE + " -size 0 -delete"])
+        .catch(() => {});
+
+    await git().checkout("-");
+    if (stashed) await git().stash(["pop"]);
+}
+exports.restoreServerFilesToLastDeployed = restoreServerFilesToLastDeployed
+
 /* ************************************ */
 async function addFileToCurrentTest(server, changedFile, changedFiles, changeType, restoreChanges) {
     let inUse = "";
-    await execa('ssh', [server, "grep -F '" + changedFile + " ' " + SERVER_IN_PROGRESS_TEST_FILE ])
+    await execa('ssh', [...getSshArgs(server), "grep -F '" + changedFile + " ' " + SERVER_IN_PROGRESS_TEST_FILE ])
         .then( value => inUse = value.stdout )
         .catch( () => { } ) /* grep returns error if no match and execa throws an excepcions when return is error */
         .finally( async () => {
@@ -133,7 +154,7 @@ async function addFileToCurrentTest(server, changedFile, changedFiles, changeTyp
 
             if(!changedFiles.has(changedFile)) {
                 const user = require("os").userInfo().username;
-                execa('ssh', [server, "echo '" + changedFile + " by " + user + "' >> " + SERVER_IN_PROGRESS_TEST_FILE]);
+                execa('ssh', [...getSshArgs(server), "echo '" + changedFile + " by " + user + "' >> " + SERVER_IN_PROGRESS_TEST_FILE]);
                 changedFiles.add(changedFile);
             }
             console.log(" " + changeType.replace(/(e)([^e]*)$/, "ed$2 ") + changedFile);
@@ -143,13 +164,14 @@ async function addFileToCurrentTest(server, changedFile, changedFiles, changeTyp
 /* ************************************ */
 async function removeFileFromCurrentTest(server, changedFile) {
     await syncFile(server, changedFile);
-    await execa('ssh', [server, "sed -i '/" + changedFile.split("/").join("\\/") + "/d'  " + SERVER_IN_PROGRESS_TEST_FILE]).catch( () => {/* ignore files removed by user */} );
+    await execa('ssh', [...getSshArgs(server), "sed -i '/" + changedFile.split("/").join("\\/") + "/d'  " + SERVER_IN_PROGRESS_TEST_FILE]).catch( () => {/* ignore files removed by user */} );
     console.log(" reset ".brightGreen + changedFile);
 }
+exports.removeFileFromCurrentTest = removeFileFromCurrentTest
 
 /* ************************************ */
 function checkNoTestsRunningOnServer(server) {
-    return execa('ssh', [server, "cat " + SERVER_IN_PROGRESS_TEST_FILE])
+    return execa('ssh', [...getSshArgs(server), "cat " + SERVER_IN_PROGRESS_TEST_FILE])
             .then(value => {
                 throw new Error(
                         "Aborted:".red + " there are tests running with files changed on the server:\n  "

package/lib/task_lists/test_syncFile.js

@@ -2,6 +2,7 @@ const execa = require('execa');
 const fs = require('fs-extra');
 const path = require('path');
 const { resolveCobPath } = require("./common_syncFiles");
+const { getSshArgs, getRsyncSsh } = require("./common_helpers");
 
 const DEBUG = false;
 const RSYNC_RETRIES = 10;
@@ -18,7 +19,7 @@ async function syncFile(server, localFile) {
 
     let count = 0;
     while (count < RSYNC_RETRIES) {
-        await execa('ssh', [server, "mkdir -p " + remoteFileDir]).catch({});
+        await execa('ssh', [...getSshArgs(server), "mkdir -p " + remoteFileDir]).catch({});
         await execa('rsync', [localFile, remoteProductPath + productScopeFilePath,
             "-acz",
             "--prune-empty-dirs",
@@ -28,17 +29,17 @@ async function syncFile(server, localFile) {
             "--no-t",
             "--filter='merge " + path.resolve(__dirname,"rsyncFilter-pre.txt") + "'",
             "--filter='merge " + path.resolve(__dirname,"rsyncFilter-post.txt") + "'"],
-           { shell: true, env: { "RSYNC_RSH": "ssh -o ConnectTimeout=30 -o ServerAliveInterval=30 -o ServerAliveCountMax=30" } })
+           { shell: true, env: { "RSYNC_RSH": getRsyncSsh() } })
             .then(count = RSYNC_RETRIES)
             .catch(async (err) => {
                 // exitCode 23 indicam que o ficheiro não existe na origem. Temod de apagar no destino.
                 if (err.exitCode == 23) {
-                    await execa('ssh', [server, "rm -fd " + remoteProductPath.split(":")[1] + productScopeFilePath]).catch({});
+                    await execa('ssh', [...getSshArgs(server), "rm -fd " + remoteProductPath.split(":")[1] + productScopeFilePath]).catch({});
 
                     let localFileDir = localFile.substring(0, localFile.lastIndexOf("/"));
                     if (!fs.existsSync(localFileDir)) {
                         if (DEBUG) console.log(remoteFileDir + " needs to be removed");
-                        await execa('ssh', [server, "rm -rf " + remoteFileDir]).catch({});
+                        await execa('ssh', [...getSshArgs(server), "rm -rf " + remoteFileDir]).catch({});
                     }
                     count = RSYNC_RETRIES;
                 } else if ((count + 1)> RSYNC_RETRIES) {

package/lib/task_lists/test_validate.js

@@ -9,6 +9,7 @@ const { checkConnectivity } = require("./common_helpers");
 const { getCurrentBranch, getLastDeployedSha } = require("./common_releaseManager");
 const { testEquality } = require("./common_syncFiles");
 const { SERVER_IN_PROGRESS_TEST_FILE } = require("../task_lists/test_otherFilesContiousReload");
+const { getSshArgs } = require("./common_helpers");
 
 
 async function validateTestingConditions(cmdEnv, args) {
@@ -79,7 +80,7 @@ async function _handleTestEquality(ctx, changes, server) {
         let offendingFiles = changes.map(f => f.substring(17)) //17 é a posição onde começa o nome dos ficheiros em cada linha
         let problemFiles = [];
         for(let changedFile of offendingFiles) {
-            await execa('ssh', [server, "grep -F " + changedFile + " " + SERVER_IN_PROGRESS_TEST_FILE ])
+            await execa('ssh', [...getSshArgs(server), "grep -F " + changedFile + " " + SERVER_IN_PROGRESS_TEST_FILE ])
             .catch( () => { 
                 try {
                     if (fs.lstatSync(changedFile.trim()).isFile()) {
@@ -94,7 +95,7 @@ async function _handleTestEquality(ctx, changes, server) {
         if(problemFiles.length > 0)  throw new Error("Aborted:".red + " server was changed since last deploy:\n   " + problemFiles.join("\n   ") +  "\n" );
 
         let inTest = "";
-        await execa('ssh', [server, "cat " + SERVER_IN_PROGRESS_TEST_FILE])
+        await execa('ssh', [...getSshArgs(server), "cat " + SERVER_IN_PROGRESS_TEST_FILE])
         .then( value => inTest = value.stdout )
         .catch( () => { } ) /* grep returns error if no match and execa throws an excepcions when return is error */
         .finally( async () => {

package/lib/task_lists/updateFromServer_definitions.js

@@ -0,0 +1,135 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const { getFilteredDefinitions } = require('./common_definitions.js')
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function getRefs(servername, def) {
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${def.id}`
+    const response = await axios.get(uri);
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    const fields = response.data.fieldDefinitions;
+    return [...new Set(get$Refs(fields))];
+}
+function get$Refs(fields){
+    let refs = [];
+    for(f of fields){
+        if(f.configuration.keys.Reference){
+            refs.push(f.configuration.keys.Reference.args.definition);
+        }
+        if(f.fields) {
+            const childRefs = get$Refs(f.fields);
+            if(childRefs.length > 0) refs.push(...childRefs)
+        }
+    }
+    return refs
+}
+
+async function exportDefinitionChanges(servername, def, dataPath) {
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${def.id}/changes`
+    const response = await axios.get(uri);
+
+    if(response.status == 204){
+        // this is probably an error
+        throw new Error(`Definition "${def.name}" [${def.id}] has no tagged Changes.`);
+
+    } else if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    // write every change on a single line
+    const stringified = "[\n" + response.data.map(change => "  " + JSON.stringify(change)).join(',\n') + "\n]\n";
+
+    await fs.writeFile(dataPath, stringified, 'utf-8');
+}
+
+async function exportDefinitionIdx(defs, dataPath) {
+   await fs.writeFile(dataPath, defs.map(d => d.name).join('\n'), 'utf-8')
+}
+
+async function getViewsForDef(servername, definition, dataPath) {
+    const definitionId = definition.id
+    const uri = `https://${servername}.cultofbits.com/recordm/user/settings/${definition.views}/definitions-${definitionId}/views?`
+
+    const response = await axios.get(uri);
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    let userViews = response.data.filter(v => v.isShared)
+
+    if(response.data.length == 0)
+        return
+
+    userViews = response.data.map(v => {
+        v.value = JSON.stringify( JSON.parse(v.value).filter(f => f.visible == true) )
+        return v
+    })
+
+    const jsonLines = userViews.map(v => JSON.stringify({
+        key: v.key,
+        value: v.value,
+        user: v.user,
+    })) .join('\n');
+
+    await fs.writeFile(dataPath, jsonLines)
+}
+
+function getDefinitionsFromServer(servername, defs, definitionsPath, viewsPath, customization, args) {
+
+    return new Listr(
+        [
+            {
+                title: `(${customization}) Definitions: `.bold + "searching for definitions matching queries",
+                task: async ctx =>  ctx.defs = await getFilteredDefinitions(servername, defs)
+            },
+            {
+                title: `(${customization}) Definitions: `.bold + "enriching metadata",
+                task: async ctx => {
+                    const definitions = await Promise.all(ctx.defs
+                        .map(async d => ({id: d.id, name: d.name, views: d.views, refs: await getRefs(servername, d)}))
+                    )
+
+                    // we'll sort by refs, Defs that reference others must come after them
+                    definitions.sort((a, b) => {
+                        if (a.refs.includes(b.name)){
+                            return 1;
+                        }
+
+                        if(b.refs.includes(a.name)){
+                            return -1;
+                        }
+
+                        return a.refs.length - b.refs.length;
+                    })
+
+                    ctx.defs = definitions
+                }
+            },
+            {
+                title: `(${customization}) Definitions: `.bold + "exporting changes",
+                task: async ctx => {
+                    for( d of ctx.defs ){ await exportDefinitionChanges(servername, d, definitionsPath + `${d.name}.json`)}
+                    await exportDefinitionIdx(ctx.defs, definitionsPath + "_index")
+                }
+            },
+            {
+                title: `(${customization}) Definitions: `.bold + "exporting views",
+                task: async ctx => { for (d of ctx.defs.filter(d => d.views )){ await getViewsForDef(servername, d, viewsPath + `${d.name}.jsonl`)} }
+            },
+        ],{
+            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+            collapse: false
+        })
+}
+
+
+exports.getDefinitionsFromServer = getDefinitionsFromServer

package/lib/task_lists/updateFromServer_domains.js

@@ -0,0 +1,50 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function domainsFromServer(servername, domainsFilter, exportPath, customization, args) {
+    await new Listr(
+        [
+            { title: `(${customization}) Domains: `.bold + "searching for domains matching the queries" ,
+                task: async ctx => ctx.domains = await getDomains(servername, domainsFilter, exportPath) },
+            { title: `(${customization}) Domains: `.bold + "exporting domains " ,
+                task: ctx => ctx.domains.forEach( d => exportDomain(d, exportPath))  }
+        ],{
+            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+            collapse: false
+        })
+        .run()
+}
+
+function match(query, domain) {
+    // simple matching for now
+    return (domain.name && domain.name.includes(query)) ||
+        (domain.description && domain.description.includes(query) )
+}
+
+async function getDomains(servername, filters, exportPath) {
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/domains`
+    const response = await axios.get( uri );
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    const matchesAny = (domain) => filters.filter(q => match(q, domain)).length > 0;
+
+    return response.data.filter( matchesAny )
+}
+
+async function exportDomain(domain, rootPath) {
+    const d = {
+        name: domain.name,
+        definitions: domain.definitions.map(d => d.name).sort()
+    }
+    await fs.writeFile(rootPath + domain.name + ".json", JSON.stringify(d, null, 2) )
+}
+
+exports.getDomainsFromServer = domainsFromServer

package/lib/task_lists/updateFromServer_groups.js

@@ -0,0 +1,111 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+async function getGroups(query, servername) {
+
+    const uri = `https://${servername}.cultofbits.com/userm/userm/search/userm/group?`
+    const response = await axios.get( uri + new URLSearchParams({
+        q: query,
+        size: "100" // probably excessive
+    }));
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+    const groupIds = response.data.hits.hits.map((h) => h._id)
+    return groupIds;
+};
+
+const sortByName = (a, b) => a.name?.localeCompare(b.name);
+
+const exportGroups = async function(ids, servername, exportPath){
+
+    const uri = `https://${servername}.cultofbits.com/userm/userm/group/export/${ids.join(',')}`
+
+    const response = await axios.get( uri );
+    if(response.status != 200 ){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+    const data = response.data;
+
+    // sort everything to be easier to diff
+    data.groups?.sort(sortByName);
+    data.groups?.forEach(group => group.roles?.sort())
+    data.roles?.sort(sortByName);
+    data.roles?.forEach(role => role.perms?.sort())
+    data.perms?.sort(sortByName)
+    const re = /:(\d+)/g
+    defnameCache.servername = servername;
+
+    for(r of data.roles) {
+        for(let i = 0; i < r.perms.length; i++){
+            r.perms[i] = await replaceAsync(r.perms[i], re, replaceIdsWithName)
+        }
+    }
+    for(p of data.perms){
+        if( p.product == 'recordm') {
+            p.name = await replaceAsync(p.name, re, replaceIdsWithName)
+        }
+    }
+
+    fs.writeFile(exportPath, JSON.stringify(data, null, 2))
+
+};
+
+async function replaceIdsWithName(_m, g1){
+    return ":" + ( "§§" + await defnameCache.get(g1) + "§§")
+}
+
+async function getGroupsFromServer(servername, groupsQueries, exportPath, customization, args) {
+    const searchTasks = groupsQueries.map( gp =>
+        ({ title: `(${customization}) Permissions: `.bold + "searching for groups with query: " + gp,
+            task: async ctx => ctx.groupIds = ctx.groupIds
+            ? ctx.groupIds.concat(await getGroups(gp, servername))
+            :  await getGroups(gp, servername) }))
+
+    await new Listr(
+        [
+            ...searchTasks,
+            {
+                title: `(${customization}) Permissions: `.bold + "exporting groups and subsequent roles/permissions",
+                task: ctx => exportGroups(ctx.groupIds, servername, exportPath)
+            },
+        ],{
+            renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+            collapse: false
+        })
+        .run()
+}
+
+const defnameCache = {
+    servername: '',
+    resolved: new Map(),
+    get: async function(id) {
+        if(!this.resolved.has(id)){
+            const uri = `https://${this.servername}.cultofbits.com/recordm/recordm/definitions/${id}`
+            const response = await axios.get( uri );
+
+            if(response.status != 200){
+                throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+            }
+
+            this.resolved.set(id, response.data.name)
+        }
+
+        return this.resolved.get(id)
+    }
+}
+
+async function replaceAsync(string, regexp, replacerFunction) {
+    const replacements = await Promise.all(
+        Array.from(string.matchAll(regexp),
+            match => replacerFunction(...match)));
+    let i = 0;
+    return string.replace(regexp, () => replacements[i++]);
+}
+
+exports.getGroupsFromServer = getGroupsFromServer

package/lib/task_lists/updateFromServer_instances.js

@@ -0,0 +1,76 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+
+async function getInstancesFromServer(servername, definitionsForInstances, rootPath, customization, args) {
+
+    const taskList = definitionsForInstances.map( def => ({
+        title: `(${customization}) Instances: `.bold + `Exporting instances from definition ${def.definition}`,
+        task: async () => getInstancesOfDefinition(servername, await getDefinition(servername, def.definition), def.query, def.crop, rootPath)
+    }))
+
+    taskList.push({
+        title: `(${customization}) Instances: `.bold + 'Exporting _index',
+        task: async () => await exportIndex(definitionsForInstances, rootPath + "/_index")
+    })
+
+    await new Listr(taskList , {
+        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+        collapse: false
+    }).run()
+}
+
+function flattenFields(fieldArray) {
+    if (!fieldArray || fieldArray.length == 0)
+        return []
+
+    return fieldArray.flatMap(field => flattenFields(field.fields).concat([field]))
+}
+function getFieldList(fields) {
+    return Array.from(new Set(fields.map(f => f.name))).join(',')
+}
+
+
+async function getInstancesOfDefinition(servername, definition, query, crop, rootPath) {
+
+    const definitionId = definition.id
+    const fields = flattenFields(definition.fieldDefinitions)
+
+    if(!fields.some(f => (f.description || '').includes('$instanceLabel'))){
+        throw new Error("Should not export instances of a Definition without an $instanceLabel field, it would generate duplicates on import")
+    }
+    if(crop && !fields.some(f => (f.description || '').includes('$importer.last'))){
+        throw new Error("Can not crop instances of a Definition without an $importer.last field")
+    }
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/search/download/${definitionId}?`
+    const response = await axios.get(uri + new URLSearchParams({
+        q: query,
+        vcn: "id," + getFieldList(fields)
+    }), {
+        responseType: 'arraybuffer',
+        headers: {'export-files': 'true'}
+    });
+
+    fs.writeFile(rootPath + `${definition.name}.xlsx`, response.data)
+
+}
+
+async function getDefinition(servername, definitionName) {
+
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/name/${encodeURIComponent(definitionName)}`
+    const response = await axios.get(uri);
+
+    return response.data
+}
+
+async function exportIndex(defs, dataPath) {
+   await fs.writeFile(dataPath, defs.map(d => d.definition + ":" + d.crop).join('\n'), 'utf-8')
+}
+
+
+exports.getInstancesFromServer = getInstancesFromServer

package/lib/task_lists/updateFromServer_kibana.js

@@ -0,0 +1,88 @@
+require('colors');
+const Listr = require('listr');
+const UpdaterRenderer = require('listr-update-renderer');
+const VerboseRenderer = require('listr-verbose-renderer');
+const axios = require("axios");
+const fs = require("fs/promises");
+
+const KIBANA_TYPES = ['config','url','index-pattern','query','tag','canvas-element','canvas-workpad','action','alert','visualization','dashboard','map','lens','cases','search'];
+
+async function exportSavedObjectsOfSpace(servername, space, filePath, args) {
+    const uri = `https://${servername}.cultofbits.com/kibana/s/${space.identifier}/api/saved_objects/_export`
+    try {
+        const response = await axios.post(uri, {
+            type: KIBANA_TYPES,
+            excludeExportDetails: true,
+        }, {
+            headers: { 'kbn-xsrf': 'true' },
+            responseType: 'text',
+        });
+
+        const allObjects = response.data
+            .split('\n')
+            .filter(line => line.trim())
+            .map(line => JSON.parse(line));
+
+        await Promise.all(allObjects
+            .filter(e => e.type === 'index-pattern')
+            .map(async element => await replaceIndexPatternsIds(servername, element))
+        )
+
+        await fs.writeFile(filePath, allObjects.map(o => JSON.stringify(o)).join('\n'))
+    } catch(err) {
+        if(args.verbose > 0) {
+            console.error("error downloading from kibana {status:", err.response?.status, ", uri:", uri)
+        }
+        throw err;
+    }
+}
+
+async function exportIndex(spaces, dataPath) {
+   await fs.writeFile(dataPath, spaces.map(s => s.identifier + ":" + s.name).join('\n'), 'utf-8')
+}
+
+
+// replaces in place
+async function replaceIndexPatternsIds(servername, indexPattern) {
+    if(indexPattern.type !== "index-pattern"){
+        throw new Error("Not an index pattern")
+    }
+
+    const definitionId = indexPattern.attributes.title.substring('recordm-'.length)
+    const name = await getDefinitionName(servername, definitionId)
+
+    indexPattern.attributes.title = `recordm-§§${name}§§`
+}
+
+async function getDefinitionName(servername, id) {
+    const uri = `https://${servername}.cultofbits.com/recordm/recordm/definitions/${id}`
+    const response = await axios.get( uri );
+
+    if(response.status != 200){
+        throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    }
+
+    return response.data.name
+}
+
+async function getKibanaFromServer(servername, spaces, roothpath, customization, args) {
+
+    const taskList = spaces.map( space => (
+        {
+           title: `(${customization}) Kibana: `.bold + `Exporting ${space.name}`,
+           task: async () => await exportSavedObjectsOfSpace(servername, space, roothpath + `/${space.name}.ndjson`, args)
+        }
+    ))
+
+   taskList.push({
+      title: `(${customization}) Kibana: `.bold + 'Exporting _index',
+      task: async () => await exportIndex(spaces, roothpath + "/_index")
+   })
+
+    await new Listr( taskList, {
+        renderer: args.verbose ? VerboseRenderer : UpdaterRenderer,
+        collapse: false
+    }).run()
+}
+
+exports.getKibanaFromServer = getKibanaFromServer

package/lib/task_lists/updateFromServer_validate.js

@@ -8,7 +8,7 @@ function validateUpdateFromServerConditions(cmdEnv) {
     return new Listr([
         {   title: "Check connectivity and permissions".bold,      task: () => checkConnectivity(cmdEnv.server) },
         {   title: "Check there's no cob-cli test' running".bold,  task: () => checkNoTestsRunningOnServer(cmdEnv.server) },
-        {   title: "Check git status".bold,                        task: () => checkWorkingCopyCleanliness() }, 
-   ])
+        {   title: "Check git status".bold,                        task: () => checkWorkingCopyCleanliness() },
+    ])
 }
-exports.validateUpdateFromServerConditions = validateUpdateFromServerConditions
+exports.validateUpdateFromServerConditions = validateUpdateFromServerConditions

package/lib/webpack/webpack.config.js

@@ -36,7 +36,7 @@ module.exports = env => {
       },
       proxy: {
         [ "**/" + process.env.dash_dir + "/**"]: {
-          target: "http://localhost:8041",
+          target: "http://localhost:" + (process.env.dash_port || 8041),
           secure: false
         },
         [ "/docs/**"]: {