cht-conf 4.1.3 → 4.3.0

package/src/fn/upload-docs.js

@@ -1,19 +1,22 @@
 const path = require('path');
 const minimist = require('minimist');
-const userPrompt = require('../lib/user-prompt');
+const semver = require('semver');
 
+const api = require('../lib/api');
 const environment = require('../lib/environment');
 const fs = require('../lib/sync-fs');
+const { getValidApiVersion } = require('../lib/get-api-version');
 const log = require('../lib/log');
 const pouch = require('../lib/db');
 const progressBar = require('../lib/progress-bar');
+const userPrompt = require('../lib/user-prompt');
 
 const { info, trace, warn } = log;
 
 const FILE_EXTENSION = '.doc.json';
 const INITIAL_BATCH_SIZE = 100;
 
-const execute = async () => {
+async function execute() {
   const args = minimist(environment.extraArgs || [], { boolean: true });
 
   const docDir = path.resolve(environment.pathToProject, args.docDirectoryPath || 'json_docs');
@@ -22,22 +25,23 @@ const execute = async () => {
     return Promise.resolve();
   }
 
-  const filesToUpload = fs.recurseFiles(docDir).filter(name => name.endsWith(FILE_EXTENSION));
-  const docIdErrors = getErrorsWhereDocIdDiffersFromFilename(filesToUpload);
-  if (docIdErrors.length > 0) {
-    throw new Error(`upload-docs: ${docIdErrors.join('\n')}`);
-  }
-
-  const totalCount = filesToUpload.length;
+  const filenamesToUpload = fs.recurseFiles(docDir).filter(name => name.endsWith(FILE_EXTENSION));
+  const totalCount = filenamesToUpload.length;
   if (totalCount === 0) {
     return; // nothing to upload
   }
 
-  warn(`This operation will permanently write ${totalCount} docs.  Are you sure you want to continue?`);
-  if (!userPrompt.keyInYN()) {
-    throw new Error('User aborted execution.');
+  const analysis = analyseFiles(filenamesToUpload);
+  const errors = analysis.map(result => result.error).filter(Boolean);
+  if (errors.length > 0) {
+    throw new Error(`upload-docs: ${errors.join('\n')}`);
   }
 
+  userPrompt.warnPromptAbort(`This operation will permanently write ${totalCount} docs.  Are you sure you want to continue?`);
+
+  const deletedDocIds = analysis.map(result => result.delete).filter(Boolean);
+  await handleUsersAtDeletedFacilities(deletedDocIds);
+
   const results = { ok:[], failed:{} };
   const progress = log.level > log.LEVEL_ERROR ? progressBar.init(totalCount, '{{n}}/{{N}} docs ', ' {{%}} {{m}}:{{s}}') : null;
   const processNextBatch = async (docFiles, batchSize) => {
@@ -93,20 +97,102 @@ const execute = async () => {
     }
   };
 
-  return processNextBatch(filesToUpload, INITIAL_BATCH_SIZE);
-};
+  return processNextBatch(filenamesToUpload, INITIAL_BATCH_SIZE);
+}
 
-const getErrorsWhereDocIdDiffersFromFilename = filePaths =>
-  filePaths
+function analyseFiles(filePaths) {
+  return filePaths
     .map(filePath => {
       const json = fs.readJson(filePath);
       const idFromFilename = path.basename(filePath, FILE_EXTENSION);
 
       if (json._id !== idFromFilename) {
-        return `File '${filePath}' sets _id:'${json._id}' but the file's expected _id is '${idFromFilename}'.`;
+        return { error: `File '${filePath}' sets _id:'${json._id}' but the file's expected _id is '${idFromFilename}'.` };
+      }
+
+      if (json._deleted && json.cht_disable_linked_users) {
+        return { delete: json._id };
       }
     })
-    .filter(err => err);
+    .filter(Boolean);
+}
+
+async function handleUsersAtDeletedFacilities(deletedDocIds) {
+  if (!deletedDocIds?.length) {
+    return;
+  }
+
+  await assertCoreVersion();
+  
+  const affectedUsers = await getAffectedUsers(deletedDocIds);
+  const usernames = affectedUsers.map(userDoc => userDoc.username).join(', ');
+  if (affectedUsers.length === 0) {
+    trace('No users found needing an update.');
+    return;
+  }
+
+  userPrompt.warnPromptAbort(`This operation will update ${affectedUsers.length} user accounts: ${usernames} and cannot be undone. Are you sure you want to continue?`);
+  await updateAffectedUsers(affectedUsers);
+}
+
+async function assertCoreVersion() {
+  const actualCoreVersion = await getValidApiVersion();
+  if (semver.lt(actualCoreVersion, '4.7.0-dev')) {
+    throw Error(`CHT Core Version 4.7.0 or newer is required to use --disable-users options. Version is ${actualCoreVersion}.`);
+  }
+
+  trace(`Core version is ${actualCoreVersion}. Proceeding to disable users.`);
+}
+
+async function getAffectedUsers(deletedDocIds) {
+  const toPostApiFormat = (apiResponse) => {
+    const places = Array.isArray(apiResponse.place) ? apiResponse.place.filter(Boolean) : [apiResponse.place];
+    const placeIds = places.map(place => place?._id);
+    return {
+      username: apiResponse.username,
+      place: placeIds,
+    };
+  };
+
+  const knownUserDocs = {};
+  for (const facilityId of deletedDocIds) {
+    const fetchedUserInfos = await api().getUsersAtPlace(facilityId);
+    for (const fetchedUserInfo of fetchedUserInfos) {
+      const userDoc = knownUserDocs[fetchedUserInfo.username] || toPostApiFormat(fetchedUserInfo);
+      removePlace(userDoc, facilityId);
+      knownUserDocs[userDoc.username] = userDoc;
+    }
+  }
+
+  return Object.values(knownUserDocs);
+}
+
+function removePlace(userDoc, placeId) {
+  if (Array.isArray(userDoc.place)) {
+    userDoc.place = userDoc.place
+      .filter(id => id !== placeId);
+  } else {
+    delete userDoc.place;
+  }
+}
+
+async function updateAffectedUsers(affectedUsers) {
+  let disabledUsers = 0, updatedUsers = 0;
+  for (const userDoc of affectedUsers) {
+    const shouldDisable = !userDoc.place || userDoc.place?.length === 0;
+    if (shouldDisable) {
+      trace(`Disabling ${userDoc.username}`);
+      await api().disableUser(userDoc.username);
+      disabledUsers++;
+    } else {
+      trace(`Updating ${userDoc.username}`);
+      await api().updateUser(userDoc);
+      updatedUsers++;
+    }
+  }
+
+  info(`${disabledUsers} users disabled. ${updatedUsers} users updated.`);
+}
 
 module.exports = {
   requiresInstance: true,

package/src/lib/api.js

@@ -39,6 +39,7 @@ const request = {
   get: _request('get'),
   post: _request('post'),
   put: _request('put'),
+  delete: _request('delete'),
 };
 
 const logDeprecatedTransitions = (settings) => {
@@ -98,6 +99,29 @@ const api = {
       .then(() => updateAppSettings(content));
   },
 
+  async getUsersAtPlace(facilityId) {
+    const result = await request.get({
+      uri: `${environment.instanceUrl}/api/v2/users?facility_id=${facilityId}`,
+      json: true,
+    });
+
+    return result || [];
+  },
+
+  disableUser(username) {
+    return request.delete({
+      uri: `${environment.instanceUrl}/api/v1/users/${username}`,
+    });
+  },
+
+  updateUser(userDoc) {
+    return request.post({
+      uri: `${environment.instanceUrl}/api/v1/users/${userDoc.username}`,
+      json: true,
+      body: userDoc,
+    });
+  },
+
   createUser(userData) {
     return request.post({
       uri: `${environment.instanceUrl}/api/v1/users`,

package/src/lib/hierarchy-operations/hierarchy-data-source.js

@@ -0,0 +1,103 @@
+const lineageManipulation = require('./lineage-manipulation');
+
+const HIERARCHY_ROOT = 'root';
+const BATCH_SIZE = 10000;
+
+/*
+Fetches all of the documents associated with the "contactIds" and confirms they exist.
+*/
+async function getContactsByIds(db, ids) {
+  const contactDocs = await db.allDocs({
+    keys: ids,
+    include_docs: true,
+  });
+
+  const missingContactErrors = contactDocs.rows.filter(row => !row.doc).map(row => `Contact with id '${row.key}' could not be found.`);
+  if (missingContactErrors.length > 0) {
+    throw Error(missingContactErrors);
+  }
+
+  const contactDict = {};
+  contactDocs.rows.forEach(({ doc }) => contactDict[doc._id] = doc);
+  return contactDict;
+}
+
+async function getContact(db, id) {
+  try {
+    if (id === HIERARCHY_ROOT) {
+      return undefined;
+    }
+
+    return await db.get(id);
+  } catch (err) {
+    if (err.status !== 404) {
+      throw err;
+    }
+
+    throw Error(`Contact with id '${id}' could not be found`);
+  }
+}
+
+/*
+Given a contact's id, obtain the documents of all descendant contacts
+*/
+async function getContactWithDescendants(db, contactId) {
+  const descendantDocs = await db.query('medic/contacts_by_depth', {
+    key: [contactId],
+    include_docs: true,
+  });
+
+  return descendantDocs.rows
+    .map(row => row.doc)
+    // We should not move or update tombstone documents
+    // Not relevant for 4.x cht-core versions, but needed in older versions.
+    .filter(doc => doc && doc.type !== 'tombstone');
+}
+
+async function getReportsForContacts(db, createdByIds, createdAtId, skip) {
+  const createdByKeys = createdByIds.map(id => [`contact:${id}`]);
+  const createdAtKeys = createdAtId ? [
+    [`patient_id:${createdAtId}`],
+    [`patient_uuid:${createdAtId}`],
+    [`place_id:${createdAtId}`],
+    [`place_uuid:${createdAtId}`]
+  ] : [];
+
+  const reports = await db.query('medic-client/reports_by_freetext', {
+    keys: [
+      ...createdByKeys,
+      ...createdAtKeys,
+    ],
+    include_docs: true,
+    limit: BATCH_SIZE,
+    skip,
+  });
+
+  const docsWithId = reports.rows.map(({ doc }) => [doc._id, doc]);
+  return Array.from(new Map(docsWithId).values());
+}
+
+async function getAncestorsOf(db, contactDoc) {
+  const ancestorIds = lineageManipulation.pluckIdsFromLineage(contactDoc.parent);
+  const ancestors = await db.allDocs({
+    keys: ancestorIds,
+    include_docs: true,
+  });
+
+  const ancestorIdsNotFound = ancestors.rows.filter(ancestor => !ancestor.doc).map(ancestor => ancestor.key);
+  if (ancestorIdsNotFound.length > 0) {
+    throw Error(`Contact '${contactDoc?.name}' (${contactDoc?._id}) has parent id(s) '${ancestorIdsNotFound.join(',')}' which could not be found.`);
+  }
+
+  return ancestors.rows.map(ancestor => ancestor.doc);
+}
+
+module.exports = {
+  BATCH_SIZE,
+  HIERARCHY_ROOT,
+  getAncestorsOf,
+  getContactWithDescendants,
+  getContact,
+  getContactsByIds,
+  getReportsForContacts,
+};

package/src/lib/hierarchy-operations/index.js

@@ -0,0 +1,187 @@
+const lineageManipulation = require('./lineage-manipulation');
+const LineageConstraints = require('./lineage-constraints');
+const { trace, info } = require('../log');
+
+const JsDocs = require('./jsdocFolder');
+const DataSource = require('./hierarchy-data-source');
+
+async function moveHierarchy(db, options, sourceIds, destinationId) {
+  JsDocs.prepareFolder(options);
+  trace(`Fetching contact details: ${destinationId}`);
+  const constraints = await LineageConstraints(db, options);
+  const destinationDoc = await DataSource.getContact(db, destinationId);
+  const sourceDocs = await DataSource.getContactsByIds(db, sourceIds);
+  constraints.assertNoHierarchyErrors(Object.values(sourceDocs), destinationDoc);
+
+  let affectedContactCount = 0;
+  let affectedReportCount = 0;
+  const replacementLineage = lineageManipulation.createLineageFromDoc(destinationDoc);
+  for (const sourceId of sourceIds) {
+    const sourceDoc = sourceDocs[sourceId];
+    const descendantsAndSelf = await DataSource.getContactWithDescendants(db, sourceId);
+    const moveContext = {
+      sourceId,
+      destinationId,
+      descendantsAndSelf,
+      replacementLineage,
+      merge: !!options.merge,
+    };
+
+    await constraints.assertNoPrimaryContactViolations(sourceDoc, destinationDoc, descendantsAndSelf);
+    
+    if (options.merge) {
+      JsDocs.writeDoc(options, {
+        _id: sourceDoc._id,
+        _rev: sourceDoc._rev,
+        _deleted: true,
+        cht_disable_linked_users: !!options.disableUsers,
+      });
+    }
+        
+    const prettyPrintDocument = doc => `'${doc.name}' (${doc._id})`;
+    trace(`Considering lineage updates to ${descendantsAndSelf.length} descendant(s) of contact ${prettyPrintDocument(sourceDoc)}.`);
+    const updatedDescendants = replaceLineageInContacts(moveContext);
+    
+    const ancestors = await DataSource.getAncestorsOf(db, sourceDoc);
+    trace(`Considering primary contact updates to ${ancestors.length} ancestor(s) of contact ${prettyPrintDocument(sourceDoc)}.`);
+    const updatedAncestors = replaceLineageInAncestors(descendantsAndSelf, ancestors);
+    
+    minifyLineageAndWriteToDisk(options, [...updatedDescendants, ...updatedAncestors]);
+    
+    const movedReportsCount = await updateReports(db, options, moveContext);
+    trace(`${movedReportsCount} report(s) created by these affected contact(s) will be updated`);
+
+    affectedContactCount += updatedDescendants.length + updatedAncestors.length;
+    affectedReportCount += movedReportsCount;
+
+    info(`Staged updates to ${prettyPrintDocument(sourceDoc)}. ${updatedDescendants.length} contact(s) and ${movedReportsCount} report(s).`);
+  }
+
+  info(`Staged changes to lineage information for ${affectedContactCount} contact(s) and ${affectedReportCount} report(s).`);
+}
+
+async function updateReports(db, options, moveContext) {
+  const descendantIds = moveContext.descendantsAndSelf.map(contact => contact._id);
+
+  let skip = 0;
+  let reportDocsBatch;
+  do {
+    info(`Processing ${skip} to ${skip + DataSource.BATCH_SIZE} report docs`);
+    const createdAtId = options.merge && moveContext.sourceId;
+    reportDocsBatch = await DataSource.getReportsForContacts(db, descendantIds, createdAtId, skip);
+
+    const lineageUpdates = replaceLineageOfReportCreator(reportDocsBatch, moveContext);
+    const reassignUpdates = reassignReports(reportDocsBatch, moveContext);
+    const updatedReports = reportDocsBatch.filter(doc => lineageUpdates.has(doc._id) || reassignUpdates.has(doc._id));
+
+    minifyLineageAndWriteToDisk(options, updatedReports);
+
+    skip += reportDocsBatch.length;
+  } while (reportDocsBatch.length >= DataSource.BATCH_SIZE);
+
+  return skip;
+}
+
+function reassignReportSubjects(report, { sourceId, destinationId }) {
+  const SUBJECT_IDS = ['patient_id', 'patient_uuid', 'place_id', 'place_uuid'];
+  let updated = false;
+  for (const subjectId of SUBJECT_IDS) {
+    if (report[subjectId] === sourceId) {
+      report[subjectId] = destinationId;
+      updated = true;
+    }
+
+    if (report.fields[subjectId] === sourceId) {
+      report.fields[subjectId] = destinationId;
+      updated = true;
+    }
+  }
+  
+  return updated;
+}
+
+function reassignReports(reports, moveContext) {
+  const updated = new Set();
+  if (!moveContext.merge) {
+    return updated;
+  }
+
+  for (const report of reports) {
+    const isUpdated = reassignReportSubjects(report, moveContext);
+    if (isUpdated) {
+      updated.add(report._id);
+    }
+  }
+
+  return updated;
+}
+
+// This ensures all documents written are fully minified. Some docs in CouchDB are not minified to start with.
+function minifyLineageAndWriteToDisk(options, docs) {
+  docs.forEach(doc => {
+    lineageManipulation.minifyLineagesInDoc(doc);
+    JsDocs.writeDoc(options, doc);
+  });
+}
+
+function replaceLineageOfReportCreator(reports, moveContext) {
+  const replaceContactLineage = doc => lineageManipulation.replaceContactLineage(doc, {
+    replaceWith: moveContext.replacementLineage,
+    startingFromId: moveContext.sourceId,
+    merge: moveContext.merge,
+  });
+    
+  const updatedReportIds = reports
+    .filter(replaceContactLineage)
+    .map(({ _id }) => _id);
+  return new Set(updatedReportIds);
+}
+
+function replaceLineageInAncestors(descendantsAndSelf, ancestors) {
+  const updatedAncestors = [];
+  for (const ancestor of ancestors) {
+    const primaryContact = descendantsAndSelf.find(descendant => descendant._id === ancestor.contact?._id);
+    if (primaryContact) {
+      ancestor.contact = lineageManipulation.createLineageFromDoc(primaryContact);
+      updatedAncestors.unshift(ancestor);
+    }
+  }
+
+  return updatedAncestors;
+}
+
+function replaceLineageInSingleContact(doc, moveContext) {
+  const docIsSource = doc._id === moveContext.sourceId;
+  if (docIsSource && moveContext.merge) {
+    return;
+  }
+
+  const startingFromId = moveContext.merge || !docIsSource ? moveContext.sourceId : undefined;
+  const replaceLineageOptions = {
+    replaceWith: moveContext.replacementLineage,
+    startingFromId,
+    merge: moveContext.merge,
+  };
+  const parentWasUpdated = lineageManipulation.replaceParentLineage(doc, replaceLineageOptions);
+
+  replaceLineageOptions.startingFromId = moveContext.sourceId;
+  const contactWasUpdated = lineageManipulation.replaceContactLineage(doc, replaceLineageOptions);
+  if (parentWasUpdated || contactWasUpdated) {
+    return doc;
+  }
+}
+
+function replaceLineageInContacts(moveContext) {
+  return moveContext.descendantsAndSelf
+    .map(descendant => replaceLineageInSingleContact(descendant, moveContext))
+    .filter(Boolean);
+}
+
+module.exports = (db, options) => {
+  return {
+    HIERARCHY_ROOT: DataSource.HIERARCHY_ROOT,
+    move: (sourceIds, destinationId) => moveHierarchy(db, { ...options, merge: false }, sourceIds, destinationId),
+    merge: (sourceIds, destinationId) => moveHierarchy(db, { ...options, merge: true }, sourceIds, destinationId),
+  };
+};
+

package/src/lib/hierarchy-operations/jsdocFolder.js

@@ -0,0 +1,37 @@
+const path = require('path');
+const userPrompt = require('../user-prompt');
+const fs = require('../sync-fs');
+const { warn, trace } = require('../log');
+
+function prepareFolder({ docDirectoryPath, force }) {
+  if (!fs.exists(docDirectoryPath)) {
+    fs.mkdir(docDirectoryPath);
+  } else if (!force && fs.recurseFiles(docDirectoryPath).length > 0) {
+    deleteAfterConfirmation(docDirectoryPath);
+  }
+}
+
+function writeDoc({ docDirectoryPath }, doc) {
+  const destinationPath = path.join(docDirectoryPath, `${doc._id}.doc.json`);
+  if (fs.exists(destinationPath)) {
+    warn(`File at ${destinationPath} already exists and is being overwritten.`);
+  }
+
+  trace(`Writing updated document to ${destinationPath}`);
+  fs.writeJson(destinationPath, doc);
+}
+
+function deleteAfterConfirmation(docDirectoryPath) {
+  warn(`The document folder '${docDirectoryPath}' already contains files. It is recommended you start with a clean folder. Do you want to delete the contents of this folder and continue?`);
+  if (!userPrompt.keyInYN()) {
+    throw new Error('User aborted execution.');
+  }
+  
+  fs.deleteFilesInFolder(docDirectoryPath);
+}
+
+module.exports = {
+  prepareFolder,
+  writeDoc,
+};
+