cht-conf 3.9.1 → 3.9.2

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [3.9.2](https://github.com/medic/cht-conf/compare/v3.9.1...v3.9.2) (2021-12-08)
+
+
+### Performance Improvements
+
+* **#432:** refactor the moving of reports to use a recursive function to improve performance ([c17b43b](https://github.com/medic/cht-conf/commit/c17b43b4d89b29b813db4be6e0f2d2c0a1e6c94d)), closes [#432](https://github.com/medic/cht-conf/issues/432)
+
 ## [3.9.1](https://github.com/medic/cht-conf/compare/v3.9.0...v3.9.1) (2021-11-16)
 
 

package/README.md

@@ -282,7 +282,7 @@ To develop a new action or improve an existing one, check the ["Actions" doc](sr
 
 ## Testing
 
-Execute `npm test` to run static analysis checks and the test suite.
+Execute `npm test` to run static analysis checks and the test suite. Requires Docker to run integration tests against a CouchDB instance. 
 
 ## Executing your local branch
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cht-conf",
-  "version": "3.9.1",
+  "version": "3.9.2",
   "description": "Configure CHT deployments",
   "main": "./src/lib/main.js",
   "engines": {
@@ -9,7 +9,9 @@
   "scripts": {
     "clean": "rm -rf ./build/",
     "eslint": "eslint src/**/*.js test/*.js test/**/*.js",
-    "test": "npm run eslint && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only ../../test/**/*.spec.js && cd ../..",
+    "docker-start-couchdb": "npm run docker-stop-couchdb && docker run -d -p 6984:5984 --rm --name cht-conf-couchdb couchdb:2.3.1 && sh test/scripts/wait_for_response_code.sh 6984 200 CouchDB",
+    "docker-stop-couchdb": "docker stop cht-conf-couchdb || true",
+    "test": "npm run eslint && npm run docker-start-couchdb && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only ../../test/**/*.spec.js && cd ../.. && npm run docker-stop-couchdb",
     "semantic-release": "semantic-release"
   },
   "bin": {

package/src/fn/move-contacts.js

@@ -10,6 +10,7 @@ const pouch = require('../lib/db');
 const { warn, trace, info, error } = require('../lib/log');
 
 const HIERARCHY_ROOT = 'root';
+const BATCH_SIZE = 10000;
 
 module.exports = {
   requiresInstance: true,
@@ -49,19 +50,15 @@ const updateLineagesAndStage = async (options, db) => {
     trace(`Considering primary contact updates to ${ancestors.length} ancestor(s) of contact ${prettyPrintDocument(contactDoc)}.`);
     const updatedAncestors = replaceLineageInAncestors(descendantsAndSelf, ancestors);
 
-    const reportsCreatedByDescendants = await fetch.reportsCreatedBy(db, descendantsAndSelf.map(descendant => descendant._id));
-    trace(`${reportsCreatedByDescendants.length} report(s) created by these affected contact(s) will update`);
-    const updatedReports = replaceLineageInReports(reportsCreatedByDescendants, replacementLineage, contactId);
+    minifyLineageAndWriteToDisk([...updatedDescendants, ...updatedAncestors], options);
 
-    [...updatedDescendants, ...updatedReports, ...updatedAncestors].forEach(updatedDoc => {
-      lineageManipulation.minifyLineagesInDoc(updatedDoc);
-      writeDocumentToDisk(options, updatedDoc);
-    });
+    const movedReportsCount = await moveReports(db, descendantsAndSelf, options, replacementLineage, contactId);
+    trace(`${movedReportsCount} report(s) created by these affected contact(s) will be updated`);
 
     affectedContactCount += updatedDescendants.length + updatedAncestors.length;
-    affectedReportCount += updatedReports.length;
+    affectedReportCount += movedReportsCount;
 
-    info(`Staged updates to ${prettyPrintDocument(contactDoc)}. ${updatedDescendants.length} contact(s) and ${updatedReports.length} report(s).`);
+    info(`Staged updates to ${prettyPrintDocument(contactDoc)}. ${updatedDescendants.length} contact(s) and ${movedReportsCount} report(s).`);
   }
 
   info(`Staged changes to lineage information for ${affectedContactCount} contact(s) and ${affectedReportCount} report(s).`);
@@ -155,6 +152,31 @@ ${bold('OPTIONS')}
 `);
 };
 
+const moveReports = async (db, descendantsAndSelf, writeOptions, replacementLineage, contactId) => {
+  const contactIds = descendantsAndSelf.map(contact => contact._id);
+
+  let skip = 0;
+  let reportDocsBatch;
+  do {
+    info(`Processing ${skip} to ${skip + BATCH_SIZE} report docs`);
+    reportDocsBatch = await fetch.reportsCreatedBy(db, contactIds, skip);
+
+    const updatedReports = replaceLineageInReports(reportDocsBatch, replacementLineage, contactId);
+    minifyLineageAndWriteToDisk(updatedReports, writeOptions);
+
+    skip += reportDocsBatch.length;
+  } while (reportDocsBatch.length >= BATCH_SIZE);
+
+  return skip;
+};
+
+const minifyLineageAndWriteToDisk = (docs, parsedArgs) => {
+  docs.forEach(doc => {
+    lineageManipulation.minifyLineagesInDoc(doc);
+    writeDocumentToDisk(parsedArgs, doc);
+  });
+};
+
 const writeDocumentToDisk = ({ docDirectoryPath }, doc) => {
   const destinationPath = path.join(docDirectoryPath, `${doc._id}.doc.json`);
   if (fs.exists(destinationPath)) {
@@ -214,10 +236,12 @@ const fetch = {
       .filter(doc => doc && doc.type !== 'tombstone');
   },
 
-  reportsCreatedBy: async (db, contactIds) => {
+  reportsCreatedBy: async (db, contactIds, skip) => {
     const reports = await db.query('medic-client/reports_by_freetext', {
       keys: contactIds.map(id => [`contact:${id}`]),
       include_docs: true,
+      limit: BATCH_SIZE,
+      skip: skip,
     });
 
     return reports.rows.map(row => row.doc);