cht-conf 3.8.0 → 3.9.3

package/.husky/commit-msg

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx --no-install commitlint --edit "$1"

package/.releaserc.json

@@ -0,0 +1,22 @@
+{
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    "@semantic-release/npm",
+    "@semantic-release/github",
+    [
+      "@semantic-release/changelog",
+      {
+        "changelogFile": "CHANGELOG.md"
+      }
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": ["package.json", "package-lock.json", "CHANGELOG.md"],
+        "message": "chore(release): ${nextRelease.version} [skip ci]"
+      }
+    ]
+  ],
+  "branches": ["master"]
+}

package/CHANGELOG.md

@@ -0,0 +1,358 @@
+## [3.9.3](https://github.com/medic/cht-conf/compare/v3.9.2...v3.9.3) (2021-12-16)
+
+
+### Bug Fixes
+
+* **#455:** allow empty translation constants ([ac170d7](https://github.com/medic/cht-conf/commit/ac170d7cce13a6a500d89884d4ca63801f06a969)), closes [#455](https://github.com/medic/cht-conf/issues/455)
+
+## [3.9.2](https://github.com/medic/cht-conf/compare/v3.9.1...v3.9.2) (2021-12-08)
+
+
+### Performance Improvements
+
+* **#432:** refactor the moving of reports to use a recursive function to improve performance ([c17b43b](https://github.com/medic/cht-conf/commit/c17b43b4d89b29b813db4be6e0f2d2c0a1e6c94d)), closes [#432](https://github.com/medic/cht-conf/issues/432)
+
+## [3.9.1](https://github.com/medic/cht-conf/compare/v3.9.0...v3.9.1) (2021-11-16)
+
+
+### Bug Fixes
+
+* **#448:** Bump Ubuntu version and Node version in container to fix docker usage. Updates to readme ([25cb955](https://github.com/medic/cht-conf/commit/25cb955377eea6c6a0a7cb3953d889b57c93884b))
+
+# [3.9.0](https://github.com/medic/cht-conf/compare/v3.8.0...v3.9.0) (2021-11-01)
+
+
+### Features
+
+* **#413:** fix tags so that next release is 3.9.0 ([6eea489](https://github.com/medic/cht-conf/commit/6eea489804cf0de3c11fe855d2b285e41553439d))
+* **#413:** implement continuous delivery ([244203e](https://github.com/medic/cht-conf/commit/244203e341fe46e11dcb7249fd3a77dcb8d1f43c)), closes [#413](https://github.com/medic/cht-conf/issues/413)
+* **#413:** provide medic-ci with write permissions to npm ([fd2522e](https://github.com/medic/cht-conf/commit/fd2522ef9c40f697dff2c71a3b638a225cf7adfe)), closes [#413](https://github.com/medic/cht-conf/issues/413)
+* **#413:** publish to npm with automation token ([1cf8550](https://github.com/medic/cht-conf/commit/1cf8550cca2d404342e28d0a9086c837fa2a011e)), closes [#413](https://github.com/medic/cht-conf/issues/413)
+* **#413:** use GH_ADMIN_TOKEN for release workflow ([0f1c581](https://github.com/medic/cht-conf/commit/0f1c581cc5e0990881ab6a99f6f26a040144bfbc)), closes [#413](https://github.com/medic/cht-conf/issues/413)
+
+## 3.8.0
+
+### Ability to edit a contact type
+
+It is now possible to edit a contact type with the `edit-contacts` action. Additionally, `edit-contacts` accepts a new flag `--updateOfflineDocs`, that allows editing json docs that already exist in the directory provided in `--docDirectoryPath` instead of downloading the docs from the database.
+
+[#408](https://github.com/medic/cht-conf/issues/408)
+
+### Fail early if .eslintrc file is not provided
+
+Running the `compile-app-settings` command now fails early and provides a more descriptive error message to users.
+
+[#333](https://github.com/medic/cht-conf/issues/333)
+
+### Check to make sure CHT instance is up
+
+Commands that require an instance now fail with a more descriptive error message if the instance provided is not reachable
+
+[#380](https://github.com/medic/cht-conf/issues/380)
+
+## 3.7.0
+
+### The repository has been renamed to `cht-conf`. 
+
+Additionally:
+- the `medic-conf` cli command is now deprecated, in favor of `cht`.
+- the `medic-logs` cli command is now deprecated, in favor of `cht-logs`
+- the `pngout-medic` cli command is now deprecated, in favor of `pngout-cht`
+
+### Default "resolvedIf" function for tasks
+
+Task definitions can now omit the `resolvedIf` function. In this case, the task is resolved by any report assigned to the contact that matches the `form` defined in the action of type `report`.
+For more information, please check the [documentation](https://docs.communityhealthtoolkit.org/apps/reference/tasks/#default-resolvedif-method).
+
+[#107](https://github.com/medic/cht-conf/issues/107)
+
+## 3.6.0
+
+### Tasks actions `modifyContent` functions now receive the event as a parameter
+
+To support task logic that depends on knowing which event the emission is generated for, `modifyContent` now receives a fourth parameter containing the event data. 
+For more information, please check the [documentation](https://docs.communityhealthtoolkit.org/apps/reference/tasks/#tasksjs).
+
+[#398](https://github.com/medic/cht-conf/issues/398)
+
+## 3.5.0
+
+### Forms are validated before being uploaded
+
+When uploading forms to CHT Core v3.11.0 or above these are submitted for validation to ensure they will work when uploaded.
+
+[#331](https://github.com/medic/cht-conf/issues/331)
+
+### Check that the code is up to date with the git repo
+
+Warns when pushing configuration that isn't up to date with the git repo.
+
+[#64](https://github.com/medic/cht-conf/issues/64)
+
+### Bug fixes
+
+- [#358](https://github.com/medic/cht-conf/issues/358): Can't create task action with type "contact"
+- [#387](https://github.com/medic/cht-conf/issues/387): Can't upload-app-settings on Node lower than 12
+
+## 3.4.0
+
+### Support for uploading branding and partners docs
+
+Two new actions, `upload-branding` and `upload-partners`, permit uploading branding and partners images, respectively. 
+For more information, please check the [documentation](https://docs.communityhealthtoolkit.org/apps/reference/resources/#branding).
+
+[#167](https://github.com/medic/cht-conf/issues/167)
+
+### Modular app_settings and updated folder structure
+
+The `forms` and `schedules` sections of `app_settings` should now be configured individually, in separate files, while other configs should be added to the new `base_settings.json` file. All settings will be compiled into the `app_settings.json`, which should now no longer be edited manually.
+For more information, please check the [documentation](https://docs.communityhealthtoolkit.org/apps/reference/app-settings/). 
+
+[#214](https://github.com/medic/cht-conf/issues/214)
+[#68](https://github.com/medic/cht-conf/issues/68)
+
+### Improvements
+
+- [#254](https://github.com/medic/cht-conf/issues/254): Add validation to the upload custom translations function
+- [#269](https://github.com/medic/cht-conf/issues/269): Warn about deprecated transitions when uploading app-settings
+
+### Bug fixes
+
+- [#345](https://github.com/medic/cht-conf/issues/345): Warn config overwrite falsely alerts about changes when uploading certain types of attachments
+- [#164](https://github.com/medic/cht-conf/issues/164): Declarative config appliesToType is indicated as optional for fields and cards but we do require them
+- [#377](https://github.com/medic/cht-conf/issues/377): Incorrect warning when specifying hidden_fields
+- [#382](https://github.com/medic/cht-conf/issues/382): Contacts with hardcoded types that also have a `contact_type` property don't get the correct contact_summary fields and don't count towards target goals
+
+## 3.3.0
+
+### Support for headless execution
+
+The `--force` argument can now be provided to make it possible to execute commands in a scripted environment by skipping any prompts. This is more dangerous as cht-conf will no longer warn about possible configuration errors, so it's recommended this option is only used when user input is not possible and you have independently verified that the configuration is correct.
+
+[#307](https://github.com/medic/cht-conf/issues/307)
+
+### Privacy policies
+
+Privacy policies can now be uploaded using cht-conf. Read [the documentation](https://docs.communityhealthtoolkit.org/apps/guides/security/privacy-policy/) for more information.
+
+[cht-core#6538](https://github.com/medic/cht-core/issues/6538)
+
+### Bug fixes
+
+- [#332](https://github.com/medic/cht-conf/issues/332): Uploading of multimedia is extremely limited
+- [#273](https://github.com/medic/cht-conf/issues/273): Add configurable hierarchy support to doc references in csv-to-docs
+- [#322](https://github.com/medic/cht-conf/issues/322): Missing optional appliesIf attribute on contact summary cards causes crash
+- [#315](https://github.com/medic/cht-conf/issues/315): Error running csv-to-docs with null properties
+- [#325](https://github.com/medic/cht-conf/issues/325): Task generation fails when given contactless reports
+- [cht-core#6291](https://github.com/medic/cht-core/issues/6291): Allow specifying subject_key in properties
+
+## 3.2.0
+
+### Prompt before configuration overwrites
+
+This new feature warns and prompts users when uploading a configuration if the configuration being overwritten is not the last known configuration. These warnings are comparable to a database write conflict. Previous versions of cht-conf would upload the new configuration overwriting whatever was there, resulting in users accidentally wiping out someone elses changes or changes made through the App Management webapp.
+
+To detect accidental overwrites, cht-conf will generate two files - `.snapshots/remote.json` and `.snapshots/local.json`. These store the last known configuration on the server. The `remote.json` file should be commited with your configuration changes, and `local.json` should be added to the .gitignore since everyone's local environment is different. If changes are made to the configuration on the server using cht-conf or the App Management webapp without committing the `remote.json` file to your config repo, and then subsequently pulling the changes, then any attempt to upload configuration changes will be notified about the risk and prompted to either overwrite or cancel.
+
+This feature was introduced but not announced in 3.1.0. In this release we have substantially changed the implementation, fixed some bugs, and improved the overall experience.
+
+### Configuration is only uploaded if something has changed
+
+As of this release no docs on the server will be touched unless some configuration has actually changed. This means the end users will not have to download docs with no actual difference giving them a better experience with upgrades.
+
+[#271](https://github.com/medic/cht-conf/issues/271)
+
+### New action to bulk-edit contacts
+
+You can now bulk edit contacts at once by providing a CSV. For more information, read [the documentation](https://github.com/medic/cht-conf/blob/master/README.md#editing-contacts-across-the-hierarchy).
+
+[#297](https://github.com/medic/cht-conf/issues/297)
+
+## 3.1.0
+
+### Declarative Configuration Support for Core Framework v3.8.0
+
+The updates to tasks and targets in Core Framework v3.8.0 require additional data about the tasks and targets emitted by partner rules code. After upgrading to v3.8, the Tasks tab and Targets tab won't function until partner code is updated to send this new data. **Declarative configuration projects must deploy their configuration using cht-conf v3.1 or later for tasks and targets to function on the Core Framework v3.8.0**
+
+It is still safe to deploy declarative configuration to any Core Framework version, including those prior to v3.8.
+
+### Declarative Configuration Tasks/Targets Schema Validation
+The updates to tasks and targets in Core Framework v3.8.0 allow for project members to write impact queries against Postgres to understand how tasks are behaving on users' phones. In previous versions of the declarative configuration system, some identification attributes were optional or able to be reused. This becomes problematic after the Core Framework v3.8.0 release because it will lead to data integrity issues if task or target elements are reorderd or removed from within the configuration.
+
+`cht-conf` v3.1 guarantees clearer data integrity with the following changes to the declarative configuration schema:
+
+* Task.name is now a required unique attribute
+* Task.event[n].id is now a required unique attribute when there is more than one event
+* Target.id although previously required, is now required to be unique
+
+### Declarative Configuration Target "GroupBy"
+Two new optional variables `groupBy` and `passesIfGroupCount` in the [declarative Target configuration schema](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#targets) allows for target emissions to be counted and scored in groups. This allows targets to support more complex scenarios like the number of families receiving n household visits/family/month.
+
+## 3.0.5
+
+### Updates to purge configuration
+
+cht-conf has changed the way it compiles purge configuration.
+To configure purge, create a file `purge.js` in your project root. This file is a javascript module that should export an object containing your purge configuration:
+
+```
+module.exports = {
+  text_expression: 'at 12 am on Sunday',
+  run_every_days: 7,
+  fn: (userCtx, contact, reports, messages) => {
+    const old = Date.now() - (1000 * 60 * 60 * 24 * 365);
+    return [
+        ...reports.filter(report => report.reported_date > old),
+        ...messages.filter(message => message.reported_date > old),
+    ].map(doc => doc._id);
+  },
+};
+```
+Storing the purge function in `purging.js` is deprecated.
+
+## 3.0.0
+
+cht-conf v3.0 contains breaking changes! This release only impacts the `compile-app-settings` action, which impacts the configuration code in `tasks.js`, `targets.js`, `contact-summary.js`, and `contact-summary.templated.js`.
+
+### Breaking Change - Static Analysis via ESLint
+cht-conf has changed the engine we use for performing static analysis from [JSHint](https://jshint.com/) to [ESLint](https://eslint.org/). This grants authors more control over the static analysis rules which are enforced on their code. We provide a set of default rules, but you can change or add rules by creating an [.eslintrc file](https://eslint.org/docs/user-guide/configuring#using-configuration-files-1). The default static analysis rules which are enforced in cht-conf 3.0 are similar to the rules which were enforced in cht-conf 2.x, but details may be slightly different.
+
+### Support for ECMAScript 6
+In previous versions of cht-conf, configuration JavaScript was been limited to ECMAScript 5. [ECMAScript 6](http://es6-features.org/) is the next generation of the JavaScript language. To use ES6 in your JavaScript configuration code, add a `.eslintrc` file to your project directory with the following contents:
+
+```
+{
+  "env": {
+    "node": true,
+    "es6": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 6
+  }
+}
+```
+
+If your project is using [CHT Android](https://github.com/medic/cht-android) you must be running a version newer than v0.1.203 (March 2017) to use `ecmaVersion: 6`. ecmaVersions above 6 are not currently supported.
+
+### Breaking Change - Modules
+As part of our effort to leverage JavaScript standards, JavaScript configuration code should now use `require('./file')` or `import lib from './file';` to share code between multiple JavaScript files. The global `extras` object is no longer present and you may see the error `'extras' is not defined` when running `compile-app-settings`. This global object was a reference to `nools-extras.js` or `contact-summary-extras.js`, so you can achieve the same behavior by adding the line `var extras = require('./nools-extras');` or `import extras from './nools-extras';` to the top of your JavaScript.  This gives configuration authors the flexibility to have multiple library files, and enables the sharing of code across projects.
+
+You may also find it helpful to leverage third-party modules like [MomentJs](https://momentjs.com/) or [lodash](https://lodash.com/). You may install them using [npm](https://docs.npmjs.com/about-npm/) packages. For example, run `npm install moment` and then add `const moment = require('moment');` in your script. Ensure that you maintain a reasonable file size and memory footprint for your scripts. We support [tree shaking](https://webpack.js.org/guides/tree-shaking/) so use it when possible.
+
+### Breaking Change - contact-summary.js as an ES Module
+As part of our effort to leverage JavaScript standards and increase the testability of code, `contact-summary.js` should be a standard ES Module instead of using a bare return. This allows you to write standard unit tests for the file instead of needing a harness or custom wrapper.
+
+`contact-summary.js` in cht-conf 1.x or 2.x:
+```
+return {
+  fields: [],
+  cards: [],
+  context: {},
+};
+```
+
+`contact-summary.js` in cht-conf 3.0:
+```
+module.exports = {
+  fields: [],
+  cards: [],
+  context: {},
+};
+```
+
+### Breaking Change - Declarative Configuration library is now internal
+The [Declarative Configuration System](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md) explicitly provides documented functions which are meant to be re-used in configuration code (eg. [Utils](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#utils)). In cht-conf 2.x and before, it was possible to also re-use some undocumented internals of the declarative library (common examples are `now`, `isReportValid()`, and `emitTargetInstance()`). In cht-conf 3.0, these internals and hidden as was intended. Documented interfaces remain unchanged.
+
+### Breaking Change - Interface change in Target.emitCustom()
+The Declarative Configuration [Target Schema](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#target-schema) for `emitCustom` has changed to `function(emit, instance, contact, report)`. The default value is `(emit, instance) => emit(instance)`.
+
+### Debug Mode for compile-app-settings
+`cht --local compile-app-settings upload-app-settings -- --debug`
+
+The `--debug` flag will change the behavior of `compile-app-settings` such that:
+
+1. ESLint failures are logged as warnings instead of causing errors
+1. Webpack warnings are logged as warnings instead of causing errors
+1. Code minification is skipped to make it easier to debug your code
+
+## 2.2.0
+
+cht-conf v2.2 includes:
+
+* New action `move-contact` facilitates the moving of contacts and places within a project's hierarchy. [Documentation](https://github.com/medic/cht-conf#moving-contacts-within-the-hierarchy). [#172](https://github.com/medic/cht-conf/issues/172)
+* Updates the `upload-custom-translations` action to support custom locales (for WebApp projects >v3.4.0). [#199](https://github.com/medic/cht-conf/issues/199)
+* The declarative configuration [target schema](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#target-schema) has been updated. The idType attribute can now be a function. [#145](https://github.com/medic/cht-conf/issues/145)
+* Support for Node 12, refactoring, and code cleanup.
+
+## 2.0.0
+
+cht-conf v2.0 contains breaking interface changes for the command-line and for the [declarative configuration system](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md).
+
+### Breaking Changes to Command-line Arguments
+Version 2.0 replaces our custom code for command-line argument parsing with [minimist](https://github.com/substack/minimist). This increases flexibility and more flexible ordering, but includes an interface change for a few commands. Refer to `cht --help` for an overview of all command-line options and syntax.
+
+#### Commands Requiring '='
+The command
+
+    cht --url http://admin:pass@localhost:5988
+
+now includes an `=` sign after the `--url`
+
+    cht --url=http://admin:pass@localhost:5988
+
+This affects the `url`, `instance`, `user`, and `shell-completion` arguments.
+
+#### Action List Requires '--' Prefix
+The command
+
+    cht --local convert-app-forms upload-app-forms my_form_1 my_form_2
+
+now requires a '--' to separate actions from the list of forms:
+
+    cht --local convert-app-forms upload-app-forms -- my_form_1 my_form_2
+
+### Breaking Changes to Declarative Configuration
+
+ The [declarative configuration system documentation](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md) contains up-to-date documentation of all interfaces.
+
+#### Using Node Modules
+We are hoping to converge on the interfaces used for [node modules](https://nodejs.org/api/modules.html) for sharing code between files. Moving forward, Medic configuration files will `export` and `require` code between files. This should 1) simplify the contract between the Medic web app and your configuration code, 2) simplify and facilitate unit testing, 3) allow you to use third-party libraries in your configuration code in future releases, and 4) simplify the cht-conf infrastructure and process for debugging issues.
+
+Your `nools-extras.js` file now need to export any values, functions, objects which are to be shared with `tasks.js` or `targets.js` via the `module.exports` interface. [Example](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#nools-extrasjs)
+
+Your `tasks.js` and `targets.js` files should now access the exported attributes in your `nools-extras.js` via the global `extras` object. [Example](https://github.com/medic/medic-docs/blob/master/configuration/developing-community-health-applications.md#targetsjs)
+
+#### Re-ordered Parameters
+It can be hard to understand (and hard to document) the parameters which define the interfaces of the declarative configuration system. Version 2.0 includes interface changes which help to keep interfaces more consistent by moving optional variables to the end of the interface. The following interfaces have changed:
+
+Interface | v1.x | v2.x
+-- | -- | --
+tasks.events[n].dueDate | (contact/reports, event, scheduledTaskIdx) | (event, contact, reports, scheduledTaskIdx)
+tasks.actions[n].modifyContent | (contact/reports, content) | (content, contact, reports)
+targets.emitCustom | (contact, reports) | (instance, contact, reports)
+targets.date | ( contact ) | (contact, reports)
+
+### New Features in Declarative Configuration
+
+#### this
+The `this` keyword is now available in all declarative configuration function interfaces to facilitate basic data sharing. This allows you to re-use a variable in another part of the task or target, rather than re-calculating it's value again.
+
+For example, you may have a complex seek through a set of reports in your `appliesIf` function, and need the same seek in calculating the `dueDate`, as seen in the example below.
+
+    {
+      appliesTo: 'contacts',
+      appliesIf: function(contact) {
+          const necessaryReport = extras.findNecessaryReport(contact.reports);
+          this.necessaryReport = necessaryReport;
+          return !!necessaryReport;
+      },
+      events: [{
+          start: 1,
+          end: 3,
+          dueDate: function (event, contact, report) {
+              // Use the saved value of this.necessaryReport
+              return this.necessaryReport.reported_date;
+          },
+      }],
+      ...
+    }

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 MAINTAINER DevOps "devops@medicmobile.org"
 
 
@@ -14,7 +14,7 @@ RUN echo "==> Installing Python dependencies" && \
 RUN echo "====> Installing cht-conf python stuff"    &&\
     python -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x -o nodesource_setup.sh
+RUN curl -sL https://deb.nodesource.com/setup_12.x -o nodesource_setup.sh
 RUN bash nodesource_setup.sh
 RUN apt-get install -y nodejs
 

package/README.md

@@ -11,29 +11,44 @@ CHT Conf is a command-line interface tool to manage and configure your apps buil
 
 # Installation
 
-## Docker
+## Operating System Specific
 
-	docker build -t cht-conf:v0 .
-	docker run cht-conf:v0
-	docker exec -it <container_name> /bin/bash
-
-## Ubuntu
+### Ubuntu
 
 	npm install -g cht-conf
 	sudo python -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
 
-## OSX
+### OSX
 
 	npm install -g cht-conf
 	pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
 
-## Windows
+### Windows
 
 As Administrator:
 
 	npm install -g cht-conf
 	python -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic --upgrade
 
+### Docker
+
+**NB** - `cht-conf` in a Docker container should only be used if you're familiar with running containers. You will have to copy or mount your config files into the container in order to be able to use the `cht` call to interact with a remote CHT instance.
+
+Build and then run the `cht-conf` container:
+
+	docker build -t cht-conf:v0 .
+	docker run -d --name cht-conf cht-conf:v0
+
+You now have a container running in the background called `cht-conf`. You can get a shell on this container with:
+
+	docker exec -it cht-conf /bin/bash
+
+And then you can run the utility with `cht`.  
+
+When you are done with the container, you can stop it with:
+
+	docker stop cht-conf
+
 ## Bash completion
 
 To enable tab completion in bash, add the following to your `.bashrc`/`.bash_profile`:
@@ -46,7 +61,7 @@ To upgrade to the latest version
 
 	npm install -g cht-conf
 
-# Usage
+## Usage
 
 `cht` will upload the configuration **from your current directory**.
 
@@ -60,9 +75,9 @@ For developers, this is the instance defined in your `COUCH_URL` environment var
 
 	cht --local
 
-### A specific Medic Mobile instance
+### A specific Medic-hosted instance
 
-For configuring against Medic Mobile-hosted instances.
+For configuring Medic-hosted instances.
 
 	cht --instance=instance-name.dev
 
@@ -80,7 +95,7 @@ If a different username is required, add the `--user` switch:
 
     cht --archive
 
-The resulting archive is consumable by Medic's API >v3.7 to create default configurations.
+The resulting archive is consumable by CHT API >v3.7 to create default configurations.
 
 ## Perform specific action(s)
 
@@ -267,7 +282,7 @@ To develop a new action or improve an existing one, check the ["Actions" doc](sr
 
 ## Testing
 
-Execute `npm test` to run static analysis checks and the test suite.
+Execute `npm test` to run static analysis checks and the test suite. Requires Docker to run integration tests against a CouchDB instance. 
 
 ## Executing your local branch
 
@@ -279,16 +294,25 @@ Execute `npm test` to run static analysis checks and the test suite.
 
 ## Releasing
 
-1. Create a pull request with prep for the new release. This should contain changes to release notes if required and anything else that needs to be done. As commit messages should be clear and readable for every change, [release-notes.md](./release-notes.md) does not need to be updated for every single change. Instead, it should include information about significant changes, breaking changes, changes to interfaces, changes in behavior, new feature details, etc.
-1. Get the pull request reviewed and approved
-1. Run `npm version patch`, `npm version minor`, or `npm version major` as appropriate. This will:
-    - Update versions in `package.json` and `package-lock.json`
-    - Commit those changes locally and tag that commit with the new version
-1. Run `npm publish` to publish the new tag to npm
-1. `git push && git push --tags` to push the npm generated commit and tag up to your pre-approved pull request
-1. Merge the pull request back into master
+1. Create a pull request with prep for the new release. 
+1. Get the pull request reviewed and approved.
+1. When doing the squash and merge, make sure that your commit message is clear and readable and follows the strict format described in the commit format section below. If the commit message does not comply, automatic release will fail.
+1. In case you are planning to merge the pull request with a merge commit, make sure that every commit in your branch respects the format. 
 1. Announce the release on the [CHT forum](https://forum.communityhealthtoolkit.org), under the "Product - Releases" category.
 
+### Commit format
+The commit format should follow this [conventional-changelog angular preset](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-angular). Examples are provided below.
+
+Type | Example commit message | Release type
+-- | -- | --
+Bug fixes | fix(#123): infinite spinner when clicking contacts tab twice | patch
+Performance | perf(#789): lazily loaded angular modules | patch
+Features | feat(#456): add home tab | minor
+Non-code | chore(#123): update README | none
+Breaking| perf(#2): remove reporting rates feature <br/> BREAKING CHANGE: reporting rates no longer supported | major
+
+
+
 ### Releasing betas
 
 1. Checkout `master`

package/commitlint.config.js

@@ -0,0 +1 @@
+module.exports = {extends: ['@commitlint/config-conventional']}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cht-conf",
-  "version": "3.8.0",
+  "version": "3.9.3",
   "description": "Configure CHT deployments",
   "main": "./src/lib/main.js",
   "engines": {
@@ -9,7 +9,10 @@
   "scripts": {
     "clean": "rm -rf ./build/",
     "eslint": "eslint src/**/*.js test/*.js test/**/*.js",
-    "test": "npm run eslint && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only ../../test/**/*.spec.js && cd ../.."
+    "docker-start-couchdb": "npm run docker-stop-couchdb && docker run -d -p 6984:5984 --rm --name cht-conf-couchdb couchdb:2.3.1 && sh test/scripts/wait_for_response_code.sh 6984 200 CouchDB",
+    "docker-stop-couchdb": "docker stop cht-conf-couchdb || true",
+    "test": "npm run eslint && npm run docker-start-couchdb && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only ../../test/**/*.spec.js && cd ../.. && npm run docker-stop-couchdb",
+    "semantic-release": "semantic-release"
   },
   "bin": {
     "medic-conf": "src/bin/index.js",
@@ -23,7 +26,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@github.com/medic/cht-conf.git"
+    "url": "https://github.com/medic/cht-conf.git"
   },
   "author": "",
   "license": "AGPL-3.0-only",
@@ -33,7 +36,7 @@
   "homepage": "https://github.com/medic/cht-conf#readme",
   "dependencies": {
     "@hapi/joi": "^16.1.8",
-    "@medic/translation-checker": "1.0.0",
+    "@medic/translation-checker": "^1.0.1",
     "canonical-json": "0.0.4",
     "csv-parse": "^4.16.0",
     "dom-compare": "^0.6.0",
@@ -64,6 +67,10 @@
     "xmldom": "^0.6.0"
   },
   "devDependencies": {
+    "@commitlint/cli": "^13.2.0",
+    "@commitlint/config-conventional": "^13.2.0",
+    "@semantic-release/changelog": "^6.0.0",
+    "@semantic-release/git": "^10.0.0",
     "body-parser": "^1.19.0",
     "chai": "^4.3.4",
     "chai-exclude": "^2.0.3",
@@ -72,10 +79,12 @@
     "eslint-plugin-node": "^11.1.0",
     "express": "^4.17.1",
     "express-pouchdb": "^4.2.0",
+    "husky": "^7.0.2",
     "mocha": "^6.2.3",
     "nyc": "^15.1.0",
     "pouchdb-adapter-memory": "^7.2.2",
     "rewire": "^5.0.0",
+    "semantic-release": "^18.0.0",
     "sinon": "^7.3.2"
   }
 }

package/src/fn/move-contacts.js

@@ -10,6 +10,7 @@ const pouch = require('../lib/db');
 const { warn, trace, info, error } = require('../lib/log');
 
 const HIERARCHY_ROOT = 'root';
+const BATCH_SIZE = 10000;
 
 module.exports = {
   requiresInstance: true,
@@ -49,19 +50,15 @@ const updateLineagesAndStage = async (options, db) => {
     trace(`Considering primary contact updates to ${ancestors.length} ancestor(s) of contact ${prettyPrintDocument(contactDoc)}.`);
     const updatedAncestors = replaceLineageInAncestors(descendantsAndSelf, ancestors);
 
-    const reportsCreatedByDescendants = await fetch.reportsCreatedBy(db, descendantsAndSelf.map(descendant => descendant._id));
-    trace(`${reportsCreatedByDescendants.length} report(s) created by these affected contact(s) will update`);
-    const updatedReports = replaceLineageInReports(reportsCreatedByDescendants, replacementLineage, contactId);
+    minifyLineageAndWriteToDisk([...updatedDescendants, ...updatedAncestors], options);
 
-    [...updatedDescendants, ...updatedReports, ...updatedAncestors].forEach(updatedDoc => {
-      lineageManipulation.minifyLineagesInDoc(updatedDoc);
-      writeDocumentToDisk(options, updatedDoc);
-    });
+    const movedReportsCount = await moveReports(db, descendantsAndSelf, options, replacementLineage, contactId);
+    trace(`${movedReportsCount} report(s) created by these affected contact(s) will be updated`);
 
     affectedContactCount += updatedDescendants.length + updatedAncestors.length;
-    affectedReportCount += updatedReports.length;
+    affectedReportCount += movedReportsCount;
 
-    info(`Staged updates to ${prettyPrintDocument(contactDoc)}. ${updatedDescendants.length} contact(s) and ${updatedReports.length} report(s).`);
+    info(`Staged updates to ${prettyPrintDocument(contactDoc)}. ${updatedDescendants.length} contact(s) and ${movedReportsCount} report(s).`);
   }
 
   info(`Staged changes to lineage information for ${affectedContactCount} contact(s) and ${affectedReportCount} report(s).`);
@@ -155,6 +152,31 @@ ${bold('OPTIONS')}
 `);
 };
 
+const moveReports = async (db, descendantsAndSelf, writeOptions, replacementLineage, contactId) => {
+  const contactIds = descendantsAndSelf.map(contact => contact._id);
+
+  let skip = 0;
+  let reportDocsBatch;
+  do {
+    info(`Processing ${skip} to ${skip + BATCH_SIZE} report docs`);
+    reportDocsBatch = await fetch.reportsCreatedBy(db, contactIds, skip);
+
+    const updatedReports = replaceLineageInReports(reportDocsBatch, replacementLineage, contactId);
+    minifyLineageAndWriteToDisk(updatedReports, writeOptions);
+
+    skip += reportDocsBatch.length;
+  } while (reportDocsBatch.length >= BATCH_SIZE);
+
+  return skip;
+};
+
+const minifyLineageAndWriteToDisk = (docs, parsedArgs) => {
+  docs.forEach(doc => {
+    lineageManipulation.minifyLineagesInDoc(doc);
+    writeDocumentToDisk(parsedArgs, doc);
+  });
+};
+
 const writeDocumentToDisk = ({ docDirectoryPath }, doc) => {
   const destinationPath = path.join(docDirectoryPath, `${doc._id}.doc.json`);
   if (fs.exists(destinationPath)) {
@@ -214,10 +236,12 @@ const fetch = {
       .filter(doc => doc && doc.type !== 'tombstone');
   },
 
-  reportsCreatedBy: async (db, contactIds) => {
+  reportsCreatedBy: async (db, contactIds, skip) => {
     const reports = await db.query('medic-client/reports_by_freetext', {
       keys: contactIds.map(id => [`contact:${id}`]),
       include_docs: true,
+      limit: BATCH_SIZE,
+      skip: skip,
     });
 
     return reports.rows.map(row => row.doc);

package/src/lib/api.js

@@ -87,6 +87,24 @@ const api = {
     });
   },
 
+  /**
+   * Check whether the API is alive or not. The request
+   * is made to a "lightweight" endpoint that only returns a HTTP 302,
+   * and an promise is returned with true (available) or false.
+   */
+  available: async () => {
+    const url = `${environment.apiUrl}/`;
+    try {
+      log.info(`Checking that ${url} is available...`);
+      await request.get(url);
+      return true;
+    } catch (err) {
+      log.error(`Failed to get a response from ${url}. Maybe you entered the wrong URL, `
+                + 'wrong port or the instance is not started? Please check and try again.');
+      return false;
+    }
+  },
+
   version() {
     return request({ uri: `${environment.instanceUrl}/api/deploy-info`, method: 'GET', json: true }) // endpoint added in 3.5
       .then(deploy_info => deploy_info && deploy_info.version);

package/src/lib/archiving-api.js

@@ -21,6 +21,8 @@ const archivingApi = {
     return Promise.resolve('{ "success": true }');
   },
 
+  available: async () => true,
+
   version() {
     return '1000.0.0'; // assume the latest version when archiving
   },

package/src/lib/main.js

@@ -10,7 +10,7 @@ const redactBasicAuth = require('redact-basic-auth');
 const shellCompletionSetup = require('../cli/shell-completion-setup');
 const supportedActions = require('../cli/supported-actions');
 const usage = require('../cli/usage');
-const request = require('request-promise-native');
+const api = require('../lib/api');
 
 const { error, info, warn } = log;
 const defaultActions = [
@@ -166,13 +166,6 @@ module.exports = async (argv, env) => {
       error('Failed to obtain a url to the API');
       return -1;
     }
-    try {
-      info(`Checking that ${apiUrl} is available...`);
-      await request.get(apiUrl);
-    } catch (err) {
-      error(`Failed to get a response from ${apiUrl}. Maybe you entered the wrong URL, wrong port or the instance is not started? Please check and try again.`);
-      return -1;
-    }
   }
 
   let extraArgs = cmdArgs['--'];
@@ -190,6 +183,10 @@ module.exports = async (argv, env) => {
     cmdArgs['skip-translation-check']
   );
 
+  if (apiUrl && !await api().available()) {
+    return false;
+  }
+
   const productionUrlMatch = environment.instanceUrl && environment.instanceUrl.match(/^https:\/\/(?:[^@]*@)?(.*)\.(app|dev)\.medicmobile\.org(?:$|\/)/);
   const expectedOptions = ['alpha', projectName];
   if (productionUrlMatch && !expectedOptions.includes(productionUrlMatch[1])) {