cht-conf 5.6.0 → 6.0.0

package/.mocharc.js

@@ -0,0 +1,19 @@
+const chai = require('chai');
+chai.use(require('chai-as-promised'));
+chai.use(require('chai-exclude'));
+chai.use(require('chai-shallow-deep-equal'));
+chai.use(require('sinon-chai'));
+chai.use(require('chai-xml'));
+
+module.exports = {
+  allowUncaught: false,
+  asyncOnly: false,
+  exclude: '../../test/e2e/**/*.spec.js',
+  exit: true,
+  color: true,
+  fullTrace: true,
+  recursive: true,
+  reporter: 'spec',
+  // These tests get run from build/test
+  spec: '../../test/**/*.spec.js',
+};

package/CHANGELOG.md

@@ -1,3 +1,16 @@
+# [6.0.0](https://github.com/medic/cht-conf/compare/v5.6.0...v6.0.0) (2026-01-12)
+
+
+* feat([#537](https://github.com/medic/cht-conf/issues/537))!: error when form internalId does not match file name ([#776](https://github.com/medic/cht-conf/issues/776)) ([f921f1e](https://github.com/medic/cht-conf/commit/f921f1e1c6c0e32f8d2a6b5f00c9700eab338700))
+* feat([#613](https://github.com/medic/cht-conf/issues/613))!: uplift pyxform to `v4.0.0-medic` ([#709](https://github.com/medic/cht-conf/issues/709)) ([d78296d](https://github.com/medic/cht-conf/commit/d78296d4df7a43699a1d7a78584be4b8cc3967f9))
+
+
+### BREAKING CHANGES
+
+* Requires consistent naming of form internalId.
+* New version of pyxform is bundled with additional form
+verifications.
+
 # [5.6.0](https://github.com/medic/cht-conf/compare/v5.5.0...v5.6.0) (2025-12-15)
 
 

package/Dockerfile

@@ -7,23 +7,14 @@ RUN apt update \
       curl \
       git \
       openssh-client \
-      python3-pip \
-      python3-setuptools \
-      python3-venv \
-      python3-wheel \
+      python3 \
       xsltproc \
     # Remove chromium to save space. We only installed it to get the transitive dependencies that are needed
     # when running tests with puppeteer. (puppeteer-chromium-resolver will always download its own version of chromium)
     && apt remove -y chromium \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install -g cht-conf
 
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv "$VIRTUAL_ENV"
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-RUN python3 -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
-
-RUN npm install -g cht-conf
 
 # Using the 1000:1000 user is recommended for VSCode dev containers
 # https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user

package/README.md

@@ -1,4 +1,5 @@
 # CHT Conf
+
 ![GitHub license](https://img.shields.io/github/license/medic/cht-conf)
 ![GitHub release](https://img.shields.io/github/v/release/medic/cht-conf)
 [![Discourse](https://img.shields.io/badge/discourse-join-blue)](https://forum.communityhealthtoolkit.org/)
@@ -9,7 +10,7 @@ Instructions on how to use this tool can be found on [the CHT documentation site
 
 ## Copyright
 
-Copyright 2013-2025 Medic Mobile, Inc. <hello@medic.org>
+Copyright 2013-2026 Medic Mobile, Inc. <hello@medic.org>
 
 ## License
 

package/bin/README.md

@@ -0,0 +1,7 @@
+# cht-conf binaries
+
+## xls2xform-medic
+
+This is a [custom version](https://github.com/medic/pyxform) of [pyxform](https://github.com/XLSForm/pyxform).
+
+The `xls2xform-medic` binary [is packaged](https://docs.communityhealthtoolkit.org/community/contributing/code/releasing/pyxform/) and included here for convenience.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cht-conf",
-  "version": "5.6.0",
+  "version": "6.0.0",
   "description": "Configure CHT deployments",
   "main": "./src/lib/main.js",
   "engines": {
@@ -9,7 +9,7 @@
   "scripts": {
     "clean": "rm -rf ./build/",
     "eslint": "eslint 'src/**/*.js' test/*.js 'test/**/*.js'",
-    "test": "npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha \"../../test/**/*.spec.js\" --exclude \"../../test/e2e/**/*.spec.js\"",
+    "test": "npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha",
     "test-e2e": "mocha --config test/e2e/.mocharc.js",
     "semantic-release": "semantic-release"
   },
@@ -68,6 +68,7 @@
     "terser-webpack-plugin": "^5.3.11",
     "uuid": "^11.1.0",
     "webpack": "^5.102.1",
+    "xml-formatter": "^3.6.7",
     "xpath": "0.0.34"
   },
   "devDependencies": {
@@ -92,6 +93,7 @@
     "pouchdb-adapter-memory": "^9.0.0",
     "rewire": "^9.0.1",
     "semantic-release": "^25.0.2",
-    "sinon": "^21.0.0"
+    "sinon": "^21.0.0",
+    "sinon-chai": "^3.7.0"
   }
 }

package/src/fn/convert-contact-forms.js

@@ -19,10 +19,9 @@ const convertContactForm = (forms) => {
 
   return convertForms(environment.pathToProject, 'contact', {
     enketo: true,
-    force_data_node: 'data',
     forms: forms,
     transformer: (xml, path) => {
-      const type = path.replace(/.*\/(.*?)(-(create|edit))?\.xml/, '$1');
+      const type = path.replace(/.*\/(.*?)(-(create|edit))?\.xml.swp$/, '$1');
 
       if (PLACE_TYPES) {
         xml = xml

package/src/lib/convert-forms/handle-db-doc-ref.js

@@ -0,0 +1,49 @@
+const {
+  getNodes,
+  XPATH_MODEL,
+  SIMPLE_XPATH_PATTERN,
+  getPrimaryInstanceNodeChildPath,
+  getNode
+} = require('../forms-utils');
+
+const getNodesWithDbDocRef = xmlDoc => {
+  return getNodes(xmlDoc, `${XPATH_MODEL}/instance//*[@db-doc-ref]`)
+    .filter(node => node.getAttribute('db-doc-ref').trim() !== '');
+};
+
+const isFormReference = (primaryInstanceNode, dbDocRef) => dbDocRef === `/${primaryInstanceNode.getAttribute('id')}`;
+const isRelativePath = dbDocRef => !dbDocRef.startsWith('/') && SIMPLE_XPATH_PATTERN.test(dbDocRef);
+
+const getCompatibleDbDocRef = (xmlDoc, node) => {
+  const currentDbDocRef = node.getAttribute('db-doc-ref').trim();
+  const absolutePath = getPrimaryInstanceNodeChildPath(node);
+  const [, dataInstanceName] = absolutePath.split('/');
+  const primaryInstanceNode = getNode(xmlDoc, `${XPATH_MODEL}/instance/${dataInstanceName}`);
+  if (isFormReference(primaryInstanceNode, currentDbDocRef)) {
+    return `/${dataInstanceName}`;
+  }
+  if (isRelativePath(currentDbDocRef)) {
+    const targetNode = getNode(node, currentDbDocRef);
+    if (targetNode) {
+      return getPrimaryInstanceNodeChildPath(targetNode);
+    }
+  }
+
+  return currentDbDocRef;
+};
+
+module.exports = {
+  /**
+   * Pyxform is now using relative paths for any field expressions inside a repeat. The instance::db-doc-ref
+   * functionality in cht-core does not handle relative paths properly. So we replace these with absolute paths.
+   * Additionally, the way to set your form field to the _id value for document currently being written is to set the
+   * db-doc-ref to "/<form_id>". Technically the `form_id` was the node name of the primary instance in the XForm xml.
+   * However, Pyxform now just uses `data` as the node name of the primary instance node for all forms. So, the logic
+   * in cht-core for handling db-doc-ref="/<form_id>" breaks. This function also replaces any db-doc-ref attributes
+   * that reference the form's own id with the correct node name of the primary instance node.
+   */
+  handleDbDocRefs: (xmlDoc) => {
+    getNodesWithDbDocRef(xmlDoc)
+      .forEach((node) => node.setAttribute('db-doc-ref', getCompatibleDbDocRef(xmlDoc, node)));
+  }
+};

package/src/lib/convert-forms/handle-form-id.js

@@ -0,0 +1,58 @@
+const { getPrimaryInstanceNode } = require('../forms-utils');
+
+const CONTACT_SUB_DIR = 'contact';
+const FORM_PREFIXES = new Set([CONTACT_SUB_DIR, 'training']);
+/*
+  subDir - the name of the directory containing the form
+  fileName - the name of the form file (without extension)
+  formName - the base name of the form (without action suffix)
+ */
+const PATH_PATTERN = /^.*\/(?<subDir>[^/]+)\/(?<fileName>(?<formName>[^/]+?)(-(?<action>create|edit))?)\.xml.swp$/;
+
+const getPrefix = subDirectory => FORM_PREFIXES.has(subDirectory) ? `${subDirectory}:` : '';
+const getSuffix = (subDirectory, action) => {
+  if (!action) {
+    return '';
+  }
+  return `${subDirectory === CONTACT_SUB_DIR ? ':' : '-'}${action}`;
+};
+
+module.exports = {
+  /**
+   * Ensures the form_id in the XML matches the expected id derived from the form's file name. If the xml for a
+   * contact/training form has an id that matches the file name but does not have the `contact:` or `training:`
+   * prefix, it updates the id to the expected format. Throws an error if there is a mismatch that cannot be resolved.
+   */
+  handleFormId: (xmlDoc, path) => {
+    const { groups: {
+      subDir,
+      fileName,
+      formName,
+      action
+    } } = path.match(PATH_PATTERN);
+    const idFromPath = `${getPrefix(subDir)}${formName}${getSuffix(subDir, action)}`;
+    const dataNode = Array
+      .from(getPrimaryInstanceNode(xmlDoc).childNodes)
+      .find(node => node.nodeType === 1);
+    const idFromXml = dataNode.getAttribute('id');
+
+    if (idFromXml === idFromPath || idFromXml === idFromPath.replace(formName, 'PLACE_TYPE')) {
+      // We already have the correct id
+      return;
+    }
+    // If the form_id is empty on the xlsx settings tab, pyxform will set the filename.
+    // For contact/training forms, we want to update the default id to match the proper format.
+    if (idFromXml === fileName && FORM_PREFIXES.has(subDir)) {
+      dataNode.setAttribute('id', idFromPath);
+      const smsPrefix = dataNode.getAttribute('prefix');
+      dataNode.setAttribute('prefix', smsPrefix.replace(idFromXml, idFromPath));
+      return;
+    }
+
+    throw new Error(`The file name for the form [${
+      fileName
+    }] does not match the form_id in the xlsx [${
+      idFromXml
+    }]. Rename the form xlsx/xml files to match the form_id.`);
+  }
+};

package/src/lib/convert-forms/handle-media.js

@@ -0,0 +1,89 @@
+const { getNodes, removeNode, XPATH_MODEL, XPATH_BODY, getNode, getInstanceNode } = require('../forms-utils');
+
+const getDynamicDefaultNode = (xmlDoc) => (ref) => {
+  return getNode(
+    xmlDoc,
+    `${XPATH_MODEL}/setvalue[@ref='${ref}' and string-length(@value) > 0 and @event="odk-instance-first-load"]`
+  );
+};
+
+const moveDynamicDefaultValueToInstance = (xmlDoc) => (setValueNode) => {
+  const ref = setValueNode.getAttribute('ref');
+  const instanceNode = getNode(xmlDoc, `${XPATH_MODEL}/instance[1]${ref}`);
+  if (!instanceNode) {
+    return;
+  }
+
+  instanceNode.textContent = setValueNode.getAttribute('value');
+  removeNode(setValueNode);
+};
+
+const createItemForInstanceNode = (xmlDoc, parentNode) => (itemNode) => {
+  const children = Array.from(itemNode.childNodes);
+  const itextIdNode = children.find(n => n.nodeName === 'itextId');
+  const nameNode = children.find(n => n.nodeName === 'name');
+  if (!itextIdNode || !nameNode) {
+    return;
+  }
+
+  const itemElem = xmlDoc.createElement('item');
+  const labelElem = xmlDoc.createElement('label');
+  labelElem.setAttribute('ref', `jr:itext('${itextIdNode.textContent}')`);
+  itemElem.appendChild(labelElem);
+  const valueElem = xmlDoc.createElement('value');
+  valueElem.textContent = nameNode.textContent;
+  itemElem.appendChild(valueElem);
+  parentNode.appendChild(itemElem);
+};
+
+const insertSelectItemsWithMedia = (xmlDoc, instanceNode) => (itemSetNode) => {
+  const { parentNode } = itemSetNode;
+  getNodes(instanceNode, 'root/item')
+    .forEach(createItemForInstanceNode(xmlDoc, parentNode));
+  removeNode(itemSetNode);
+};
+
+const insertMediaSelectItemsForInstance = (xmlDoc) => (instanceNode) => {
+  const instanceId = instanceNode.getAttribute('id');
+  getNodes(xmlDoc, `${XPATH_BODY}//itemset[@nodeset="instance('${instanceId}')/root/item"]`)
+    .forEach(insertSelectItemsWithMedia(xmlDoc, instanceNode));
+};
+
+module.exports = {
+  /**
+   * When setting a base64 image string as a default value in a form, pyxform thinks it is "dynamic". This causes the
+   * value to not be displayed properly. So, in cases were a default value is being used for a display-base64-image
+   * field, this function will move the default value into the instance node where it will be treated as a static value.
+   * https://github.com/XLSForm/pyxform/issues/495
+   */
+  replaceBase64ImageDynamicDefaults: (xmlDoc) => {
+    getNodes(xmlDoc, `${XPATH_BODY}//input[contains(@appearance, 'display-base64-image')]`)
+      .map(node => node.getAttribute('ref'))
+      .map(getDynamicDefaultNode(xmlDoc))
+      .filter(Boolean)
+      .forEach(moveDynamicDefaultValueToInstance(xmlDoc));
+  },
+
+  /**
+   * Selects no longer include all item elements in the body, but instead the choices data is stored centrally as an
+   * instance and referenced in the body as an itemset. Unfortunately, this approach breaks the CHT logic for properly
+   * updating the media src urls associated with the choices. So, for these selects, we replace the itemset with the
+   * actual item elements, which the CHT logic can then handle as before.
+   *
+   * This is a temporary workaround until we can update the CHT logic to properly handle itemsets with media.
+   * https://github.com/XLSForm/pyxform/pull/614
+   */
+  replaceItemSetsWithMedia: (xmlDoc) => {
+    const mediaXpath = `${XPATH_MODEL}/itext/translation/text[value[@form="image" or @form="audio" or @form="video"]]`;
+    const allInstanceIdsWithMedia = getNodes(xmlDoc, mediaXpath)
+      .map(textNode => textNode.getAttribute('id'))
+      .map(id => id.match(/^(.+)-\d+$/))
+      .filter(Boolean)
+      .map(([, match]) => match);
+    const instanceNodes = Array
+      .from(new Set(allInstanceIdsWithMedia))
+      .map(instanceId => getInstanceNode(xmlDoc, instanceId))
+      .filter(Boolean);
+    instanceNodes.forEach(insertMediaSelectItemsForInstance(xmlDoc));
+  }
+};

package/src/lib/convert-forms/handle-no-label-placeholders.js

@@ -0,0 +1,23 @@
+const { getNodes, removeNode, XPATH_MODEL, XPATH_BODY } = require('../forms-utils');
+
+module.exports = {
+  /**
+   * Removes the "NO_LABEL" labels placeholder. XLSForm does not allow converting a field without a label, so the CHT
+   * convention is to use a "NO_LABEL" placeholder value.
+   */
+  removeNoLabelNodes: (xmlDoc) => {
+    const noLabelItextNodes = getNodes(
+      xmlDoc,
+      `${XPATH_MODEL}/itext/translation//text[count(*)=1 and (value="NO_LABEL" or value="DELETE_THIS_LINE")]`
+    );
+    const noLabelNodeIds = Array.from(new Set(noLabelItextNodes.map(textNode => textNode.getAttribute('id'))));
+    noLabelNodeIds
+      .flatMap(id => getNodes(xmlDoc, `${XPATH_BODY}//label[@ref="jr:itext('${id}')"]`))
+      .forEach(removeNode);
+    noLabelItextNodes.forEach(removeNode);
+
+    // Remove any additional NO_LABEL values from translation nodes that have other (multimedia) values
+    getNodes(xmlDoc, `${XPATH_MODEL}/itext/translation//value[text()="NO_LABEL" or text()="DELETE_THIS_LINE"]`)
+      .forEach(removeNode);
+  }
+};

package/src/lib/convert-forms/handle-repeat.js

@@ -0,0 +1,83 @@
+const { getNode, getNodes, removeNode, XPATH_MODEL, XPATH_BODY, SIMPLE_XPATH_PATTERN } = require('../forms-utils');
+
+const repeatCountPath = (nodeset) => `${nodeset}_count`;
+
+const repeatBodyNodeNeedsCountField = (bodyNode) => {
+  const nodeset = bodyNode
+    .getAttribute('nodeset')
+    .trim();
+  const jrCount = bodyNode
+    .getAttribute('jr:count')
+    .trim();
+  return nodeset && jrCount && jrCount !== repeatCountPath(nodeset) && SIMPLE_XPATH_PATTERN.test(jrCount);
+};
+
+const repeatCountDoesNotExist = (xmlDoc) => (bodyNode) => {
+  const path = repeatCountPath(bodyNode.getAttribute('nodeset'));
+  const existingField = getNode(xmlDoc, `${XPATH_MODEL}/instance${path}`);
+  return !existingField;
+};
+
+const addRepeatCountToInstance = (xmlDoc) => (bodyNode) => {
+  const path = repeatCountPath(bodyNode.getAttribute('nodeset'));
+  const pathSegments = path.split('/');
+  const fieldName = pathSegments.pop();
+  const parentPath = pathSegments.join('/');
+  const parentNode = getNode(xmlDoc, `${XPATH_MODEL}/instance${parentPath}`);
+  if (!parentNode) {
+    return false;
+  }
+  const newFieldNode = xmlDoc.createElement(fieldName);
+  parentNode.appendChild(newFieldNode);
+  return true;
+};
+
+const addRepeatCountBind = (xmlDoc) => (bodyNode) => {
+  const nodeset = repeatCountPath(bodyNode.getAttribute('nodeset'));
+  const bindNode = xmlDoc.createElement('bind');
+  bindNode.setAttribute('nodeset', nodeset);
+  bindNode.setAttribute('type', 'string');
+  bindNode.setAttribute('readonly', 'true()');
+  bindNode.setAttribute('calculate', bodyNode.getAttribute('jr:count'));
+  const modelNode = getNode(xmlDoc, XPATH_MODEL);
+  modelNode.appendChild(bindNode);
+  return bodyNode;
+};
+
+const updateJrCount = (bodyNode) => {
+  const jrCount = repeatCountPath(bodyNode.getAttribute('nodeset'));
+  bodyNode.setAttribute('jr:count', jrCount);
+  return bodyNode;
+};
+
+module.exports = {
+  /**
+   * Pyxform now will not always generate a `${repeat_name}_count` field for repeats with a dynamic repeat_count.
+   * When the repeat_count is a simple reference to another field Pyxform omits generating the `${repeat_name}_count`
+   * field and instead just directly references the other field in the jr:count attribute of the repeat in the body.
+   * Unfortunately, this can cause buggy behavior in Enketo. So, to ensure consistent behavior, we add the
+   * `${repeat_name}_count` field back into the instance model and update the jr:count attribute to reference it.   *
+   * https://github.com/XLSForm/pyxform/issues/576
+   */
+  addRepeatCount: (xmlDoc) => {
+    getNodes(xmlDoc, `${XPATH_BODY}//repeat[@jr:count]`)
+      .filter(repeatBodyNodeNeedsCountField)
+      .filter(repeatCountDoesNotExist(xmlDoc))
+      .filter(addRepeatCountToInstance(xmlDoc))
+      .map(addRepeatCountBind(xmlDoc))
+      .forEach(updateJrCount);
+  },
+
+  /**
+   * Pyxform now inserts an initial entry into repeats directly in the instance model even when there is no
+   * repeat count. This causes the form to load with an entry already present in the repeat (as if the user already
+   * clicked the `+` button to add one). To maintain previous behavior, we remove this initial entry.
+   * https://github.com/XLSForm/pyxform/issues/182
+   */
+  removeExtraRepeatInstance: (xmlDoc) => {
+    getNodes(xmlDoc, `${XPATH_MODEL}/instance//*[@jr:template=""]`)
+      .map(({ parentNode, nodeName }) => getNode(parentNode, `./${nodeName}[not(@jr:template="")]`))
+      .filter(Boolean)
+      .forEach(removeNode);
+  }
+};

package/src/lib/convert-forms/index.js

@@ -0,0 +1,196 @@
+const argsFormFilter = require('../args-form-filter');
+const exec = require('../exec-promise');
+const fs = require('../sync-fs');
+const nodeFs = require('node:fs');
+const {
+  getFormDir,
+  escapeWhitespacesInPath,
+} = require('../forms-utils');
+const { info, trace, warn, LEVEL_NONE } = require('../log');
+const path = require('node:path');
+const { DOMParser, XMLSerializer } = require('@xmldom/xmldom');
+const xmlFormat = require('xml-formatter');
+const { replaceBase64ImageDynamicDefaults, replaceItemSetsWithMedia } = require('./handle-media');
+const { removeNoLabelNodes } = require('./handle-no-label-placeholders');
+const { removeExtraRepeatInstance, addRepeatCount } = require('./handle-repeat');
+const { handleDbDocRefs } = require('./handle-db-doc-ref');
+const { handleFormId } = require('./handle-form-id');
+
+const domParser = new DOMParser();
+const serializer = new XMLSerializer();
+const XLS2XFORM = path.join(__dirname, '..', '..', '..', 'bin', 'xls2xform-medic');
+
+const FORM_EXTENSION = '.xlsx';
+const formFileMatcher = (fileName) => {
+  if (fileName.endsWith(FORM_EXTENSION) &&
+    !fileName.startsWith('~$') // ignore Excel "owner files"
+    && fileName !== 'PLACE_TYPE-create.xlsx' && fileName !== 'PLACE_TYPE-edit.xlsx') {
+    return fileName.slice(0, fileName.length - FORM_EXTENSION.length);
+  }
+  return null;
+};
+
+const execute = async (projectDir, subDirectory, options = {}) => {
+  const formsDir = getFormDir(projectDir, subDirectory);
+
+  if (!fs.exists(formsDir)) {
+    warn(`Forms dir not found: ${formsDir}`);
+    return;
+  }
+
+  const filesToConvert = argsFormFilter(formsDir, FORM_EXTENSION, options)
+    .filter(name => formFileMatcher(name));
+
+  for (const xls of filesToConvert) {
+    const sourcePath = `${formsDir}/${xls}`;
+    const formPathNoExt = fs.withoutExtension(sourcePath);
+    const xmlSwpPath = `${formPathNoExt}.xml.swp`;
+    const targetPath = `${formPathNoExt}.xml`;
+
+    info('Converting form', sourcePath, '…');
+    nodeFs.rmSync(targetPath, { force: true });
+
+    try {
+      await xls2xform(escapeWhitespacesInPath(sourcePath), escapeWhitespacesInPath(xmlSwpPath), xls);
+      const hiddenFields = await getHiddenFields(`${fs.withoutExtension(sourcePath)}.properties.json`);
+      fixXml(xmlSwpPath, hiddenFields, options.transformer, options.enketo);
+    } catch (e) {
+      nodeFs.rmSync(xmlSwpPath, { force: true });
+      throw e;
+    }
+    nodeFs.renameSync(xmlSwpPath, targetPath);
+
+    trace('Converted form', sourcePath);
+  }
+};
+
+module.exports = {
+  SUPPORTED_EXTENSIONS: [FORM_EXTENSION],
+  formFileMatcher,
+  execute
+};
+
+const PYXFORM_EMPTY_GROUP_ERROR = '\'NoneType\' object is not iterable';
+const PYXFORM_CODES = {
+  OK: 100,
+  WARNING: 101,
+  // ERROR: 999
+};
+
+const pyxformErrorMessage = (e) => {
+  const errorMsg = typeof e === `string` ? e : e.message || JSON.stringify(e);
+  if (PYXFORM_EMPTY_GROUP_ERROR === errorMsg) {
+    // Pyxform does not gracefully handle empty groups https://github.com/XLSForm/pyxform/issues/754
+    return `Check the form for an empty group or repeat.`;
+  }
+  return errorMsg;
+};
+
+const xls2xform = async (sourcePath, targetPath, xlsxFileName) => {
+  const result = await exec([XLS2XFORM, '--skip_validate', '--json', sourcePath, targetPath], LEVEL_NONE)
+    .then((output) => {
+      const entries = output
+        .split('\n')
+        .map(s => s.trim())
+        .filter(Boolean);
+      // It is possible for pyxform dependencies to log to stdout, so we take the last line
+      entries
+        .slice(0, -1)
+        .forEach(line => warn(line));
+      return JSON.parse(entries.at(-1));
+    })
+    .catch(e => {
+      throw new Error(`There was a problem executing xls2xform. Make sure you have Python 3.10+ installed.\n${
+        pyxformErrorMessage(e)
+      }`);
+    });
+  const {
+    code,
+    warnings = [],
+  } = result;
+
+  switch (code) {
+  case PYXFORM_CODES.OK:
+    break;
+  case PYXFORM_CODES.WARNING:
+    warn(`Converted ${xlsxFileName} with warnings:`);
+    warnings.forEach(w => warn(w));
+    break;
+  default:
+    throw new Error(`Could not convert ${xlsxFileName}: ${pyxformErrorMessage(result)}`);
+  }
+};
+
+// here we fix the form content in arcane ways.  Seeing as we have out own fork
+// of pyxform, we should probably be doing this fixing there.
+const fixXml = (path, hiddenFields, transformer, enketo) => {
+  // This is not how you should modify XML, but we have reasonable control over
+  // the input and so far this works OK.  Keep an eye on the tests, and any
+  // future changes to the output of xls2xform.
+  let xml = fs.read(path)
+
+    // The following copies behaviour from old bash scripts, and will create a
+    // second <meta> element if one already existed.  We may want to actually
+    // merge the two instead.
+    .replace(/<inputs>/, META_XML_SECTION);
+
+  // Enketo _may_ not work with forms which define a default language - see
+  // https://github.com/medic/cht-core/issues/3174
+  if (enketo) {
+    xml = xml.replaceAll('default="true()"', '');
+  }
+
+  if (hiddenFields) {
+    const r = new RegExp(`<(${hiddenFields.join('|')})(/?)>`, 'g');
+    xml = xml.replace(r, '<$1 tag="hidden"$2>');
+  }
+
+  // Check for deprecations
+  if (xml.includes('repeat-relevant')) {
+    warn('From webapp version 2.14.0, repeat-relevant is no longer required.  See https://github.com/medic/cht-core/issues/3449 for more info.');
+  }
+  const xmlDoc = domParser.parseFromString(xml);
+
+  handleFormId(xmlDoc, path);
+  replaceItemSetsWithMedia(xmlDoc);
+  replaceBase64ImageDynamicDefaults(xmlDoc);
+  removeNoLabelNodes(xmlDoc);
+  removeExtraRepeatInstance(xmlDoc);
+  addRepeatCount(xmlDoc);
+  handleDbDocRefs(xmlDoc);
+
+  const xmlString = serializer.serializeToString(xmlDoc);
+  xml = xmlFormat(xmlString, {
+    collapseContent: true,
+    forceSelfClosingEmptyTag: true,
+    indentation: '  ',
+    ignoredPaths: [
+      'value' // Avoid trimming whitespace in the label text.
+    ],
+    lineSeparator: '\n'
+  }).replaceAll(/\s+<\/value>/g, '</value>'); // Ignoring the 'value' path results in extra trailing whitespace
+
+  if (transformer) {
+    xml = transformer(xml, path);
+  }
+
+  fs.write(path, xml);
+};
+
+function getHiddenFields(propsJson) {
+  if (fs.exists(propsJson)) {
+    return fs.readJson(propsJson).hidden_fields;
+  }
+
+  return [];
+}
+
+const META_XML_SECTION = `<inputs>
+            <meta>
+              <location>
+                <lat/>
+                <long/>
+                <error/>
+                <message/>
+              </location>
+            </meta>`;

package/src/lib/exec-promise.js

@@ -25,7 +25,7 @@ module.exports = (args, logLevel=log.level) => new Promise((resolve, reject) =>
         reject(stderr);
       }
       else {
-        resolve(stdout);
+        resolve(stdout || stderr);
       }
     });
 

package/src/lib/forms-utils.js

@@ -2,6 +2,7 @@ const xpath = require('xpath');
 const fs = require('./sync-fs');
 
 const XPATH_MODEL = '/h:html/h:head/model';
+const XPATH_BODY = '/h:html/h:body';
 
 const getNode = (currentNode, path) =>
   xpath.parse(path).select1({ node: currentNode, allowAnyNamespaceForNoPrefix: true });
@@ -18,6 +19,15 @@ const getFullNodePath = (childNode) => {
 };
 
 module.exports = {
+  XPATH_MODEL,
+  XPATH_BODY,
+
+  /**
+   * Matches XPath expressions that are only paths to a node (either absolute or relative) without any
+   * predicates, functions, operators, etc.
+   */
+  SIMPLE_XPATH_PATTERN: /^[/\w.-]+$/,
+
   /**
    * Get the full path of the form, or null if the path doesn't exist.
    * @returns {string|null}
@@ -46,6 +56,12 @@ module.exports = {
     };
   },
 
+  /**
+   * Removes the given XML node from its parent. The xmldom library does not support node.remove(), so we use
+   * parentNode.removeChild(node).
+   */
+  removeNode: node => node.parentNode.removeChild(node), // NOSONAR
+
   /**
    * Returns the node from the form XML specified by the given XPath.
    * @param {Element} currentNode the current node in the form XML document
@@ -69,6 +85,14 @@ module.exports = {
    */
   getBindNodes: xmlDoc => getNodes(xmlDoc, `${XPATH_MODEL}/bind`),
 
+  /**
+   * Returns the `instance` node with the given ID for the given form XML.
+   * @param {Document} xmlDoc the form XML document
+   * @param instanceId the id of the instance
+   * @returns {Element} the selected node or `undefined` if not found
+   */
+  getInstanceNode: (xmlDoc, instanceId) => getNode(xmlDoc, `${XPATH_MODEL}/instance[@id='${instanceId}']`),
+
   /**
    * Returns the primary (first) `instance` node for the given form XML.
    * @param {Document} xmlDoc the form XML document
@@ -103,21 +127,10 @@ module.exports = {
   readTitleFrom: xml =>
     xml.substring(xml.indexOf('<h:title>') + 9, xml.indexOf('</h:title>')),
 
-  /**
-   * Get the ID of the form
-   * @param {string} xml the XML string
-   * @returns {string}
-   */
-  readIdFrom: xml =>
-    xml.match(/<model>[^]*<\/model>/)[0]
-      .match(/<instance>[^]*<\/instance>/)[0]
-      .match(/id="([^"]*)"/)[1],
-
   /**
    * Escape whitespaces in a path.
    * @param {string} path the path string
    * @returns {string}
    */
   escapeWhitespacesInPath: path => path.replace(/(\s+)/g, '\\$1'),
-  
 };

package/src/lib/upload-forms.js

@@ -11,7 +11,6 @@ const {
   getFormDir,
   getFormFilePaths,
   readTitleFrom,
-  readIdFrom
 } = require('./forms-utils');
 
 const SUPPORTED_PROPERTIES = ['context', 'icon', 'title', 'xml2sms', 'subject_key', 'hidden_fields'];
@@ -60,7 +59,7 @@ const execute = async (projectDir, subDirectory, options) => {
     log.info(`Preparing form for upload: ${fileName}…`);
 
     const { baseFileName, mediaDir, xformPath, filePath } = getFormFilePaths(formsDir, fileName);
-    const baseDocId = (options.id_prefix || '') + baseFileName.replace(/-/g, ':');
+    const internalId = (options.id_prefix || '') + baseFileName.replaceAll('-', ':');
 
     const mediaDirExists = fs.exists(mediaDir);
     if (!mediaDirExists) {
@@ -75,12 +74,7 @@ const execute = async (projectDir, subDirectory, options) => {
       sha256: hashSum.digest('hex'),
     };
 
-    const internalId = readIdFrom(xml);
-    if (internalId !== baseDocId) {
-      log.warn('DEPRECATED', 'Form:', fileName, 'Bad ID set in XML.  Expected:', baseDocId, 'but saw:', internalId, ' Support for setting these values differently will be dropped.  Please see https://github.com/medic/cht-core/issues/3342.');
-    }
-
-    const docId = `form:${baseDocId}`;
+    const docId = `form:${internalId}`;
     const doc = {
       _id: docId,
       type: 'form',
@@ -91,7 +85,7 @@ const execute = async (projectDir, subDirectory, options) => {
     };
 
     const propertiesPath = `${formsDir}/${baseFileName}${FORM_PROPERTIES_EXTENSION}`;
-    updateFromPropertiesFile(doc, propertiesPath, PROPERTIES);
+    updateFromPropertiesFile(baseFileName, doc, propertiesPath, PROPERTIES);
 
     doc._attachments = mediaDirExists ? attachmentsFromDir(mediaDir) : {};
     doc._attachments.xml = attachmentFromFile(xformPath);
@@ -118,24 +112,36 @@ module.exports = {
   execute
 };
 
-const updateFromPropertiesFile = (doc, path, supported_properties) => {
+const updateFromPropertiesFile = (baseFileName, doc, path, supported_properties) => {
   if (fs.exists(path)) {
 
     const ignoredKeys = [];
     const properties = fs.readJson(path);
 
-    Object.keys(properties).forEach(key => {
-      if (typeof properties[key] !== 'undefined') {
+    Object
+      .keys(properties)
+      .forEach(key => {
+        if (properties[key] === undefined) {
+          return;
+        }
         if (supported_properties.includes(key)) {
           doc[key] = properties[key];
-        } else if (key === 'internalId') {
+          return;
+        }
+        if (key === 'internalId') {
           log.warn(`DEPRECATED: ${path}. Please do not manually set internalId in .properties.json for new projects. Support for configuring this value will be dropped. Please see https://github.com/medic/cht-core/issues/3342.`);
-          doc.internalId = properties.internalId;
-        } else {
-          ignoredKeys.push(key);
+          if (doc.internalId === properties.internalId) {
+            return;
+          }
+          throw new Error(`The file name for the form [${
+            baseFileName
+          }] does not match the internalId in the ${baseFileName}.properties.json [${
+            properties.internalId
+          }]. Rename the form xlsx/xml files to match the internalId.`);
         }
-      }
-    });
+
+        ignoredKeys.push(key);
+      });
 
     if (ignoredKeys.length) {
       log.warn(`Ignoring unknown properties in ${path}: ${ignoredKeys.join(', ')}`);

package/src/lib/convert-forms.js

@@ -1,166 +0,0 @@
-const { execSync } = require('child_process');
-
-const argsFormFilter = require('./args-form-filter');
-const exec = require('./exec-promise');
-const fs = require('./sync-fs');
-const { getFormDir, escapeWhitespacesInPath } = require('./forms-utils');
-const { info, trace, warn } = require('./log');
-
-const XLS2XFORM = 'xls2xform-medic';
-const INSTALLATION_INSTRUCTIONS = `\nE To install the latest pyxform, try one of the following:
-E
-E Ubuntu
-E	sudo python -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
-E OSX
-E	pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic
-E Windows (as Administrator)
-E	python -m pip install git+https://github.com/medic/pyxform.git@medic-conf-1.17#egg=pyxform-medic --upgrade`;
-const UPDATE_INSTRUCTIONS = `\nE To remove the old version of pyxform:
-E
-E	pip uninstall pyxform-medic
-E` + INSTALLATION_INSTRUCTIONS;
-
-const FORM_EXTENSION = '.xlsx';
-const formFileMatcher = (fileName) => {
-  if (fileName.endsWith(FORM_EXTENSION) &&
-    !fileName.startsWith('~$') // ignore Excel "owner files"
-    && fileName !== 'PLACE_TYPE-create.xlsx' && fileName !== 'PLACE_TYPE-edit.xlsx') {
-    return fileName.slice(0, fileName.length - FORM_EXTENSION.length);
-  }
-  return null;
-};
-
-const execute = async (projectDir, subDirectory, options) => {
-  if (!options) {
-    options = {};
-  }
-
-  const formsDir = getFormDir(projectDir, subDirectory);
-
-  if (!fs.exists(formsDir)) {
-    warn(`Forms dir not found: ${formsDir}`);
-    return Promise.resolve();
-  }
-
-  const filesToConvert = argsFormFilter(formsDir, FORM_EXTENSION, options)
-    .filter(name => formFileMatcher(name));
-
-  for (const xls of filesToConvert) {
-    const originalSourcePath = `${formsDir}/${xls}`;
-    let sourcePath;
-
-    if (options.force_data_node) {
-      const temporaryPath = `${fs.mkdtemp()}/${options.force_data_node}.xlsx`;
-      fs.copy(originalSourcePath, temporaryPath);
-      sourcePath = temporaryPath;
-    } else {
-      sourcePath = originalSourcePath;
-    }
-
-    const targetPath = `${fs.withoutExtension(originalSourcePath)}.xml`;
-
-    info('Converting form', originalSourcePath, '…');
-  
-    await xls2xform(escapeWhitespacesInPath(sourcePath), escapeWhitespacesInPath(targetPath));
-    const hiddenFields = await getHiddenFields(`${fs.withoutExtension(originalSourcePath)}.properties.json`);
-    await fixXml(targetPath, hiddenFields, options.transformer, options.enketo);
-    trace('Converted form', originalSourcePath);
-  }
-};
-
-module.exports = {
-  SUPPORTED_EXTENSIONS: [FORM_EXTENSION],
-  formFileMatcher,
-  execute
-};
-
-const xls2xform = (sourcePath, targetPath) =>
-  exec([XLS2XFORM, '--skip_validate', sourcePath, targetPath])
-    .catch(e => {
-      if (executableAvailable()) {
-        if (e.includes('unrecognized arguments: --skip_validate')) {
-          throw new Error('Your xls2xform installation appears to be out of date.' + UPDATE_INSTRUCTIONS);
-        } else {
-          throw e;
-        }
-      } else {
-        throw new Error(
-          'There was a problem executing xls2xform.  It may not be installed.' + INSTALLATION_INSTRUCTIONS
-        );
-      }
-    });
-
-// here we fix the form content in arcane ways.  Seeing as we have out own fork
-// of pyxform, we should probably be doing this fixing there.
-const fixXml = (path, hiddenFields, transformer, enketo) => {
-  // This is not how you should modify XML, but we have reasonable control over
-  // the input and so far this works OK.  Keep an eye on the tests, and any
-  // future changes to the output of xls2xform.
-  let xml = fs.read(path)
-
-    // The following copies behaviour from old bash scripts, and will create a
-    // second <meta> element if one already existed.  We may want to actually
-    // merge the two instead.
-    .replace(/<inputs>/, META_XML_SECTION)
-
-    // XLSForm does not allow converting a field without a label, so we use
-    // the placeholder NO_LABEL.
-    .replace(/NO_LABEL/g, '')
-
-    // No comment.
-    .replace(/.*DELETE_THIS_LINE.*(\r|\n)/g, '')
-    ;
-
-  // Enketo _may_ not work with forms which define a default language - see
-  // https://github.com/medic/cht-core/issues/3174
-  if (enketo) {
-    xml = xml.replace(/ default="true\(\)"/g, '');
-  }
-
-  if (hiddenFields) {
-    const r = new RegExp(`<(${hiddenFields.join('|')})(/?)>`, 'g');
-    xml = xml.replace(r, '<$1 tag="hidden"$2>');
-  }
-
-  if (transformer) {
-    xml = transformer(xml, path);
-  }
-
-
-  // Check for deprecations
-  if (xml.includes('repeat-relevant')) {
-    warn('From webapp version 2.14.0, repeat-relevant is no longer required.  See https://github.com/medic/cht-core/issues/3449 for more info.');
-  }
-
-  fs.write(path, xml);
-};
-
-function getHiddenFields(propsJson) {
-  if (!fs.exists(propsJson)) {
-    return [];
-  }
-  else {
-    return fs.readJson(propsJson).hidden_fields;
-  }
-}
-
-function executableAvailable() {
-  try {
-    execSync(`${XLS2XFORM} -h`, {
-      stdio: ['ignore', 'ignore', 'ignore'],
-    });
-    return true;
-  } catch (e) {
-    return false;
-  }
-}
-
-const META_XML_SECTION = `<inputs>
-            <meta>
-              <location>
-                <lat/>
-                <long/>
-                <error/>
-                <message/>
-              </location>
-            </meta>`;