cht-conf 3.11.1 → 3.12.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [3.12.0](https://github.com/medic/cht-conf/compare/v3.11.1...v3.12.0) (2022-06-23)
+
+
+### Features
+
+* **#484:** add actions for validating forms ([f2a6a8e](https://github.com/medic/cht-conf/commit/f2a6a8ef75561426ce2a6b542cdf1a255c869c6e)), closes [#484](https://github.com/medic/cht-conf/issues/484) [#481](https://github.com/medic/cht-conf/issues/481)
+
 ## [3.11.1](https://github.com/medic/cht-conf/compare/v3.11.0...v3.11.1) (2022-06-13)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cht-conf",
-  "version": "3.11.1",
+  "version": "3.12.0",
   "description": "Configure CHT deployments",
   "main": "./src/lib/main.js",
   "engines": {

package/src/cli/usage.js

@@ -63,6 +63,9 @@ ${bold('OPTIONS')}
   --skip-translation-check
     Skips checking message translations
 
+  --skip-validate
+    Skips form validation  
+
   --force
     CAN BE DANGEROUS! Passes yes to all commands and any where that would prompt to overwrite changes will overwrite automatically. 
 `);

package/src/fn/validate-app-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateAppForms = (forms) => {
+  return validateForms(environment.pathToProject, 'app', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateAppForms,
+  execute: () => validateAppForms(environment.extraArgs)
+};

package/src/fn/validate-collect-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateCollectForms = (forms) => {
+  return validateForms(environment.pathToProject, 'collect', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateCollectForms,
+  execute: () => validateCollectForms(environment.extraArgs)
+};

package/src/fn/validate-contact-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateContactForms = (forms) => {
+  return validateForms(environment.pathToProject, 'contact', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateContactForms,
+  execute: () => validateContactForms(environment.extraArgs)
+};

package/src/fn/watch-project.js

@@ -5,6 +5,9 @@ const fs = require('fs');
 const { error, warn, info } = require('../lib/log');
 const Queue = require('queue-promise');
 const watcher = require('@parcel/watcher');
+const { validateAppForms } = require('./validate-app-forms');
+const { validateContactForms } = require('./validate-contact-forms');
+const { validateCollectForms } = require('./validate-collect-forms');
 const { uploadAppForms } = require('./upload-app-forms');
 const { uploadContactForms } = require('./upload-contact-forms');
 const { uploadCollectForms } = require('./upload-collect-forms');
@@ -25,8 +28,17 @@ const watcherEvents = {
     UpdateEvent: 'update'
 };
 
+const runValidation = (validation, forms) => {
+    if(environment.skipValidate) {
+        return;
+    }
+    return validation(forms);
+};
+
 const uploadInitialState = async (api) => {
     await uploadResources();
+    await runValidation(validateAppForms, environment.extraArgs);
+    await runValidation(validateContactForms, environment.extraArgs);
     await uploadAppForms(environment.extraArgs);
     await uploadContactForms(environment.extraArgs);
     await uploadCustomTranslations();
@@ -82,6 +94,7 @@ const processAppForm = (eventType, fileName) => {
     let form = uploadForms.formFileMatcher(fileName);
     if (form) {
         eventQueue.enqueue(async () => {
+            await runValidation(validateAppForms, [form]);
             await uploadAppForms([form]);
             return fileName;
         });
@@ -103,6 +116,7 @@ const processAppFormMedia = (formMediaDir, fileName) => {
     const form = uploadForms.formMediaMatcher(formMediaDir);
     if (form) {
         eventQueue.enqueue(async () => {
+            await runValidation(validateAppForms,[form]);
             await uploadAppForms([form]);
             return fileName;
         });
@@ -127,6 +141,7 @@ const processContactForm = (eventType, fileName) => {
     form = uploadForms.formFileMatcher(fileName);
     if (form) {
         eventQueue.enqueue(async () => {
+            await runValidation(validateContactForms,[form]);
             await uploadContactForms([form]);
             return fileName;
         });
@@ -142,6 +157,7 @@ const processCollectForm = (eventType, fileName) => {
     let form = uploadForms.formFileMatcher(fileName);
     if (form) {
         eventQueue.enqueue(async () => {
+            await runValidation(validateCollectForms,[form]);
             await uploadCollectForms([form]);
             return fileName;
         });

package/src/lib/environment.js

@@ -12,7 +12,8 @@ const initialize = (
   extraArgs,
   apiUrl,
   force,
-  skipTranslationCheck
+  skipTranslationCheck,
+  skipValidate
 ) => {
   if (state.initialized) {
     throw Error('environment is already initialized');
@@ -26,7 +27,8 @@ const initialize = (
     isArchiveMode,
     pathToProject,
     force,
-    skipTranslationCheck
+    skipTranslationCheck,
+    skipValidate
   });
 };
 
@@ -51,6 +53,7 @@ module.exports = {
   get apiUrl() { return getState('apiUrl'); },
   get force() { return getState('force'); },
   get skipTranslationCheck() { return getState('skipTranslationCheck'); },
+  get skipValidate() { return getState('skipValidate'); },
 
   /**
    * Return `true` if the environment **seems** to be production.

package/src/lib/main.js

@@ -21,6 +21,9 @@ const defaultActions = [
   'convert-app-forms',
   'convert-collect-forms',
   'convert-contact-forms',
+  'validate-app-forms',
+  'validate-collect-forms',
+  'validate-contact-forms',
   'backup-all-forms',
   'delete-all-forms',
   'upload-app-forms',
@@ -38,6 +41,9 @@ const defaultArchiveActions = [
   'convert-app-forms',
   'convert-collect-forms',
   'convert-contact-forms',
+  'validate-app-forms',
+  'validate-collect-forms',
+  'validate-contact-forms',
   'upload-app-forms',
   'upload-collect-forms',
   'upload-contact-forms',
@@ -123,19 +129,40 @@ module.exports = async (argv, env) => {
   // Build up actions
   //
   let actions = cmdArgs._;
-  if (!actions.length) {
+  if (actions.length) {
+    const unsupported = actions.filter(a => !supportedActions.includes(a));
+    if(unsupported.length) {
+      throw new Error(`Unsupported action(s): ${unsupported.join(' ')}`);
+    }
+  } else {
     actions = !cmdArgs.archive ? defaultActions : defaultArchiveActions;
   }
 
-  const unsupported = actions.filter(a => !supportedActions.includes(a));
-  if(unsupported.length) {
-    throw new Error(`Unsupported action(s): ${unsupported.join(' ')}`);
-  }
-
   if (cmdArgs['skip-git-check']) {
     actions = actions.filter(a => a !== 'check-git');
   }
 
+  const skipValidate = cmdArgs['skip-validate'];
+  if(skipValidate) {
+    warn('Skipping all form validation.');
+    const validateActions = [
+      'validate-app-forms',
+      'validate-collect-forms',
+      'validate-contact-forms'
+    ];
+    actions = actions.filter(action => !validateActions.includes(action));
+  } else {
+    const addFormValidationIfNecessary = (formType) => {
+      const updateFormsIndex = actions.indexOf(`upload-${formType}-forms`);
+      if (updateFormsIndex >= 0 && actions.indexOf(`validate-${formType}-forms`) < 0) {
+        actions.splice(updateFormsIndex, 0, `validate-${formType}-forms`);
+      }
+    };
+    addFormValidationIfNecessary('app');
+    addFormValidationIfNecessary('collect');
+    addFormValidationIfNecessary('contact');
+  }
+
   actions = actions.map(actionName => {
     const action = require(`../fn/${actionName}`);
 
@@ -157,9 +184,9 @@ module.exports = async (argv, env) => {
   //
   const projectName = fs.path.basename(pathToProject);
 
-  let apiUrl;
-  if (actions.some(action => action.requiresInstance)) {
-    apiUrl = getApiUrl(cmdArgs, env);
+  const apiUrl = getApiUrl(cmdArgs, env);
+  const requiresInstance = actions.some(action => action.requiresInstance);
+  if (requiresInstance) {
     if (!apiUrl) {
       throw new Error('Failed to obtain a url to the API');
     }
@@ -177,10 +204,11 @@ module.exports = async (argv, env) => {
     extraArgs,
     apiUrl,
     cmdArgs.force,
-    cmdArgs['skip-translation-check']
+    cmdArgs['skip-translation-check'],
+    skipValidate
   );
 
-  if (apiUrl) {
+  if (requiresInstance && apiUrl) {
     await api().available();
   }
 

package/src/lib/upload-forms.js

@@ -13,7 +13,6 @@ const {
   readTitleFrom,
   readIdFrom
 } = require('./forms-utils');
-const validateForms = require('./validate-forms');
 
 const SUPPORTED_PROPERTIES = ['context', 'icon', 'title', 'xml2sms', 'subject_key', 'hidden_fields'];
 const FORM_EXTENSTION = '.xml';
@@ -40,7 +39,6 @@ const formMediaMatcher = (formMediaDir) => {
 };
 
 const execute = async (projectDir, subDirectory, options) => {
-  await validateForms(projectDir, subDirectory, options);
   const db = pouch();
   if (!options) options = {};
   const formsDir = getFormDir(projectDir, subDirectory);

package/src/lib/validate-forms.js

@@ -1,64 +1,82 @@
-const api = require('./api');
 const argsFormFilter = require('./args-form-filter');
+const environment = require('./environment');
 const fs = require('./sync-fs');
 const log = require('./log');
 const {
   getFormDir,
-  getFormFilePaths,
-  formHasInstanceId
+  getFormFilePaths
 } = require('./forms-utils');
 
+const VALIDATIONS_PATH = fs.path.resolve(__dirname, './validation/form');
+const validations = fs.readdir(VALIDATIONS_PATH)
+  .filter(name => name.endsWith('.js'))
+  .map(validationName => {
+    const validation = require(fs.path.join(VALIDATIONS_PATH, validationName));
+    validation.name = validationName;
+    if(!Object.hasOwnProperty.call(validation, 'requiresInstance')) {
+      validation.requiresInstance = true;
+    }
+    if(!Object.hasOwnProperty.call(validation, 'skipFurtherValidation')) {
+      validation.skipFurtherValidation = false;
+    }
+
+    return validation;
+  })
+  .sort((a, b) => {
+    if(a.skipFurtherValidation && !b.skipFurtherValidation) {
+      return -1;
+    }
+    if(!a.skipFurtherValidation && b.skipFurtherValidation) {
+      return 1;
+    }
+    return a.name.localeCompare(b.name);
+  });
+
 module.exports = async (projectDir, subDirectory, options={}) => {
 
   const formsDir = getFormDir(projectDir, subDirectory);
   if(!formsDir) {
-    log.info(`Forms dir not found: ${formsDir}`);
+    log.info(`Forms dir not found: ${projectDir}/forms/${subDirectory}`);
     return;
   }
 
-  let idValidationsPassed = true;
-  let validateFormsPassed = true;
+  const instanceProvided = environment.apiUrl;
+  let validationSkipped = false;
 
   const fileNames = argsFormFilter(formsDir, '.xml', options);
-  for (const fileName of fileNames) {
+
+  let errorFound = false;
+  for(const fileName of fileNames) {
     log.info(`Validating form: ${fileName}…`);
 
-    const { xformPath, filePath } = getFormFilePaths(formsDir, fileName);
+    const { xformPath } = getFormFilePaths(formsDir, fileName);
     const xml = fs.read(xformPath);
 
-    if(!formHasInstanceId(xml)) {
-      log.error(`Form at ${xformPath} appears to be missing <meta><instanceID/></meta> node. This form will not work on CHT webapp.`);
-      idValidationsPassed = false;
-      continue;
-    }
-    try {
-      await _formsValidate(xml);
-    } catch (err) {
-      log.error(`Error found while validating "${filePath}". Validation response: ${err.message}`);
-      validateFormsPassed = false;
+    const valParams = { xformPath, xmlStr: xml };
+    for(const validation of validations) {
+      if(validation.requiresInstance && !instanceProvided) {
+        validationSkipped = true;
+        continue;
+      }
+
+      const output = await validation.execute(valParams);
+      if(output.warnings) {
+        output.warnings.forEach(warnMsg => log.warn(warnMsg));
+      }
+      if(output.errors && output.errors.length) {
+        output.errors.forEach(errorMsg => log.error(errorMsg));
+        errorFound = true;
+        if(validation.skipFurtherValidation) {
+          break;
+        }
+      }
     }
   }
-  // Once all the fails were checked raise an exception if there were errors
-  let errors = [];
-  if (!validateFormsPassed) {
-    errors.push('One or more forms appears to have errors found by the API validation endpoint.');
-  }
-  if (!idValidationsPassed) {
-    errors.push('One or more forms appears to be missing <meta><instanceID/></meta> node.');
-  }
-  if (errors.length) {
-    // the blank spaces are a trick to align the errors in the log ;)
-    throw new Error(errors.join('\n             '));
-  }
-};
 
-let validateEndpointNotFoundLogged = false;
-const _formsValidate = async (xml) => {
-  const resp = await api().formsValidate(xml);
-  if (resp.formsValidateEndpointFound === false && !validateEndpointNotFoundLogged) {
-    log.warn('Form validation endpoint not found in your version of CHT Core, ' +
-      'no form will be checked before push');
-    validateEndpointNotFoundLogged = true; // Just log the message once
+  if(validationSkipped) {
+    log.warn('Some validations have been skipped because they require a CHT instance.');
+  }
+  if(errorFound) {
+    throw new Error('One or more forms have failed validation.');
   }
-  return resp;
 };

package/src/lib/validation/form/README.md

@@ -0,0 +1,32 @@
+# Form validations
+
+Form validations should be added to this directory.
+
+Their module should look something like this:
+
+```js
+module.exports = {
+  requiresInstance: false,
+  skipFurtherValidation: true,
+  execute: async ({ xformPath, xmlStr }) => {
+    ...
+  }
+};
+```
+
+## Module Exports
+
+| Field                   | Required                       | Notes                                                                                                                                                                                                                                                        |
+|-------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `requiresInstance`      | Optional, defaults to `true`   | The validation needs the user to have provided a instance location, e.g. via `--local` or `--instance`                                                                                                                                                       |
+| `skipFurtherValidation` | Optional, defaults to `false`  | When `true`, additional validations will not be performed on a form that fails the current validation. The goal is to avoid unnecessary noise in the validation log for forms that are grossly invalid (e.g. XML files that are not actually xForms at all). |
+| `execute`               | Required                       | The function that is run when the validation is executed. The provided input argument contains the form XML string.                                                                                                                                          |
+
+## Result
+
+The result has the following format:
+
+| Field    | Notes                                                                                              |
+|----------|----------------------------------------------------------------------------------------------------|
+| warnings | Array containing the descriptions of any warnings.                                                 |
+| errors   | Array containing the descriptions of any errors. If this is empty, the form has passed validation. |

package/src/lib/validation/form/can-generate-xform.js

@@ -0,0 +1,18 @@
+const api = require('../../api');
+
+module.exports = {
+  execute: async ({ xformPath, xmlStr }) => {
+    const warnings = [];
+    const errors = [];
+    try {
+      const resp = await api().formsValidate(xmlStr);
+      if (resp.formsValidateEndpointFound === false) {
+        warnings.push('Form validation endpoint not found in your version of CHT Core, no form will be checked before push');
+      }
+    } catch (err) {
+      errors.push(`Error found while validating "${xformPath}". Validation response: ${err.message}`);
+    }
+
+    return { errors, warnings };
+  }
+};

package/src/lib/validation/form/has-instance-id.js

@@ -0,0 +1,14 @@
+const { formHasInstanceId } = require('../../forms-utils');
+
+module.exports = {
+  requiresInstance: false,
+  skipFurtherValidation: true,
+  execute: async({ xformPath, xmlStr }) => {
+    const errors = [];
+    if(!formHasInstanceId(xmlStr)) {
+      errors.push(`Form at ${xformPath} appears to be missing <meta><instanceID/></meta> node. This form will not work on CHT webapp.`);
+    }
+
+    return { errors, warnings: [] };
+  }
+};