cht-conf 3.10.2 → 3.12.0

package/src/fn/upload-resources.js

@@ -6,13 +6,17 @@ const processJson = (json) => {
     resources: json
   };
 };
+const { RESOURCE_CONFIG_PATH, RESOURCES_DIR_PATH } = require('../lib/project-paths');
 
 module.exports = {
   requiresInstance: true,
+  RESOURCE_CONFIG_PATH,
+  RESOURCES_DIR_PATH,
   execute: () => {
-    const configurationPath = `${environment.pathToProject}/resources.json`;
-    const directoryPath = `${environment.pathToProject}/resources`;
+    const configurationPath = `${environment.pathToProject}/${RESOURCE_CONFIG_PATH}`;
+    const directoryPath = `${environment.pathToProject}/${RESOURCES_DIR_PATH}`;
 
     return uploadConfigurationDocs(configurationPath, directoryPath, 'resources', processJson);
   }
 };
+

package/src/fn/validate-app-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateAppForms = (forms) => {
+  return validateForms(environment.pathToProject, 'app', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateAppForms,
+  execute: () => validateAppForms(environment.extraArgs)
+};

package/src/fn/validate-collect-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateCollectForms = (forms) => {
+  return validateForms(environment.pathToProject, 'collect', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateCollectForms,
+  execute: () => validateCollectForms(environment.extraArgs)
+};

package/src/fn/validate-contact-forms.js

@@ -0,0 +1,12 @@
+const validateForms = require('../lib/validate-forms');
+const environment = require('../lib/environment');
+
+const validateContactForms = (forms) => {
+  return validateForms(environment.pathToProject, 'contact', { forms });
+};
+
+module.exports = {
+  requiresInstance: false,
+  validateContactForms,
+  execute: () => validateContactForms(environment.extraArgs)
+};

package/src/fn/watch-project.js

@@ -0,0 +1,287 @@
+const environment = require('../lib/environment');
+const process = require('process');
+const path = require('path');
+const fs = require('fs');
+const { error, warn, info } = require('../lib/log');
+const Queue = require('queue-promise');
+const watcher = require('@parcel/watcher');
+const { validateAppForms } = require('./validate-app-forms');
+const { validateContactForms } = require('./validate-contact-forms');
+const { validateCollectForms } = require('./validate-collect-forms');
+const { uploadAppForms } = require('./upload-app-forms');
+const { uploadContactForms } = require('./upload-contact-forms');
+const { uploadCollectForms } = require('./upload-collect-forms');
+const convertForms = require('../lib/convert-forms');
+const uploadForms = require('../lib/upload-forms');
+const { deleteForms } = require('../fn/delete-forms');
+const { convertAppForms, APP_FORMS_PATH } = require('./convert-app-forms');
+const { convertContactForm, CONTACT_FORMS_PATH } = require('./convert-contact-forms');
+const { convertCollectForms, COLLECT_FORMS_PATH } = require('./convert-collect-forms');
+const { uploadAppSettings, APP_SETTINGS_DIR_PATH, APP_SETTINGS_JSON_PATH } = require('./upload-app-settings');
+const { execute: compileAppSettings, configFileMatcher } = require('./compile-app-settings');
+const { execute: uploadCustomTranslations, TRANSLATIONS_DIR_PATH } = require('./upload-custom-translations');
+const { execute: uploadResources, RESOURCES_DIR_PATH, RESOURCE_CONFIG_PATH } = require('./upload-resources');
+
+const watcherEvents = {
+    CreateEvent: 'create',
+    DeleteEvent: 'delete',
+    UpdateEvent: 'update'
+};
+
+const runValidation = (validation, forms) => {
+    if(environment.skipValidate) {
+        return;
+    }
+    return validation(forms);
+};
+
+const uploadInitialState = async (api) => {
+    await uploadResources();
+    await runValidation(validateAppForms, environment.extraArgs);
+    await runValidation(validateContactForms, environment.extraArgs);
+    await uploadAppForms(environment.extraArgs);
+    await uploadContactForms(environment.extraArgs);
+    await uploadCustomTranslations();
+    await uploadAppSettings(api);
+};
+
+let fsEventsSubscription;
+let eventQueue;
+
+const cleanUp = async () => {
+    if (fsEventsSubscription) {
+        await fsEventsSubscription.unsubscribe();
+        fsEventsSubscription = undefined;
+    }
+    if (eventQueue) {
+        await eventQueue.clear();
+        eventQueue = undefined;
+    }
+};
+
+const waitForKillSignal = () => {
+    return new Promise((resolve) => {
+        process.on('SIGINT', async () => {
+            await cleanUp();
+            resolve();
+        });
+    });
+};
+
+const deleteForm = (fileName, formDir) => {
+    const form = uploadForms.formFileMatcher(fileName) || convertForms.formFileMatcher(fileName);
+    if (!form) {
+        return false;
+    }
+    const absFormDirPath = path.join(environment.pathToProject, formDir);
+    const formFiles = fs.readdirSync(absFormDirPath).map(file => (uploadForms.formFileMatcher(file) || convertForms.formFileMatcher(file))).filter(_form => _form === form);
+    const canDelete = formFiles.length <= 0;
+    if (canDelete) {
+        eventQueue.enqueue(async () => {
+            await deleteForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+    warn(`You have to delete all form files (.xlsx, .properties.json, .xml) associated with the ${form} form to delete it from ${environment.apiUrl}`);
+    return false;
+};
+
+const processAppForm = (eventType, fileName) => {
+    if (eventType === watcherEvents.DeleteEvent && deleteForm(fileName, APP_FORMS_PATH)) {
+        return true;
+    }
+    let form = uploadForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await runValidation(validateAppForms, [form]);
+            await uploadAppForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+
+    form = convertForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await convertAppForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+    return false;
+};
+
+const processAppFormMedia = (formMediaDir, fileName) => {
+    const form = uploadForms.formMediaMatcher(formMediaDir);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await runValidation(validateAppForms,[form]);
+            await uploadAppForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+    return false;
+};
+
+const processContactForm = (eventType, fileName) => {
+    if (eventType === watcherEvents.DeleteEvent && deleteForm(fileName, CONTACT_FORMS_PATH)) {
+        return true;
+    }
+    let form = convertForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await convertContactForm([form]);
+            return fileName;
+        });
+        return true;
+    }
+
+    form = uploadForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await runValidation(validateContactForms,[form]);
+            await uploadContactForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+    return false;
+};
+
+const processCollectForm = (eventType, fileName) => {
+    if (eventType === watcherEvents.DeleteEvent && deleteForm(fileName, COLLECT_FORMS_PATH)) {
+        return true;
+    }
+    let form = uploadForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await runValidation(validateCollectForms,[form]);
+            await uploadCollectForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+
+    form = convertForms.formFileMatcher(fileName);
+    if (form) {
+        eventQueue.enqueue(async () => {
+            await convertCollectForms([form]);
+            return fileName;
+        });
+        return true;
+    }
+    return false;
+};
+
+const processConfigFiles = (api, fileName) => {
+    if (fileName === APP_SETTINGS_JSON_PATH) {
+        eventQueue.enqueue(async () => {
+            await uploadAppSettings(api);
+            return fileName;
+        });
+        return true;
+    }
+    if (configFileMatcher(fileName)) {
+        eventQueue.enqueue(async () => {
+            await compileAppSettings();
+            return fileName;
+        });
+        return true;
+    }
+    return false;
+};
+
+const watchProject = {
+    watch: async (api, blockFn, callback = {}, uploadStateOnStart = false) => {
+        if (uploadStateOnStart) {
+            await uploadInitialState(api);
+            info('Initial State Uploaded');
+        }
+
+        eventQueue = new Queue({ concurrent: 1, start: true });
+
+        fsEventsSubscription = await watcher.subscribe(environment.pathToProject, async (err, events) => {
+            if (err) {
+                error(err);
+                return;
+            }
+            for (const event of events) {
+                const changePath = event.path;
+                const parsedPath = path.parse(changePath);
+                const fileName = parsedPath.base;
+
+                if (parsedPath.dir === path.join(environment.pathToProject, APP_FORMS_PATH)) {
+                    processAppForm(event.type, fileName);
+                    continue;
+                }
+
+                if (parsedPath.dir.startsWith(path.join(environment.pathToProject, APP_FORMS_PATH))
+                    && path.parse(parsedPath.dir).dir.endsWith(APP_FORMS_PATH)) { // check if the directory's immediate parent is forms/app
+                    const dirName = path.parse(parsedPath.dir).base;
+                    processAppFormMedia(dirName, fileName);
+                    continue;
+                }
+
+                if (parsedPath.dir === path.join(environment.pathToProject, CONTACT_FORMS_PATH)) {
+                    processContactForm(event.type, fileName);
+                    continue;
+                }
+
+                if (parsedPath.dir === path.join(environment.pathToProject, COLLECT_FORMS_PATH)) {
+                    processCollectForm(event.type, fileName);
+                    continue;
+                }
+
+
+                if (parsedPath.dir === path.join(environment.pathToProject, TRANSLATIONS_DIR_PATH)) {
+                    eventQueue.enqueue(async () => {
+                        await uploadCustomTranslations();
+                        return fileName;
+                    });
+                    continue;
+                }
+
+                if (parsedPath.dir === path.join(environment.pathToProject, APP_SETTINGS_DIR_PATH)) {
+                    processConfigFiles(api, fileName);
+                    continue;
+                }
+
+                if (parsedPath.dir === environment.pathToProject && parsedPath.base !== RESOURCE_CONFIG_PATH) {
+                    processConfigFiles(api, fileName);
+                    continue;
+                }
+
+                if (parsedPath.base === RESOURCE_CONFIG_PATH || parsedPath.dir === path.join(environment.pathToProject, RESOURCES_DIR_PATH)) {
+                    eventQueue.enqueue(async () => {
+                        await uploadResources();
+                        return fileName;
+                    });
+                    continue;
+                }
+            }
+        });
+
+        eventQueue.on('resolve', file => {
+            if (callback) {
+                callback(file);
+            }
+        });
+        eventQueue.on('reject', err => error('watch-mode: Error while processing change,', err));
+
+        info('watching', environment.pathToProject, 'for changes');
+        await blockFn();
+    },
+    close: async () => {
+        info('stopping watchers');
+        await cleanUp();
+    }
+};
+
+const api = require('../lib/api');
+module.exports = {
+    watchProject,
+    requiresInstance: true,
+    execute: () => watchProject.watch(api(), waitForKillSignal, false, true)
+};

package/src/lib/convert-forms.js

@@ -20,25 +20,34 @@ E
 E	pip uninstall pyxform-medic
 E` + INSTALLATION_INSTRUCTIONS;
 
-module.exports = async (projectDir, subDirectory, options) => {
-  if(!options) options = {};
+const FORM_EXTENSION = '.xlsx';
+const formFileMatcher = (fileName) => {
+  if (fileName.endsWith(FORM_EXTENSION) &&
+    !fileName.startsWith('~$') // ignore Excel "owner files"
+    && fileName !== 'PLACE_TYPE-create.xlsx' && fileName !== 'PLACE_TYPE-edit.xlsx') {
+    return fileName.slice(0, fileName.length - FORM_EXTENSION.length);
+  }
+  return null;
+};
+
+const execute = async (projectDir, subDirectory, options) => {
+  if (!options) options = {};
 
   const formsDir = getFormDir(projectDir, subDirectory);
 
-  if(!fs.exists(formsDir)) {
+  if (!fs.exists(formsDir)) {
     warn(`Forms dir not found: ${formsDir}`);
     return Promise.resolve();
   }
 
-  const filesToConvert = argsFormFilter(formsDir, '.xlsx', options)
-    .filter(name => !name.startsWith('~$')) // ignore Excel "owner files"
-    .filter(name => name !== 'PLACE_TYPE-create.xlsx' && name !== 'PLACE_TYPE-edit.xlsx');
+  const filesToConvert = argsFormFilter(formsDir, FORM_EXTENSION, options)
+    .filter(name => formFileMatcher(name));
 
   for (let xls of filesToConvert) {
     const originalSourcePath = `${formsDir}/${xls}`;
     let sourcePath;
 
-    if(options.force_data_node) {
+    if (options.force_data_node) {
       const temporaryPath = `${fs.mkdtemp()}/${options.force_data_node}.xlsx`;
       fs.copy(originalSourcePath, temporaryPath);
       sourcePath = temporaryPath;
@@ -54,15 +63,21 @@ module.exports = async (projectDir, subDirectory, options) => {
   }
 };
 
+module.exports = {
+  SUPPORTED_EXTENSIONS: [FORM_EXTENSION],
+  formFileMatcher,
+  execute
+};
+
 const xls2xform = (sourcePath, targetPath) =>
-    exec([XLS2XFORM, '--skip_validate', sourcePath, targetPath])
-      .catch(e => {
-        if(executableAvailable()) {
-          if(e.includes('unrecognized arguments: --skip_validate')) {
-            throw new Error('Your xls2xform installation appears to be out of date.' + UPDATE_INSTRUCTIONS);
-          } else throw e;
-        } else throw new Error('There was a problem executing xls2xform.  It may not be installed.' + INSTALLATION_INSTRUCTIONS);
-      });
+  exec([XLS2XFORM, '--skip_validate', sourcePath, targetPath])
+    .catch(e => {
+      if (executableAvailable()) {
+        if (e.includes('unrecognized arguments: --skip_validate')) {
+          throw new Error('Your xls2xform installation appears to be out of date.' + UPDATE_INSTRUCTIONS);
+        } else throw e;
+      } else throw new Error('There was a problem executing xls2xform.  It may not be installed.' + INSTALLATION_INSTRUCTIONS);
+    });
 
 // here we fix the form content in arcane ways.  Seeing as we have out own fork
 // of pyxform, we should probably be doing this fixing there.
@@ -72,33 +87,33 @@ const fixXml = (path, hiddenFields, transformer, enketo) => {
   // future changes to the output of xls2xform.
   let xml = fs.read(path)
 
-      // The following copies behaviour from old bash scripts, and will create a
-      // second <meta> element if one already existed.  We may want to actually
-      // merge the two instead.
-      .replace(/<inputs>/, META_XML_SECTION)
+    // The following copies behaviour from old bash scripts, and will create a
+    // second <meta> element if one already existed.  We may want to actually
+    // merge the two instead.
+    .replace(/<inputs>/, META_XML_SECTION)
 
-      // XLSForm does not allow converting a field without a label, so we use
-      // the placeholder NO_LABEL.
-      .replace(/NO_LABEL/g, '')
+    // XLSForm does not allow converting a field without a label, so we use
+    // the placeholder NO_LABEL.
+    .replace(/NO_LABEL/g, '')
 
-      // No comment.
-      .replace(/.*DELETE_THIS_LINE.*(\r|\n)/g, '')
-      ;
+    // No comment.
+    .replace(/.*DELETE_THIS_LINE.*(\r|\n)/g, '')
+    ;
 
   // Enketo _may_ not work with forms which define a default language - see
   // https://github.com/medic/cht-core/issues/3174
-  if(enketo) xml = xml.replace(/ default="true\(\)"/g, '');
+  if (enketo) xml = xml.replace(/ default="true\(\)"/g, '');
 
-  if(hiddenFields) {
+  if (hiddenFields) {
     const r = new RegExp(`<(${hiddenFields.join('|')})(/?)>`, 'g');
     xml = xml.replace(r, '<$1 tag="hidden"$2>');
   }
 
-  if(transformer) xml = transformer(xml, path);
+  if (transformer) xml = transformer(xml, path);
 
 
   // Check for deprecations
-  if(xml.includes('repeat-relevant')) {
+  if (xml.includes('repeat-relevant')) {
     warn('From webapp version 2.14.0, repeat-relevant is no longer required.  See https://github.com/medic/cht-core/issues/3449 for more info.');
   }
 
@@ -106,7 +121,7 @@ const fixXml = (path, hiddenFields, transformer, enketo) => {
 };
 
 function getHiddenFields(propsJson) {
-  if(!fs.exists(propsJson)) return [];
+  if (!fs.exists(propsJson)) return [];
   else return fs.readJson(propsJson).hidden_fields;
 }
 
@@ -116,7 +131,7 @@ function executableAvailable() {
       stdio: ['ignore', 'ignore', 'ignore'],
     });
     return true;
-  } catch(e) {
+  } catch (e) {
     return false;
   }
 }

package/src/lib/environment.js

@@ -12,7 +12,8 @@ const initialize = (
   extraArgs,
   apiUrl,
   force,
-  skipTranslationCheck
+  skipTranslationCheck,
+  skipValidate
 ) => {
   if (state.initialized) {
     throw Error('environment is already initialized');
@@ -26,7 +27,8 @@ const initialize = (
     isArchiveMode,
     pathToProject,
     force,
-    skipTranslationCheck
+    skipTranslationCheck,
+    skipValidate
   });
 };
 
@@ -51,6 +53,7 @@ module.exports = {
   get apiUrl() { return getState('apiUrl'); },
   get force() { return getState('force'); },
   get skipTranslationCheck() { return getState('skipTranslationCheck'); },
+  get skipValidate() { return getState('skipValidate'); },
 
   /**
    * Return `true` if the environment **seems** to be production.

package/src/lib/git-exec.js

@@ -64,7 +64,7 @@ module.exports.getDefaultRemote = async () => {
 
 /**
  * Returns the upstream of the current branch,
- * in the form of repo/branch, eg. 'origin/master'.
+ * in the form of repo/branch, eg. 'origin/main'.
  *
  * Returns `null` if there are no upstreams repos configured.
  */

package/src/lib/main.js

@@ -21,6 +21,9 @@ const defaultActions = [
   'convert-app-forms',
   'convert-collect-forms',
   'convert-contact-forms',
+  'validate-app-forms',
+  'validate-collect-forms',
+  'validate-contact-forms',
   'backup-all-forms',
   'delete-all-forms',
   'upload-app-forms',
@@ -38,6 +41,9 @@ const defaultArchiveActions = [
   'convert-app-forms',
   'convert-collect-forms',
   'convert-contact-forms',
+  'validate-app-forms',
+  'validate-collect-forms',
+  'validate-contact-forms',
   'upload-app-forms',
   'upload-collect-forms',
   'upload-contact-forms',
@@ -123,19 +129,40 @@ module.exports = async (argv, env) => {
   // Build up actions
   //
   let actions = cmdArgs._;
-  if (!actions.length) {
+  if (actions.length) {
+    const unsupported = actions.filter(a => !supportedActions.includes(a));
+    if(unsupported.length) {
+      throw new Error(`Unsupported action(s): ${unsupported.join(' ')}`);
+    }
+  } else {
     actions = !cmdArgs.archive ? defaultActions : defaultArchiveActions;
   }
 
-  const unsupported = actions.filter(a => !supportedActions.includes(a));
-  if(unsupported.length) {
-    throw new Error(`Unsupported action(s): ${unsupported.join(' ')}`);
-  }
-
   if (cmdArgs['skip-git-check']) {
     actions = actions.filter(a => a !== 'check-git');
   }
 
+  const skipValidate = cmdArgs['skip-validate'];
+  if(skipValidate) {
+    warn('Skipping all form validation.');
+    const validateActions = [
+      'validate-app-forms',
+      'validate-collect-forms',
+      'validate-contact-forms'
+    ];
+    actions = actions.filter(action => !validateActions.includes(action));
+  } else {
+    const addFormValidationIfNecessary = (formType) => {
+      const updateFormsIndex = actions.indexOf(`upload-${formType}-forms`);
+      if (updateFormsIndex >= 0 && actions.indexOf(`validate-${formType}-forms`) < 0) {
+        actions.splice(updateFormsIndex, 0, `validate-${formType}-forms`);
+      }
+    };
+    addFormValidationIfNecessary('app');
+    addFormValidationIfNecessary('collect');
+    addFormValidationIfNecessary('contact');
+  }
+
   actions = actions.map(actionName => {
     const action = require(`../fn/${actionName}`);
 
@@ -157,9 +184,9 @@ module.exports = async (argv, env) => {
   //
   const projectName = fs.path.basename(pathToProject);
 
-  let apiUrl;
-  if (actions.some(action => action.requiresInstance)) {
-    apiUrl = getApiUrl(cmdArgs, env);
+  const apiUrl = getApiUrl(cmdArgs, env);
+  const requiresInstance = actions.some(action => action.requiresInstance);
+  if (requiresInstance) {
     if (!apiUrl) {
       throw new Error('Failed to obtain a url to the API');
     }
@@ -177,10 +204,11 @@ module.exports = async (argv, env) => {
     extraArgs,
     apiUrl,
     cmdArgs.force,
-    cmdArgs['skip-translation-check']
+    cmdArgs['skip-translation-check'],
+    skipValidate
   );
 
-  if (apiUrl) {
+  if (requiresInstance && apiUrl) {
     await api().available();
   }
 

package/src/lib/project-paths.js

@@ -0,0 +1,12 @@
+const path = require('path');
+// Paths Relative to the Project
+module.exports = {
+    APP_SETTINGS_DIR_PATH: 'app_settings',
+    APP_SETTINGS_JSON_PATH: 'app_settings.json',
+    APP_FORMS_PATH: path.join('forms', 'app'),
+    CONTACT_FORMS_PATH: path.join('forms', 'contact'),
+    COLLECT_FORMS_PATH: path.join('forms', 'collect'),
+    TRANSLATIONS_DIR_PATH: 'translations',
+    RESOURCE_CONFIG_PATH: 'resources.json',
+    RESOURCES_DIR_PATH: 'resources'
+};