@suffolklitlab/alkiln 5.0.0-pre-7 → 5.0.0-pre-8

package/CHANGELOG.md

@@ -59,6 +59,8 @@ Format:
 - Change interface for testing languages. See [#713](https://github.com/SuffolkLITLab/ALKiln/issues/713).
 
 ### Fixed
+- DOCKER UPDATE (for ALKilnInThePlayground): Allows ALKilnInThePlayground to upload files. See https://github.com/SuffolkLITLab/docassemble-ALKilnInThePlayground/issues/18. This may require some developers to update their system (updating their docker image) if they are below [docassemble-os 1.0.8](https://github.com/jhpyle/docassemble-os/releases/tag/v1.0.8). You can tell if you are below docassemble-os 1.0.8 if you run `docker image inspect jhpyle/docassemble | grep Created` on your host server (Lightsail, etc.) and the created date is before 2023-04-09. You can update as described in [the documentation](https://suffolklitlab.org/legal-tech-class/docs/practical-guide-docassemble/maintaining-docassemble#updates-to-the-docassemble-container). Updating to a new docker images is something that all docassemble users will have to do sometime anyway, sometimes for security.
+- Conform upload file behavior to that of other Story Table rows - avoid erroring when field isn't set properly unless using all variables is required.
 - Projects created in da each have a unique name. https://github.com/SuffolkLITLab/ALKiln/issues/663
 - Shorten path names to try to accommodate limitations of windows systems while still keeping enough useful information to help devs identify the test outputs. https://github.com/SuffolkLITLab/ALKiln/issues/618
 - Updated field decoding to handle new object field encoding. See [#711](https://github.com/SuffolkLITLab/ALKiln/issues/711)

package/lib/scope.js

@@ -2,6 +2,7 @@ const { expect } = require('chai');
 const cheerio = require('cheerio');
 const path = require('path');
 const fs = require('fs');
+const fg = require('fast-glob');
 const { AxePuppeteer } = require('@axe-core/puppeteer');
 const safe_filename = require("sanitize-filename");
 // "en" is faster to import: https://github.com/faker-js/faker/issues/1114#issuecomment-1169532948
@@ -1528,7 +1529,10 @@ module.exports = {
     let row = matching_input_rows[ matching_input_rows.length - 1 ];
     let answer = row.value;
 
-    await scope.funnel_the_answer( scope, { handle, answer, field });
+    // Will implement `was_used` in cases as they come along. For now, this is
+    // necessary for file uploads to be as flexible as any other Story Table row.
+    let was_used = await scope.funnel_the_answer( scope, { handle, answer, field });
+    if ( was_used === false ) { row = null; }
 
     return row;
   },  // Ends scope.setVariable()
@@ -1590,16 +1594,18 @@ module.exports = {
       custom_datatype = await scope.get_custom_datatype( scope, { handle });  
     }
 
+    let row_was_used = true;
     // If there is a custom datatype function available, use it to set the field
     if ( scope.setCustomDatatype[ custom_datatype ] !== undefined ) {
       // Set da package custom fields. TODO: find `custom_handle` in the setter?
       let custom_handle = await handle.evaluateHandle(( elem )=> { return elem.closest( `.da-form-group` ) });
-      await scope.setCustomDatatype[ custom_datatype ]( scope, { handle: custom_handle, answer });
+      row_was_used = await scope.setCustomDatatype[ custom_datatype ]( scope, { handle: custom_handle, answer });
     } else {
       // If no custom datatype, set a regular field
-      await scope.enter_answer[ field.tag ]( scope, { handle, answer });
+      row_was_used = await scope.enter_answer[ field.tag ]( scope, { handle, answer });
     }
 
+    return row_was_used;
   },  // Ends scope.funnel_the_answer()
 
   // Handle setting values for da custom datatypes. E.g. `da-field-container-datatype-BirthDate`
@@ -1660,10 +1666,14 @@ module.exports = {
   },  // ends setCustomDatatype {}
 
   enter_answer: {
-    button: async function ( scope, { handle, answer }) { await scope.tapElement( scope, handle ); },
+    button: async function ( scope, { handle, answer }) {
+      await scope.tapElement( scope, handle );
+      return true;
+    },
     textarea: async function ( scope, { handle, answer }) {
       await scope.setText( scope, { handle, answer });
       await scope.afterStep(scope);  // No showifs for this one?
+      return true;
     },
     select: async function ( scope, { handle, answer }) {
       // Note: I don't remember what's really going on in here.
@@ -1692,13 +1702,16 @@ module.exports = {
       }
       
       await scope.afterStep(scope, { waitForShowIf: true });
+      return true;
     },
     canvas: async function ( scope, { handle, answer }) {
       await scope.draw_signature( scope );
       await scope.afterStep( scope );
+      return true;
     },
     input: async function ( scope, { handle, answer }) {
       /* Set value of some `input` element to the given value. */
+      let row_was_used = true;
 
       let type = await handle.evaluate( elem => { return elem.getAttribute('type'); });
       if ( type === `radio` || type === `checkbox` ) {
@@ -1717,7 +1730,7 @@ module.exports = {
         await scope.afterStep(scope, { waitForShowIf: true });
         
       } else if ( type === `file` ) {
-        await scope.uploadFiles( scope, { handle, answer });
+        row_was_used = await scope.uploadFiles( scope, { handle, answer });
         await scope.afterStep(scope, { waitForShowIf: true });
 
       } else if (type == `date`) {
@@ -1746,6 +1759,8 @@ module.exports = {
         await scope.setText( scope, { handle, answer });
         await scope.afterStep(scope, { waitForShowIf: true });
       }
+
+      return row_was_used;
     },  // Ends scope.enter_answer.input()
     hidden: async function ( scope, { handle, answer }) {
       // Do nothing. The only way the code should get here is if
@@ -1762,89 +1777,98 @@ module.exports = {
   uploadFiles: async function ( scope, { handle, answer }) {
     /* Try to upload one or more files to a file-type input field.
     *    Does not submit, so no danger of timeout with server reload.
+    * 
+    * Returns whether the row was used or not.
     */
 
-    // Tests should be one place: docassemble/*/data/sources, where * is the only folder in `docassemble`.
-    // Makes an absolute path from the folder in which `npm run` was used plus our internal folders
-    let docassemble_pkg_path = `${ process.cwd() }/docassemble`;
-    let subpackage_name = session_vars.get_package_name().substring(12);
-    let pkg_paths = [];
-    try {
-      fs.accessSync(docassemble_pkg_path);
-      // https://stackoverflow.com/a/24594123/11416267
-      let dirs = fs.readdirSync(docassemble_pkg_path, {withFileTypes: true}).filter(dirent => dirent.isDirectory()).map(dirent => dirent.name);
-      if (dirs.length < 1) {
-        let msg = `You can store files to upload in 'docassemble/${ subpackage_name }/data/source'. These tests could not find a folder like that starting at ${ process.cwd() }.`;
-        await scope.addToReport( scope, { type: `warning`, value: msg});
-        throw Error(msg);  // Most people will want to know the sources folder path;
-      }
-      for (let dir of dirs) {
-        pkg_paths.push(`${docassemble_pkg_path}/${dir}`);
-      }
-    } catch ( error_da ) {
-      let msg = `You can store files to upload in 'docassemble/${ subpackage_name }/data/source'. ` +
-          `These tests could not find a folder like that. The current directory is ` +
-          ` ${ process.cwd() }; is that in the top level folder of your docassemble package? (${ session_vars.get_package_name() })?`;
-      await scope.addToReport( scope, { type: `warning`, value: msg});
-      throw error_da;  // Most people will want to know the sources folder path;
+    // First make sure the list of files was defined. Don't remove spaces yet -
+    // allow a typo where someone forgot to include a space after a comma.
+    let uploads_names = answer.split(`,`);
+    if ( uploads_names.length === 0 ) {
+      await scope.addToReport( scope, {
+        type: `error`,
+        value: `Your list of files to upload is empty.`
+      });
+      return false;  // The row wasn't used
     }
 
-    let chosen_dir = null;
-    let err_paths = [];
-    for (let pkg_path of pkg_paths) {
-      let dir = `${ pkg_path }/data/sources`;
-      try {
-        fs.accessSync(dir);
-        chosen_dir = dir;
-        break;
-      } catch ( error_full ) {
-        err_paths.push(`'${dir}'`);
-      }
-    }
-    if (chosen_dir === null) {
-      let msg = `You can store files to upload in ${err_paths.join(", or ")}. These tests could not find any of those folders.`;
-      await scope.addToReport( scope, { type: `warning`, value: msg });
-      throw Error(msg);  // Most people will want to know the sources folder path;
+    let dirs = [];
+    if ( session_vars.get_origin() === `playground` ) {
+      let files_playground = fg.sync(
+        [ `/usr/share/docassemble/files/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
+        { onlyFiles: false, suppressErrors: true }
+      );
+      let files_s3 = fg.sync(
+        [ `/tmp/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
+        { onlyFiles: false, suppressErrors: true }
+      );
+
+      dirs = files_playground.concat( files_s3 );
+    } else {
+      dirs = fg.sync(
+        [ `**/docassemble/**/data/sources` ],
+        { onlyFiles: false, suppressErrors: true }
+      );
     }
 
-    if (pkg_paths.length > 1 && pkg_paths.length - err_paths.length > 1) {
-      await scope.addToReport( scope, { type: `warning`, value: `You have multiple folders in the 'docassemble' folder (${ pkg_paths }). These tests will use the first folder with a "data/sources" subfolder (${chosen_dir}).` });
+    if ( session_vars.get_debug() ) {
+      console.log( `All discovered dirs to upload from:`, dirs );
     }
 
-    // Allow a typo where someone forgot to include a space after a comma
-    let file_names = answer.split(`,`);
+    // Make sure sources directory exists and has files in it
+    if ( dirs.length === 0 ) {
+      await scope.addToReport( scope, {
+        type: `warning`,
+        value: `The test files need to be in the "sources" directory, `
+          + `but ALKiln couldn't find the directory or it was empty.`
+      });
+      return false;  // The row wasn't used
+    }
 
-    // If a file exist, add it to a list of files to be uploaded. If it doesn't, add a warning.
+    // Get the full paths of all the files
+    let dir = null;
     let paths = [];
-    for ( let name of file_names ) {
+    let missing_files = [];  // For any user warnings
+    for ( let name of uploads_names ) {
+     
+      let trimmed_name = name.trim();
+      let matching_paths = [];
+
+      for ( let one_dir of dirs ) {
+        // accumulate all matching paths
+        let new_paths = fg.sync([ `${ one_dir }/${ trimmed_name }` ]);
+        matching_paths.push( ...new_paths );
+      }  // ends for each dir
+
+      if ( matching_paths.length === 0 ) {
+        missing_files.push( trimmed_name );
+      } else {
+        paths.push( ...matching_paths );
+      }
 
-      // If the developer correctly used a ", ", there will be a space at the
-      // start of the string. Remove it. If they forgot the space, that's fine too.
-      name = name.replace(/^\s/, '');
-      let file_path = path.join( chosen_dir, name );
+    }  // ends for each file
 
-      try {
-        fs.accessSync( file_path );
-        paths.push( file_path );
-      } catch ( err ) {
-        // TODO: Should this throw an error?
-        await scope.addToReport( scope, { type: `warning`, value: `Could not find "${ name }" in "${ chosen_dir }"` });
-      }
+    if ( missing_files.length > 0 ) {
+      await scope.addToReport( scope, {
+        type: `warning`,
+        value: `ALKiln could not find "${ missing_files.join('", "') }" in any of these folders: "${ dirs.join('", "') }".`
+      });
+    }
+
+    // If all files missing, return that the row wasn't used
+    if ( missing_files.length === uploads_names.length ) {
+      return false;  // The row wasn't used
     }
 
-    if ( paths.length === 0 ) {
-      await scope.addToReport( scope, { type: `error`, value: `Could not find "${answer}".` });
-      throw ReferenceError(`Could not find "${answer}".`)
-    } 
     // Prepare to wait for upload to complete
     let name = await handle.evaluate( elem => { return elem.getAttribute(`name`); });
     let selector = `*[name="${ name }"]`
     let disabled_selector = `${ selector }[disabled]`;
     let enabled_selector = `${ selector }:not([disabled])`;
 
-    // This comes before the upload starts
-    // Notice `await` is absent. Don't actually wait here.
-    // Just start looking for it now. That way we'll catch it before it's gone
+    // This comes before the upload starts.
+    // Notice `await` is absent. Don't actually wait here yet.
+    // Start looking for it before it exists or we might miss it.
     let wait_for_disable = scope.page.waitForSelector( disabled_selector );
     
     // Upload
@@ -1855,8 +1879,9 @@ module.exports = {
     await wait_for_disable;
     // When it's again enabled, the files have finished loading. Or so I have observed.
     await scope.page.waitForSelector( enabled_selector );
-  
 
+    // Return that the row was used
+    return true;
   },  // Ends scope.uploadFiles()
 
   setCheckbox: async function ( scope, { label, answer }) {

package/lib/utils/langs.js

@@ -2,14 +2,14 @@
 
 let _path = require('path');
 let fs = require('fs');
-let glob = require('glob');
+const fg = require('fast-glob');
 
 let session_vars = require('./session_vars');
 
 // Get the glob filepaths to find all the test files
 let dir_root = process.env.INIT_CWD;  // Path to root of project
 let test_glob = process.argv[2];  // String of glob (regex for filepaths)
-let filepaths = glob.sync( _path.resolve( dir_root, test_glob ) );
+let filepaths = fg.sync( _path.resolve( dir_root, test_glob ) );
 let flag = '_lang_';
 
 let langs = session_vars.get_languages();