@suffolklitlab/alkiln 4.5.2 → 4.6.2

package/.github/workflows/test_tests.yml

@@ -39,6 +39,11 @@ jobs:
 
     name: Run Puppeteer tests
     steps:
+      - name: Set name of artifact folder with date and UTC time
+        # https://www.shell-tips.com/linux/how-to-format-date-and-time-in-linux-macos-and-bash/#how-to-format-a-date-in-bash
+        run: |
+          echo "ARTIFACT_NAME=alkiln_tests_output-$(date +'%Y-%m-%d at %Hhrs %Mmins %Ssecs' -u) UTC" >> $GITHUB_ENV
+          echo "UNIT_TESTS_ARTIFACT_NAME=_alkiln_tests_misc-$(date +'%Y-%m-%d at %Hhrs %Mmins %Ssecs' -u) UTC" >> $GITHUB_ENV
       - name: Set languages
         run: echo "EXTRA_LANGUAGES=${{ github.event.inputs.extra_languages }}" >> $GITHUB_ENV
         if: ${{ github.event.inputs.extra_languages != '' }}
@@ -53,39 +58,21 @@ jobs:
       - run: npm install
       - run: npm run setup
       - run: npm run test -- ${{ github.event.inputs.tags && format('"--tags" "{0}"', github.event.inputs.tags) }}
-      - name: upload error artifacts
+      # Artifacts
+      - name: upload unit tests artifacts folder
         uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: errors
-          path: ./**/error*.jpg
-      - name: upload screenshot steps
-        uses: actions/upload-artifact@v2
-        # They will upload even if a previosu step has failed
-        if: ${{ always() }}
-        with:
-          name: screenshots
-          path: ./**/screenshot*.jpg
-      - name: upload downloads
-        uses: actions/upload-artifact@v2
-        # They will upload even if a previous step has failed
-        if: ${{ always() }}
-        with:
-          name: downloads
-          path: ./**/downloads_*
-      - name: upload report
-        uses: actions/upload-artifact@v2
-        # They will upload even if a previous step has failed
+        # This will upload even if a previous step has failed
         if: ${{ always() }}
         with:
-          name: report
-          path: ./**/*_report_*
-      - name: upload debug_log artifact
+          name: ${{ env.UNIT_TESTS_ARTIFACT_NAME }}
+          path: ./_alkiln_tests_misc-*
+      - name: upload integration tests artifacts folder
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: debug_log file
-          path: ./debug_log.txt
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ./alkiln_tests_output-*
+      # Delete Playground Project and project name file
       - name: Cleanup
         if: ${{ always() }}
         run: npm run takedown

package/CHANGELOG.md

@@ -40,6 +40,20 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [4.6.2] - 2022-06-28
+### Changed
+- Put cucumber tests' artifacts in one folder. See https://github.com/SuffolkLITLab/ALKiln/issues/552.
+- Add separate artifacts dir for our own internal tests, like unit tests
+- Abstract names of some of those artifact directories
+
+## [4.6.1] - 2022-06-28
+### Fixed
+- Single dropdown fields on pages are now recognized as fields properly
+
+## [4.6.0] - 2022-06-10
+### Added
+- Steps to test web accessibility either on individual page or for all subsequent pages.
+
 ## [4.5.2] - 2022-06-10
 ### Fixed
 - (internal) Expected in report is reset properly

package/action.yml

@@ -36,12 +36,14 @@ inputs:
 runs:
   using: "composite"
   steps:
-    
     # Set environment variables
     - name: "ALKiln: Set environment variables"
       # If extra languages were set manually, override the repository
       # EXTRA_LANGUAGES secret with the manually set values
+      # Human-readable date/time:
+      # https://www.shell-tips.com/linux/how-to-format-date-and-time-in-linux-macos-and-bash/#how-to-format-a-date-in-bash
       run: |
+        echo "ARTIFACT_NAME=alkiln_tests_output $(date +'%Y-%m-%d at %Hhrs %Mmins %Ssecs' -u) UTC" >> $GITHUB_ENV
         if ${{ github.event.inputs.extra_languages != '' }}
         then
           echo "Manually set languages: ${{ github.event.inputs.extra_languages }}"
@@ -121,32 +123,12 @@ runs:
       shell: bash
     
     # Upload artifacts that subscribers can download on the Actions summary page
-    - name: "ALKiln: Upload errors to action summary artifacts"
-      if: ${{ failure() }}
-      uses: actions/upload-artifact@v2
-      with:
-        name: errors
-        path: ./**/error*.jpg
-    - name: "ALKiln: Upload screenshots to action summary artifacts"
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v2
-      #if-no-files-found: ignore
-      with:
-        name: screenshots
-        path: ./**/screenshot*.jpg
-    - name: "ALKiln: Upload downloaded files to action summary artifacts"
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v2
-      #if-no-files-found: ignore
-      with:
-        name: downloads
-        path: ./**/downloads_*
-    - name: "ALKiln: Upload report to action summary artifacts"
+    - name: "ALKiln: Upload artifacts folder"
       if: ${{ always() }}
       uses: actions/upload-artifact@v2
       with:
-        name: report
-        path: ./**/*_report_*
+        name: ${{ env.ARTIFACT_NAME }}
+        path: ./alkiln_tests_output-*
 
 # ##################################
 # # A developer's workflow should look similar to the below
@@ -177,8 +159,4 @@ runs:
 #         with:
 #           SERVER_URL: "${{ secrets.SERVER_URL }}"
 #           DOCASSEMBLE_DEVELOPER_API_KEY: "${{ secrets.DOCASSEMBLE_DEVELOPER_API_KEY }}"
-#           PLAYGROUND_EMAIL: "${{ secrets.PLAYGROUND_EMAIL }}"
-#           PLAYGROUND_PASSWORD: "${{ secrets.PLAYGROUND_PASSWORD }}"
-#           PLAYGROUND_ID: "${{ secrets.PLAYGROUND_ID }}"
-#           EXTRA_LANGUAGES: "${{ secrets.EXTRA_LANGUAGES }}"
 #       - run: echo "Finished running ALKiln tests"

package/lib/cucumber_8_shim.js

@@ -1,7 +1,7 @@
 module.exports = async function wrapPromiseWithTimeout (
   promise,
   timeoutInMilliseconds,
-  timeoutMessage
+  timeoutMessage=''
 ) {
   /** Race between a timeout and another function and clean up
   *    async afterwards.

package/lib/docassemble/setup.js

@@ -3,6 +3,7 @@
 const da_i = require('./docassemble_api_interface');
 const log = require('../utils/log');
 const session_vars = require('../utils/session_vars' );
+const files = require('../utils/files' );
 
 
 const setup = async () => {
@@ -30,6 +31,11 @@ const setup = async () => {
     await da_i.wait_for_server_to_restart( task_id );
   }
 
+  // Make the folder to store all the files for this run of the tests
+  const artifacts_path_name = files.make_artifacts_folder();
+  // Save its name in a file. `session_vars` can get it back later.
+  session_vars.save_artifacts_path_name( artifacts_path_name );
+
   // Finish
   log.info({ type: `setup`, pre: `Success! Test setup has finished successfully.` });
 };  // Ends setup();

package/lib/scope.js

@@ -2,12 +2,16 @@ const { expect } = require('chai');
 const cheerio = require('cheerio');
 const path = require('path');
 const fs = require('fs');
+const { AxePuppeteer } = require('@axe-core/puppeteer');
 const safe_filename = require("sanitize-filename");
 
 // Ours
 const session_vars = require('./utils/session_vars');
 const get_base_interview_url = require(`./docassemble/get_base_interview_url`);
 const time = require('./utils/time');
+const log = require('./utils/log');
+const { waitForTimeout } = require('./utils/time');
+const files = require('./utils/files' );
 
 
 let ordinal_to_integer = {
@@ -25,7 +29,7 @@ let escape_regex = function( to_escape ) {
 let get_original_row = function (row) {
   /* `row` might be a row, a source row, or an original row. */
 
-  // Until we flatten row data a litte, we need to find the `orignal` row, which has the flags
+  // Until we flatten row data a little, we need to find the `original` row, which has the flags
   let source_row = row.source || row;
   let original = source_row.original || source_row;
   return original;
@@ -47,6 +51,7 @@ let is_secret_var = function(row) {
 }
 
 let proxies_regex = /(\bx\.)|(\bx\[)|(\[[ijklmn]\])/g;
+let misc_artifacts_dir = `_alkiln_test-misc_artifacts`;
 
 module.exports = {
   trigger_not_needed_flag: `ALKiln: no trigger variable needed`,
@@ -56,11 +61,26 @@ module.exports = {
     * 
     * The report has a key for each scenario and each of those
     * keys should correspond to a list containing the relevant data. */
-    if ( session_vars.get_debug() ) { console.log(`Report:\ntype: ${ type }, value: ${ value }`) }
+    if ( session_vars.get_debug() ) { console.log(`Report:\ntype: ${ type }, value: ${ value }\n`) }
     let scenario = await scope.makeSureReportPartsExist( scope );
     scenario.lines.push({ type, value });
 
-    fs.appendFileSync('debug_log.txt', `Report:\ntype: ${ type }, value: ${ value }`);
+    // TODO: Not sure how to use .add_to_debug_log() here considering possibly dynamic filepath
+    // log.add_to_debug_log(`Report:\ntype: ${ type }, value: ${ value }\n`);
+
+    // Ensure filepath for saving local unit test logs
+    if ( !scope.paths ) {
+      // TODO: would it be useful to put in existing artifacts folder if possible.
+      // Can this be usefully abstracted?
+      scope.paths = { debug_log: `${ misc_artifacts_dir }/unit_tests_logs-${ files.readable_date() }.txt` };
+    }
+
+    try {
+      fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
+    } catch (err) {
+      fs.mkdirSync( misc_artifacts_dir );
+      fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
+    }
     return scenario;
     
   },  // Ends scope.addToReport()
@@ -69,7 +89,10 @@ module.exports = {
     /* If a part of the report needs to exist right now, but doesn't exist,
     * create it. */
     if ( !scope.report ) { scope.report = new Map(); }
-    if ( !scope.scenario_id ) { scope.scenario_id = `_report_${ Date.now() }` }
+    if ( !scope.scenario_id ) {
+      let date = files.readable_date();
+      scope.scenario_id = `_report_${ date }`;
+    }
     if ( !scope.report.get( scope.scenario_id )) {
       scope.report.set( scope.scenario_id, {} )
       scope.report.get( scope.scenario_id ).status = `in progress`;
@@ -184,41 +207,49 @@ module.exports = {
       name_parts.push( scope.language );
     }
     name_parts.push( scenario.pickle.name );
-    for ( let tag of scenario.pickle.tags ) {
-      name_parts.push( tag.name );
-    }
 
     let filename = name_parts.join(`-`);
+    let safe_name = safe_filename( filename, { replacement: `_` });
+    // Allow room for extensions, date, etc.
+    let safer_name = safe_name.substring(0, (255 - 45));
 
-    return safe_filename( filename, { replacement: `_` });
+    return safer_name;
   },  // Ends scope.getSafeScenarioBaseFilename()
 
-  getSafeScenarioFilename: async function( scope, options={} ) {
+  // TODO: Does this need a name change? getSafeScenarioBasedFilename?
+  getSafeScenarioFilename: async function( scope, options={ prefix: '' }) {
     /* Return a string that's safe to be a filename. Use a timestamp,
     *    a prefix the caller wants, and the base filename. We hope it can
     *    contain non-english characters too. We do not offer a suffix option
     *    because it may be cut off if the filename is too long.
     */
-    // Ensure prefix is a string
-    let prefix = options.prefix || '';
+    let name = options.prefix || '';
+
     // As long as it's not an empty string, add a separator between it and the rest of the name
-    // For now, it needs to be `_` for the action and .gitignore. In future,
-    // aiming for `-`.
-    if ( prefix ) {
-      prefix = `${ prefix }_`;
+    if ( name.length > 0 ) {
+      name += `-`;
     }
 
-    let safe_name = safe_filename(`${ prefix }${ Date.now() }-${ scope.base_filename }`, { replacement: `_` });
-    let safer_name = safe_name.substring(0, (255 - 10));  // Allow room for extensions
+    if ( scope.page ) {
+      let { id } = await scope.examinePageID( scope, 'none to match' );
+      name += `${ id }-`;
+    }
+
+    name += scope.base_filename;
+    name += `-${ files.readable_date() }`;
+
+    let safe_name = safe_filename( name, { replacement: `_` });
+    // Allow room for extensions, etc.
+    let safer_name = safe_name.substring(0, (255 - 10));
     return safer_name;
   },  // Ends scope.getSafeScenarioFilename()
 
   getJSONFilename: async function( scope ) {
-    let filename = await scope.getSafeScenarioFilename( scope, {
-      prefix: `downloads_${ scope.page_id }_JSON`
-    });
-    filename += `.json`;
-    return filename;
+    // Get a unique path for a json file
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    let date = files.readable_date();
+    let json_path = path.join( scope.paths.scenario, `json_for-${ id }-${ date }.json` );
+    return json_path;
   },
 
   savePageJSONData: async function( scope ) {
@@ -228,13 +259,12 @@ module.exports = {
     * @returns {string} info.filename The name of the file that was saved.
     * @returns {Object} info.json The variables object.
     */
-    // Get a unique json file filename
-    let filename = await scope.getJSONFilename( scope );
+    let json_path = await scope.getJSONFilename( scope );
     // Get the data
     let json = await scope.getPageJSON( scope );
     // Save the data
-    fs.writeFileSync( filename, JSON.stringify( json, null, 2 ));
-    return { filename, json };
+    fs.writeFileSync( json_path, JSON.stringify( json, null, 2 ));
+    return { json_path, json };
   },  // Ends scope.savePageJSONData()
 
   waitForShowIf: async function waitForShowIf(scope) {
@@ -383,7 +413,14 @@ module.exports = {
 
   detectDownloadComplete: async function detectDownloadComplete(scope, endTime) {
     /* Resolve if a download flag has changed. Timeout after a bit less than
-    *    the global permitted timeout */
+    *    the global permitted timeout
+    * 
+    * TODO: Properly wait for download to complete. See 
+    * https://github.com/SuffolkLITLab/ALKiln/issues/492 and
+    * https://stackoverflow.com/a/56951024 and
+    * https://github.com/puppeteer/puppeteer/issues/4676 and
+    * https://github.com/puppeteer/puppeteer/issues/299 and
+    */
 
     // Stop/resolve if too much time has passed
     let expirationTime = scope.timeout - 200;  // ms
@@ -475,6 +512,45 @@ module.exports = {
     await scope.tapElement( scope, elem, tab_id );
   },
 
+  checkForA11y: async function (scope ) {
+    /** Return if there were any violations and the file containing aXe output with more details of the checks */
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    if ( scope.page_id !== id ) {
+      await scope.addToReport(scope, { type: `page_id`, value: `${ id }`, });
+      scope.page_id = id;
+    }
+    const axe = new AxePuppeteer(scope.page);
+    axe.options({
+      runOnly: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"],
+      resultTypes: ['violations', 'incomplete']
+    });
+    let axe_result = await axe.analyze();
+    axe_result.page_id = scope.page_id;
+    axe_result.passes_ids = axe_result.passes.map(pass_obj => pass_obj.id);
+    axe_result.inapplicable_ids = axe_result.inapplicable.map(inapp_obj => inapp_obj.id);
+    axe_result.passes = [];
+    axe_result.inapplicable = [];
+
+    let axe_path = null;
+    const passed = axe_result.violations.length == 0;
+    if (!passed) {
+      let axe_filename = await scope.getSafeScenarioFilename( scope, {prefix: `aXe_failure`}) + '.json';
+      axe_path = `${ scope.paths.scenario }/${ axe_filename }`;
+      fs.writeFileSync( axe_path, JSON.stringify( axe_result, null, 2 ));
+      let msg = `Found potential accessibility issues on the ${ scope.page_id } screen. Details in ${ axe_path }.`;
+      await scope.addToReport( scope, {
+        type: `error`,
+        value: msg
+      });
+    } else {
+      await scope.addToReport(scope, { type: `info`, value: `Accessibility on ${ scope.page_id } passed!`})
+    }
+    return {
+      passed: axe_result.violations.length == 0,
+      axe_filepath: axe_path
+    };
+  },
+
   tapElementBySelector: async function ( scope, selector, waitForTimeout ) {
     let elem = await scope.page.$(`${selector}`);
     let error_msg = `Couldn't find "${selector}" on the page, is there a typo?`;
@@ -625,7 +701,7 @@ module.exports = {
     // All the different types of fields
     // buttons, canvases, inputs of all kinds, selects (dropdowns), textareas. Are there more?
     // Will deal with `option` once inside `select`
-    let all_nodes = $( `#dasigpage, fieldset button, .daquestionactionbutton, fieldset input, .da-form-group input, .da-form-group select, .da-form-group textarea` );
+    let all_nodes = $( `#dasigpage, fieldset button, .daquestionactionbutton, fieldset input, .da-form-group input, .da-form-group select, form select, .da-form-group textarea` );
     for ( let node of all_nodes ) {
       // Decision: Do not abstract the below. There's too much data to pass around for it to make sense
       let $node = $( node );
@@ -1641,9 +1717,17 @@ module.exports = {
       //   await scope.throwPageError( scope, { id: id });  
       // }
 
+      did_navigate = true;
       process.stdout.write(`\x1b[36m${ '>' }\x1b[0m`);  // continued successfully
     }  // ends if !did_navigate && ensure_navigation
 
+    if ( did_navigate && scope.check_all_for_a11y ) {
+      let { passed } = await scope.checkForA11y(scope);
+      if (!passed) {
+        scope.passed_all_for_a11y = false;
+      }
+    }
+
     // TODO: Since we now continue only once no matter what, we can put
     // the error in the caller of the function, though we may want
     // to pass the symbol to print as well.
@@ -1836,12 +1920,18 @@ module.exports = {
       /* Download and save a screenshot. `name` does not have to be
       * unique as the filename will be made unique. */
       name = name || '';
-      let safe_path = await scope.getSafeScenarioFilename(
-        scope, { prefix: `screenshot_${ name }` }
-      );
+
+      let date = files.readable_date();
+      let { id } = await scope.examinePageID( scope, 'none to match' );
+      // Include the custom name if it has one
+      if ( name.length > 0 ) {
+        name = `screenshot-${ name }-on-${ id }-${ date }`;
+      } else {
+        name = `screenshot_on-${ id }-${ date }`;
+      }
 
       await scope.page.screenshot({
-        path: `./${ safe_path }.jpg`,
+        path: `./${ scope.paths.scenario }/${ name }.jpg`,
         type: 'jpeg',
         fullPage: true
       });
@@ -1941,6 +2031,9 @@ module.exports = {
       *    WARNING: Cannot download the same file twice in a single scenario.
       *    WARNING: Must not have any characters that are totally unique to that
       *    page load. It's usually just the name of the file.
+      * 
+      * TODO: Properly wait for download to complete. See notes in
+      * scope.js scope.detectDownloadComplete()
       */
       let [elem] = await scope.page.$x(`//a[contains(@href, "${ filename }")]`);
 

package/lib/steps.js

@@ -18,11 +18,15 @@ const { v4: uuidv4 } = require('uuid');
 
 // Ours
 const scope = require('./scope');
+const log = require('./utils/log');
 const session_vars = require('./utils/session_vars');
+const files = require('./utils/files' );
+
 /* Of Note:
-- We're using `*=` because sometimes da text has funny characters in it that are hard to anticipate
+- We're using `*=` for selectors because sometimes da text has funny characters in it that are hard to anticipate
 */
 
+
 require("dotenv").config();  // Where did this go?
 
 /* TODO:
@@ -68,7 +72,23 @@ BeforeAll(async () => {
   scope.timeout = 30 * 1000;
   scope.showif_timeout = 600;
   scope.report = new Map();
-  fs.writeFileSync('debug_log.txt', "");
+
+  // Store path names for files and docs created for the tests
+  scope.paths = {};
+  // Test setup should have created this filepath. Run setup if there's a problem with this.
+  scope.paths.artifacts = session_vars.get_artifacts_path_name();
+  // For GitHub tests, use the artifacts folder that was created during setup
+  // For local tests, make a new folder for each time scenarios start
+  if ( !session_vars.is_dev_env() ) {
+    // Make a new folder to store all the files for this run of the tests
+    scope.paths.artifacts = files.make_artifacts_folder();
+    // Save its name in a file. `session_vars` can get it back later.
+    session_vars.save_artifacts_path_name( scope.paths.artifacts );
+  }
+  scope.paths.report = `${ scope.paths.artifacts }/report.txt`;
+
+  scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_file }`;
+  fs.writeFileSync( scope.paths.debug_log, "" );
 });
 
 Before(async (scenario) => {
@@ -77,7 +97,14 @@ Before(async (scenario) => {
   scope.expected_in_report = null;
 
   await scope.addReportHeading(scope, {scenario});
+  // Make folder for this Scenario in the all-tests artifacts folder
   scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario});
+  // Add a date for uniqueness in case dev has accidentally copied a Scenario description
+  let date = files.readable_date();
+  scope.paths.scenario = `${ scope.paths.artifacts }/${ scope.base_filename }-${ date }`;
+  fs.mkdirSync( scope.paths.scenario );
+  // Store path name for this Scenario's individualized report
+  scope.paths.scenario_report = `${ scope.paths.scenario }/report.txt`;
 
   // Downloads
   scope.downloadComplete = false;
@@ -88,6 +115,8 @@ Before(async (scenario) => {
   scope.disable_error_screenshot = false;
 
   scope.page_id = null;
+  scope.check_all_for_a11y = false;
+  scope.passed_all_for_a11y = true;
 
   // Reset default timeout
   scope.timeout = 30 * 1000;
@@ -160,10 +189,9 @@ Given(/I start the interview at "([^"]+)"(?: in lang "([^"]+)")?/, {timeout: -1}
       // I've seen stuff take an extra moment or two. Shame to have it everywhere
       try {
         // Interview files should get downloaded to downloads folder
-        let docs_dir = await scope.getSafeScenarioFilename( scope, { prefix: `downloads` });
         await scope.page._client.send('Page.setDownloadBehavior', {
           behavior: 'allow',
-          downloadPath: path.resolve( docs_dir ),  // Save in root of this project
+          downloadPath: path.resolve( scope.paths.scenario ),  // Save in root of this scenario
         });
         resolve();
       } catch ( err ) { reject( err ); }
@@ -174,7 +202,11 @@ Given(/I start the interview at "([^"]+)"(?: in lang "([^"]+)")?/, {timeout: -1}
   await custom_download_behavior_timeout;
 
   // ---- EVENT LISTENERS ----
-  // Wait for possible download. From https://stackoverflow.com/a/51423688
+
+  // TODO: Wait for download to complete doesn't work. See notes in
+  // scope.js scope.detectDownloadComplete()
+
+  // ~Wait for possible download. From https://stackoverflow.com/a/51423688~
   scope.page.on('response', async response => {
     // Note: We can expect that only one file will be getting downloaded
     // at any one time since each step completes synchronously
@@ -186,7 +218,7 @@ Given(/I start the interview at "([^"]+)"(?: in lang "([^"]+)")?/, {timeout: -1}
       // destroyed when the page closes anyway.
 
       // Watch event on download folder or file
-      await fs.watchFile( docs_dir, function (curr, prev) {
+      await fs.watchFile( scope.paths.scenario, function (curr, prev) {
         // If current size eq to size from response then close
         if (parseInt(curr.size) === parseInt(response._headers['content-length'])) {
           // Lets the related function in `scope` know that the download
@@ -646,7 +678,7 @@ Then(/(?:the )?text(?: in)?(?: the)?(?: JSON)?(?: var)?(?:iable)?(?: )?"([^"]+)"
   */
 
   // Get JSON variable value
-  let { filename, json } = await scope.savePageJSONData( scope );
+  let { json_path, json } = await scope.savePageJSONData( scope );
   let actual_value = json.variables[ var_name ];
 
   // Add warning to report if value is not as expected
@@ -654,8 +686,8 @@ Then(/(?:the )?text(?: in)?(?: the)?(?: JSON)?(?: var)?(?:iable)?(?: )?"([^"]+)"
     await scope.addToReport( scope, {
       type: `error`,
       value: `The variable "${ var_name }" was not equal to the expected `
-        + `value on the "${ scope.page_id }" screen. Check the downloads `
-        + `file ${ filename } to see the page's JSON variables.`
+        + `value on the "${ scope.page_id }" screen. Check `
+        + `${ json_path } to see the page's JSON variables.`
     });
   }
 
@@ -730,6 +762,19 @@ When(/I tap the "([^"]+)" element and wait ([0-9]+) second(?:s)?/i, { timeout: -
   )
 });
 
+When(/I check the page for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
+  let { passed, axe_filepath } = await wrapPromiseWithTimeout(
+    scope.checkForA11y(scope),
+    scope.timeout
+  )
+  let msg = `Found potential accessibility issues on the ${ scope.page_id } screen. Details in ${ axe_filepath }.`;
+  expect( passed, msg ).to.be.true;
+});
+
+When(/I check all pages for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
+  scope.check_all_for_a11y = true;
+  scope.passed_all_for_a11y = true;
+});
 
 //#####################################
 // UI element interaction
@@ -893,6 +938,11 @@ Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected
 //#####################################
 
 After(async function(scenario) {
+  // The accessibility failures don't happen til the end, so the rest of the test can continue
+  if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
+    // The fact that the scenario failed has already been noted earlier in the report
+    scenario.result.status = `FAILED`;
+  }
 
   // Record status before internal report tests can interfere with values
   await scope.setReportScenarioStatus( scope, { status: scenario.result.status });
@@ -913,8 +963,22 @@ After(async function(scenario) {
         });
       } else {
         // Save/download a picture of the screen that's showing during the unexpected status
-        let safe_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error` });
-        await scope.page.screenshot({ path: `${ safe_filename }.jpg`, type: `jpeg`, fullPage: true });
+        // Save one copy in the outer-most artifact folder
+        let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` });
+        await scope.page.screenshot({
+          path: `${ scope.paths.artifacts }/${ scenario_filename }.jpg`,
+          type: `jpeg`,
+          fullPage: true
+        });
+        // Save another copy in the artifact's Scenario folder
+        let screenshot_name = `error_on`;
+        let { id } = await scope.examinePageID( scope, 'none to match' );
+        screenshot_name += `-${ id }`;
+        await scope.page.screenshot({
+          path: `${ scope.paths.scenario }/${ screenshot_name }.jpg`,
+          type: `jpeg`,
+          fullPage: true
+        });
       }  // ends if scope.disable_error_screenshot
     }  // ends if scope.page exists
 
@@ -969,6 +1033,13 @@ After(async function(scenario) {
     await scope.page.close();
     scope.page = null;
   }  // ends if scope.page
+
+  // Add report text to Scenario folder after everything has been added to the report
+  let current_report_obj = scope.report.get( scope.scenario_id );
+  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+  let report = await scope.getPrintableScenario( scope, { scenario: scenario_report_obj });
+  // Save the report as in the Scenario folder
+  fs.writeFileSync( scope.paths.scenario_report, report );
 });
 
 AfterAll(async function() {
@@ -978,7 +1049,7 @@ AfterAll(async function() {
   console.log( `\n\n${ report }` );
 
   // Save/download the report as an artifact
-  fs.writeFileSync( `alkiln_report_${ Date.now() }.txt`, report );
+  fs.writeFileSync( scope.paths.report, report );
 
   // If there is a browser open, then close it
   if (scope.browser) {

package/lib/utils/clean_reports.js

@@ -1,23 +1,29 @@
 #!/usr/bin/env node
 
 const fs = require('fs');
+const log = require('./log');
+const session_vars = require('./session_vars');
 
 // This is hardcoded information from `cucumber.js`. Once we move to cucumber 8, we should
-// change that the cucumber.json and actually parse the info from it. 
+// change that the cucumber.json and actually parse the info from it.
+// The file won't be used for anything other than detecting and removing color codes 
 let summary_file_name = 'cucumber-report.txt';
 fs.readFile(summary_file_name, 'utf8', (err, data) => {
   if (err) {
     console.error(err);
     return;
   }
+
   // Replace terminal color codes, if present
   // https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797#colors--graphics-mode
   data = data.replace(/\x1b\[[0-9][0-9]?[0-9]?m/g, '');
-  const debug_log_file = 'debug_log.txt';
-  fs.appendFile(debug_log_file, data, err => {
+
+  // Put it in the correct file in the correct folder
+  let artifacts_path = session_vars.get_artifacts_path_name();
+  fs.appendFile( `${ artifacts_path }/${ log.debug_log_file }`, data, err => {
     if (err) {
       console.error(err);
       return;
     }
-  });
+  });  // Ends appendFileSync
 });

package/lib/utils/files.js

@@ -0,0 +1,32 @@
+const fs = require('fs');
+const safe_filename = require('sanitize-filename');
+
+let files = {};
+
+files.readable_date = function() {
+  /* Return the current timestamp in a human-readable filename-safe format to
+  *  match the workflow artifact date format. */
+  let date = new Date();  // now
+  let str = date.toLocaleString('en-GB', { timeZone: 'UTC' });
+  let fancy_str = str.replace(/\//g, '-').replace(',', ' at').replace(':', 'hrs ').replace(':', 'mins ') + 'secs UTC';
+  return fancy_str;
+};  // Ends files.readable_date()
+
+files.make_artifacts_folder = function (folder_name) {
+  /** Makes a folder in the root of the project, either with the given name
+  *   or with a default name.
+  * 
+  * Works for local development as well, unlike action.yml or the workflow file.
+  */
+  if (folder_name) {
+    folder_name = safe_filename( folder_name );
+  } else {
+    folder_name = `alkiln_tests_output-${ files.readable_date() }`
+  }
+
+  fs.mkdirSync( folder_name );
+  return folder_name;
+};  // Ends files.make_artifacts_folder()
+
+module.exports = files;
+

package/lib/utils/log.js

@@ -1,3 +1,4 @@
+const fs = require('fs');
 const session_vars = require( './session_vars' );
 
 let log = {};
@@ -49,3 +50,9 @@ log.debug = function ({ type='', pre='', data='', post='' }) {
     if ( post ) { console.log( post ); }
   }
 };
+
+log.add_to_debug_log = function(value) {
+  fs.appendFileSync(log.debug_log_file, value);
+};
+
+log.debug_log_file = 'debug_log.txt'

package/lib/utils/session_vars.js

@@ -21,9 +21,11 @@ const session_vars = {};
 module.exports = session_vars;
 
 // These default to null
+// TODO: How to isolate debug elsewhere so this file can use the logger?
 session_vars.get_debug = function () { return process.env.DEBUG || null; };
 session_vars.get_dev_api_key = function () { return process.env.DOCASSEMBLE_DEVELOPER_API_KEY || null; };
 session_vars.get_repo_url = function () { return process.env.REPO_URL || null; };
+session_vars.is_dev_env = function () { return process.env.DEV || null; }
 
 // More complex logic
 session_vars.get_setup_timeout_ms = function () {
@@ -80,9 +82,10 @@ session_vars.get_languages = function () {
 };  // Ends session_vars.get_languages()
 
 
-const project_name_file_path = `_al_test_project_name.json`;
+const runtime_config_path = `runtime_config.json`;
+const project_name_key = `da_project_name`;
 session_vars.save_project_name = function ( project_name ) {
-  /* Saves the name of the project to a file so deletion can use it later.
+  /* Saves the name of the project to a json obj so deletion can use it later.
   *    Built for local as well as remote testing. Assumes a kosher string.
   *    We avoided a list because of concerns over getting out of date.
   *    Should we try to prevent someone overwriting a previous project name?
@@ -90,16 +93,23 @@ session_vars.save_project_name = function ( project_name ) {
   * @param project_name { string } - purely alphanumeric string */
 
   // Warn local developers if file already exists
-  if ( fs.existsSync( project_name_file_path ) ) {
+  if ( fs.existsSync( runtime_config_path ) ) {
     let old_project_name = session_vars.get_project_name();
     let session_project_already_exists_msg = `ALKiln WARNING: `
       + `This session had a project name already: ${ old_project_name }. `
       + `A new project has now been created called ${ project_name }. `
       + `Remember to delete ${ old_project_name }`;
     console.warn( session_project_already_exists_msg );
+  } else {
+    // If the file doesn't exist, make one because next we need to read it
+    fs.writeFileSync( runtime_config_path, JSON.stringify( {}, null, 2 ) );
   }
 
-  let file = fs.writeFileSync( project_name_file_path, JSON.stringify( project_name ) );
+  // Get the json, changing it, and re-write the file from scratch
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ) );
+  json[ project_name_key ] = project_name;
+  let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
+
   if ( session_vars.get_debug() ) {
     console.log( `ALKiln debug: Stored name of Project "${ project_name }" locally.` );
   }
@@ -107,26 +117,59 @@ session_vars.save_project_name = function ( project_name ) {
 };  // Ends save_project_name()
 
 session_vars.get_project_name = function () {
-  /* Get the name of the current project  */
-  let project_name =  JSON.parse( fs.readFileSync( project_name_file_path ));
-  if ( session_vars.get_debug() ) { console.log( `Project name from file is "${ project_name }".` ); }
+  /* Get the name of the current docassemble Project  */
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+  let project_name = json[ project_name_key ] || null;
+  if ( session_vars.get_debug() ) { console.log( `ALKiln debug: Project name from file is "${ project_name }".` ); }
   return project_name;
 };  // Ends get_project_name()
 
 session_vars.delete_project_name = function () {
-  /* Delete the file storing the name of the docassemble project */
-  // Avoid race conditions possibly caused by checking for existence first
-  // Delete file if possible
+  /* Empty the key storing the name of the docassemble Project */
   try {
-    fs.unlinkSync( project_name_file_path );
-    console.log( `ALKiln: Deleted the file that stored the name of the Project.` );
+    // Get the json, changing it, and re-write the file from scratch
+    let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+    json[ project_name_key ] = ``;
+    console.log( `ALKiln: Deleted the data that stored the name of the Project.` );
+    fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
     return true;
   } catch ( error ) {
-    console.warn( `ALKiln WARNING: Could not delete the Project name file. Error: ${ error }.` );
+    console.warn( `ALKiln WARNING: Could not delete the Project name data. Error: ${ error }.` );
     return false;
   }
 };
 
+// The file storing the artifacts path contains the name of the current artifacts folder.
+// The directory name it contains will be created or changed with each run of all the tests.
+// const file_storing_artifact_path = `_alkiln_test_artifacts_dir_name.json`;
+const artifacts_path_key = `artifacts_path`;
+session_vars.save_artifacts_path_name = function ( path_name ) {
+  /* Save a key with the name of the current artifacts folder in case it doesn't exist. */
+
+  // Get the json, changing it, and re-write the file from scratch
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ) );
+  json[ artifacts_path_key ] = path_name;
+  let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
+
+  if ( session_vars.get_debug() ) {
+    console.log( `ALKiln debug: Stored name of artifacts directory "${ path_name }" locally.` );
+  }
+  return file;
+};  // Ends get_project_name()
+
+session_vars.get_artifacts_path_name = function () {
+  /* Get the name of the current artifacts path. */
+  try {
+    let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+    let path_name = json[ artifacts_path_key ];
+
+    if ( session_vars.get_debug() ) { console.log( `ALKiln debug: Artifacts directory is "${ path_name }".` ); }
+    return path_name;
+  } catch ( error ) {
+    return null;
+  }
+};  // Ends get_project_name()
+
 /**
  * Ensure all required environment variables exist. Add each missing variable
  *    to an error that will be thrown at the end.

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "4.5.2",
+  "version": "4.6.2",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
     "test": "npm run unit && npm run langs_setup && npm run cucumber -- $@",
     "cucumber_base": "run_cucumber(){ ./node_modules/.bin/cucumber-js \"$@\"; status=$?; ./lib/utils/clean_reports.js; return $status; }; run_cucumber",
     "cucumber": "run_base(){ npm run cucumber_base -- \"$@\" tests/features/*.feature; }; run_base",
+    "cucumber_external": "run_cucumber(){ tmp_path=`pwd`; ( new_path=\"$1\"; shift; cd \"$new_path\"; npm run --prefix \"${tmp_path}\" cucumber_base -- \"$@\" \"$new_path\"/docassemble/*/data/sources/*.feature;) }; run_cucumber",
     "langs": "npm run langs_setup && ./node_modules/.bin/cucumber-js --require ./lib/index.js tests/features/*_lang*.feature",
     "langs_setup": "./lib/utils/langs.js 'tests/features/*.feature'",
     "langs_local": "npm run langs_setup && npm run local",
@@ -34,6 +35,7 @@
   },
   "homepage": "https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/",
   "dependencies": {
+    "@axe-core/puppeteer": "4.4.2",
     "@cucumber/cucumber": "7.3.2",
     "axios": "0.24.0",
     "chai": "4.2.0",

package/tests/features/interactive_steps.feature

@@ -4,7 +4,7 @@ Feature: Interactive steps
 # In tag names, 'i' is for 'interactive'
 
 @slow @i1
-Scenario: I set text-type values
+Scenario: I set various values
   Given I start the interview at "all_tests"
   Then the question id should be "upload files"
   When I set the var "upload_files_visible" to "some_png_1.png, some_png_2.png"
@@ -87,6 +87,10 @@ Scenario: I set text-type values
   Then the question id should be "button event action"
   And I tap to continue
   # Next page
+  Then the question id should be "single-dropdown-field"
+  And I set the var "single_dropdown_field" to "email"
+  And I tap to continue
+  # Next page
   Then the question id should be "the end"
 
 @fast @i2 @secret

package/tests/features/observation_steps.feature

@@ -98,7 +98,7 @@ Scenario: Test "Then I don’t continue" with an apostrophe
   Then I don’t continue
   And I will be told an answer is invalid
 
-@fast @o9 @table
+@fast @o9 @table @json
 Scenario: I can match JSON page var to str
   Given I start the interview at "all_tests.yml"
   And I get to "showifs" with this data:
@@ -123,8 +123,31 @@ Scenario: I get the page's JSON
   Given I start the interview at "all_tests"
   Then I get the page's JSON variables and values
 
-@fast @o9
+@fast @o11 @screenshot
 Scenario: I take a screenshot
   Given I start the interview at "all_tests"
   Then I take a screenshot
   Then I take a screenshot named "some-screenshot"
+
+@slow @o12 @accessibility @a11y
+Scenario: I check the pages for accessibility
+  Given I start the interview at "all_tests"
+  And I check all pages for accessibility issues
+  And I tap to continue
+  Then I get to "screen features" with this data:
+    | var | value | trigger |
+    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
+    | checkboxes_other['checkbox_other_opt_1'] | true |  |
+    | dropdown_test | dropdown_opt_2 | |
+    | radio_yesno | False | false |
+    | radio_other | radio_other_opt_3 | |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | True | |
+    | object_dropdown | obj_opt_2 | |
+    | object_select_test | obj_chkbx_opt_2 | |
+    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
+    | text_input | Regular text input field value | |
+    | textarea | Multiline text\narea value | |
+    | date_input | today | |
+    | button_continue | True |  |
+    | buttons_other | button_2 |  |
+    | buttons_yesnomaybe | True |  |

package/tests/features/story_tables.feature

@@ -68,6 +68,7 @@ Covers story table tests for:
     | signature_1 |  |  |
     | signature_2 |  |  |
     | single_quote_dict['single_quote_key']['sq_two'] | true |  |
+    | single_dropdown_field | email | |
     | x[i].name.first | Proxyname1 | proxy_list[0].name.first |
     | x[i].name.first | Proxyname2 | proxy_list[1].name.first |
     | x.target_number | 2 | proxy_list.target_number |

package/tests/unit_tests/getSafeScenarioBaseFilename.fixtures.js

@@ -1,8 +1,6 @@
 let names = {};
 
 names.with_spaces = `a tests description`;
-names.tag1 = `tag1`;
-names.tag2 = `tag2`;
 
 names.non_english_chars = `是汉字`;
 

package/tests/unit_tests/getSafeScenarioBaseFilename.test.js

@@ -6,14 +6,11 @@ const scope = require(`../../lib/scope.js`);
 const getSafeScenarioBaseFilename = scope.getSafeScenarioBaseFilename;
 
 
-let mock_scenario = function ( description, tag_names=[] ) {
+let mock_scenario = function ( description ) {
   /** Mock the complex structure of a Scenario object */
   let scenario = {
-    pickle: { name: description, tags: [], }
+    pickle: { name: description }
   };
-  for ( let tag of tag_names ) {
-    scenario.pickle.tags.push({ name: tag });
-  }
   return scenario;
 };  // Ends mock_scenario();
 
@@ -23,39 +20,31 @@ describe(`When I use scope.getSafeScenarioBaseFilename()`, function() {
     scope.language = undefined;
   });
 
-  // No language, no tags
+  // No language
   it(`preserves spaces`, async function() {
-    let scenario = mock_scenario( names.with_spaces, [] );
+    let scenario = mock_scenario( names.with_spaces );
     let result = await getSafeScenarioBaseFilename( scope, { scenario });
     expect( result ).to.equal( names.with_spaces );
   });
 
   it(`preserves non-english characters`, async function() {
-    let scenario = mock_scenario( names.non_english_chars, [] );
+    let scenario = mock_scenario( names.non_english_chars );
     let result = await getSafeScenarioBaseFilename( scope, { scenario });
     expect( result ).to.equal( names.non_english_chars );
   });
 
   it(`replaces invalid characters with "_"`, async function() {
-    let scenario = mock_scenario( names.colon_input, [] );
+    let scenario = mock_scenario( names.colon_input );
     let result = await getSafeScenarioBaseFilename( scope, { scenario });
     expect( result ).to.equal( names.colon_output );
   });
 
-  // language, 1 tag
-  it(`adds a language and 1 tag correctly, preserving numbers`, async function() {
+  // language
+  it(`adds a language correctly to a description`, async function() {
     scope.language = names.non_english_chars;
-    let scenario = mock_scenario( names.with_spaces, [ names.tag1 ] );
+    let scenario = mock_scenario( names.with_spaces );
     let result = await getSafeScenarioBaseFilename( scope, { scenario });
-    let expected = `${ names.non_english_chars }-${ names.with_spaces }-${ names.tag1 }`;
-    expect( result ).to.equal( expected );
-  });
-
-  // 2 tags
-  it(`adds 2 tags correctly, preserving numbers`, async function() {
-    let scenario = mock_scenario( names.with_spaces, [ names.tag1, names.tag2 ] );
-    let result = await getSafeScenarioBaseFilename( scope, { scenario });
-    let expected = `${ names.with_spaces }-${ names.tag1 }-${ names.tag2 }`;
+    let expected = `${ names.non_english_chars }-${ names.with_spaces }`;
     expect( result ).to.equal( expected );
   });
 

package/tests/unit_tests/getSafeScenarioFilename.test.js

@@ -8,9 +8,12 @@
  
  
 let test_filename = function ( expected_prefix, result ) {
-   /* Test a name to see if it's valid output with a language defined. */
-  let expected_name_regex_str = `^${ expected_prefix }\\d{13}-${ names.base_filename }$`;
-   let regex = new RegExp( expected_name_regex_str );
+  /* Test a name to see if it's valid output with a language defined. */
+  let expected_name_regex_str = `^${ expected_prefix }-${ names.base_filename }-\.{37}$`;
+  if ( expected_prefix == `` ) {
+   expected_name_regex_str = `^${ names.base_filename }-\.{37}$`;
+  }
+  let regex = new RegExp( expected_name_regex_str );
  
   // Log more info if it failed
   let found_match = regex.test( result );

package/tests/unit_tests/names.fixtures.js

@@ -3,16 +3,16 @@ let names = {};
 names.base_filename = `a_safe_id`;
 
 names.chinese_input = `是汉字`;
-names.chinese_output = `是汉字_`;
+names.chinese_output = `是汉字`;
 
 names.no_underscore_input = `ends_in_no_underscore`;
-names.no_underscore_output = `ends_in_no_underscore_`;
+names.no_underscore_output = `ends_in_no_underscore`;
 
 names.one_underscore_input = `ends_in_one_underscore_`;
-names.one_underscore_output = `ends_in_one_underscore__`;
+names.one_underscore_output = `ends_in_one_underscore_`;
 
 names.two_underscores_input = `ends_in_two_underscores__`;
-names.two_underscores_output = `ends_in_two_underscores___`;
+names.two_underscores_output = `ends_in_two_underscores__`;
 
 names.empty_string_input = ``;
 names.empty_string_output = ``;
@@ -21,15 +21,15 @@ names.undefined_input;
 names.undefined_output = ``;
 
 names.colon_input = `invalid:colon`;
-names.colon_output = `invalid_colon_`;
+names.colon_output = `invalid_colon`;
 
 names.slash_input = `invalid/slash`;
-names.slash_output = `invalid_slash_`;
+names.slash_output = `invalid_slash`;
 
 names.two_consecutive_invalid_input = `two_consecutive//invalid`;
-names.two_consecutive_invalid_output = `two_consecutive__invalid_`;
+names.two_consecutive_invalid_output = `two_consecutive__invalid`;
 
 names.numerical_input = `scenario_w1th_number00`;
-names.numerical_output = `scenario_w1th_number00_`;
+names.numerical_output = `scenario_w1th_number00`;
 
 module.exports = names;