@suffolklitlab/alkiln 5.2.1 → 5.3.0

package/CHANGELOG.md

@@ -44,6 +44,18 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [5.3.0] - 2023-10-14
+
+### Added
+- Screenshots now come with an HTML file of the same page. Pages can be examined for their full errors and for accessibility details. Note that styles are weird because we don't control the page with js like docassemble does. Otherwise hidden elements are visible, etc. See [#763](https://github.com/SuffolkLITLab/ALKiln/issues/763).
+
+### Fixed
+- Fixed and improved error for accessibility failures. See [#744](https://github.com/SuffolkLITLab/ALKiln/issues/744).
+
+### Internal
+- Refatored report functions into their own file. See https://github.com/SuffolkLITLab/ALKiln/issues/770.
+- Changed the format of the aXe file name to shorten it by avoiding repeating the scenario name.
+
 ## [5.2.1] - 2023-09-29
 
 ### Added

package/lib/scope.js

@@ -17,6 +17,7 @@ const { waitForTimeout } = require('./utils/time');
 const files = require('./utils/files' );
 const server = require('./docassemble/docassemble_api_interface');
 const wrapPromiseWithTimeout = require('./cucumber_8_shim');
+const reports = require(`./utils/reports`);
 
 
 let ordinal_to_integer = {
@@ -82,152 +83,10 @@ let toBase64 = function ( utf8_str, exclude_padding=true) {
 
 let proxies_regex = /(\bx\.)|(\bx\[)|(\[[ijklmn]\])/g;
 let da_base64_regex = /^[A-Za-z0-9+/]+$/;
-let misc_artifacts_dir = `_alkiln-misc_artifacts`;
 
 module.exports = {
   trigger_not_needed_flag: `ALKiln: no trigger variable needed`,
 
-  addToReport: async function( scope, { type, value }) {
-    /* Add an item to a specific list in a scenario's report.
-    * Create report if necessary.
-    * 
-    * The report has a key for each scenario and each of those
-    * keys should correspond to a list containing the relevant data. */
-    if ( session_vars.get_debug() ) { console.log(`Report:\ntype: ${ type }, value: ${ value }\n`) }
-    let scenario = await scope.makeSureReportPartsExist( scope );
-    scenario.lines.push({ type, value });
-
-    // TODO: Not sure how to use .add_to_debug_log() here considering possibly dynamic filepath
-    // log.add_to_debug_log(`Report:\ntype: ${ type }, value: ${ value }\n`);
-
-    // Ensure filepath for saving local unit test logs
-    if ( !scope.paths ) {
-      // TODO: would it be useful to put in existing artifacts folder if possible.
-      // Can this be usefully abstracted?
-      scope.paths = { debug_log: `${ misc_artifacts_dir }/unit_tests_logs-${ files.readable_date() }.txt` };
-    }
-
-    try {
-      fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
-    } catch (err) {
-      fs.mkdirSync( misc_artifacts_dir );
-      fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
-    }
-    return scenario;
-    
-  },  // Ends scope.addToReport()
-
-  makeSureReportPartsExist: async function( scope ) {
-    /* If a part of the report needs to exist right now, but doesn't exist,
-    * create it. */
-    if ( !scope.report ) { scope.report = new Map(); }
-    if ( !scope.scenario_id ) {
-      scope.scenario_id = `_report_${ Date.now() }`;
-    }
-    if ( !scope.report.get( scope.scenario_id )) {
-      scope.report.set( scope.scenario_id, {} )
-      scope.report.get( scope.scenario_id ).status = `in progress`;
-    }
-
-    let scenario = scope.report.get( scope.scenario_id );
-    if ( !scenario.lines ) { scenario.lines = []; }
-
-    return scenario;
-  },  // Ends scope.makeSureReportPartsExist()
-
-  setReportScenarioStatus: async function( scope, { status=`PASSED` }) {
-    /* Add final status of scenario to scenario report object - passed, failed, etc. */
-    scope.report.get( scope.scenario_id ).status = status;
-  },  // Ends scope.setReportScenarioStatus()
-
-  convertToOriginalStoryTableRow: async function( scope, { row_data }) {
-    /* Returns the original data of row into a string formatted as a story table row.
-    * Should this show the actual row data that was used instead? That could
-    * be pretty confusing to most devs. */
-    let source_row = row_data.source || row_data;
-    let original = source_row.original;
-    let var_str = await scope.toTrimmedJSON( scope, { str: original.var });
-    let value_str = await scope.toTrimmedJSON( scope, { str: original.value });
-    let trigger_str = await scope.toTrimmedJSON( scope, { str: original.trigger });
-    if ( trigger_str === scope.trigger_not_needed_flag ) {
-      // This text was added by ALKiln itself. It's not part of the user's data
-      trigger_str = ``;
-    }
-    return `${ ' '.repeat(6) }| ${ var_str } | ${ value_str } | ${ trigger_str } |`;
-  },  // Ends scope.convertToOriginalStoryTableRows()
-
-  toTrimmedJSON: async function( scope, { str='' }) {
-    /* Turn into JSON string then trim quotes from the start and end. */
-    let json = JSON.stringify( str );
-    let trimmed = json.replace( /^"/, '' ).replace( /"$/, '' ).replace( /\\\\/g, `\\`);
-    return trimmed;
-  },  // Ends scope.toTrimmedJSON()
-
-  addReportHeading: async function (scope, { scenario }) {
-    /** Return printable heading based on Scenario name and tags.
-    *    Allow characters from other languages.
-    */
-    if ( session_vars.get_debug() ) { console.log( scenario.pickle.name ); }
-
-    // Collect all tag names
-    let tag_names = [];
-    for ( let tag of scenario.pickle.tags ) {
-      tag_names.push( tag.name );
-    }
-
-    let heading = `\n---------------\n`
-      + `Scenario: ${ scenario.pickle.name }`;
-
-    if ( tag_names.length > 0 ) {
-      heading += `\nTags: ${ tag_names.join(' ') }`;
-    }
-
-    heading += `\n---------------`;
-
-    await scope.addToReport( scope, {
-      type: `heading`,
-      value: heading,
-    });
-  },  // Ends scope.addReportHeading()
-
-  getPrintableReport: async function( scope ) {
-    /* Return a string generated from test report data. */
-    let report = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
-
-    // First list failed scenarios, then non-passed scenarios, then passed scenarios
-    let failed_reports = ``;
-    let passed_reports = ``;
-    let other_reports = ``;
-    for ( let scenario of scope.report ) {
-
-      let report = await scope.getPrintableScenario( scope, { scenario });
-      let [ scenario_id, data ] = scenario;
-      if ( data.status === `FAILED` ) { failed_reports += report; }
-      else if ( data.status === `PASSED` ) { passed_reports += report; }
-      else { other_reports += report; }
-    }  // ends for every scenario
-
-    if ( failed_reports !== `` ) { report += `\n\n===============================\n===============================\nFailed scenarios:\n${ failed_reports }`; }
-    if ( other_reports !== `` ) { report += `\n\n===============================\n===============================\nScenarios that did not pass:\n${ other_reports }`; }
-    if ( passed_reports !== `` ) { report += `\n\n===============================\n===============================\nPassed scenarios:\n${ passed_reports }`; }
-
-    return report;
-  },  // Ends scope.getPrintableReport()
-
-  getPrintableScenario: async function( scope, { scenario }) {
-    /* Return a string generated from scenario list of report items. */
-    let [ scenario_id, data ] = scenario;
-    let report = ``;
-    for ( let line of data.lines ) {
-      if ( line.type === `page_id` ) { report += `screen id: `; }
-      else if ( line.type === `outcome` ) { report += `\n`; }
-      else if ( line.type === `warning` ) { report += `WARNING: `; }
-      else if ( line.type === `error` ) { report += `\nERROR: ` }
-      report += `${ line.value }\n`;
-    }
-    return report;
-  },  // Ends scope.getPrintableScenario()
-
   getSafeScenarioBaseFilename: async function( scope, { scenario }) {
     /** Return a string that's safe to be a filename. Include the Scenario
     *    description and tags.
@@ -275,6 +134,15 @@ module.exports = {
     return safe_name;
   },  // Ends scope.getSafeScenarioFilename()
 
+  getA11yPath: async function( scope ) {
+    // Get a unique path for a json file
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    let short_id = `${ id }`.substring(0, 20);
+    // Need timestamp in case of a loop where the same page gets assessed for a11y multiple times
+    let a11y_path = path.join( scope.paths.scenario, `aXe_failure-${ short_id }-${ Date.now() }.json` );
+    return a11y_path;
+  },
+
   getJSONFilename: async function( scope ) {
     // Get a unique path for a json file
     let { id } = await scope.examinePageID( scope, 'none to match' );
@@ -356,7 +224,7 @@ module.exports = {
         let error_text_prop = await error_elem.getProperty( 'textContent' );
         let system_error_text = await error_text_prop.jsonValue();
 
-        await scope.addToReport(scope, { type: `error`, value: system_error_text });
+        reports.addToReport(scope, { type: `error`, value: system_error_text });
         throw system_error_text;
       };
 
@@ -386,7 +254,7 @@ module.exports = {
     // first in user-triggered cases - users used to be able to access this almost
     // directly, but they can't anymore. Our functions should know to only pass good things.
     let msg = `Cannot find the element to tap.`;
-    if ( !elem ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+    if ( !elem ) { reports.addToReport(scope, { type: `error`, value: msg }); }
 
     expect( elem, msg ).to.exist;
 
@@ -425,7 +293,7 @@ module.exports = {
       } else {
         // Throw any non-timeout error
         let err_msg = `Error occurred when ALKiln tried to tap "${ tapperText }".`
-        scope.addToReport( scope, { type: `error`, value: err_msg });
+        reports.addToReport( scope, { type: `error`, value: err_msg });
         throw error;
       }  // ends if error is timeout error
 
@@ -517,7 +385,7 @@ module.exports = {
       await scope.get_server_reload_promise( scope );
     } else {
       // Otherwise just throw the error
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `error`,
         value: non_reload_report_msg });
       throw error;
@@ -579,7 +447,7 @@ module.exports = {
           + `${ Math.round( ms_till_server_timeout/1000 ) } seconds to respond. `
           + `If the server was just reloading and needed more time, try setting the `
           + `"SERVER_RELOAD_TIMEOUT_SECONDS" GitHub secret to a longer amount of time.`
-        await scope.addToReport( scope, { type: `error`, value: timeout_message });
+        reports.addToReport( scope, { type: `error`, value: timeout_message });
       }
 
       // Once server has responded, throw an error to end the test. After the first
@@ -599,8 +467,8 @@ module.exports = {
     let reload_failure_msg = `The test was unable to continue because the `
       + `server was reloading. ALKiln will try this Scenario a total of 2 times. An `
       + `error will still be printed for this attempt.`;
-    await scope.addToReport( scope, { type: `error`, value: reload_failure_msg });
-    // await scope.addToReport( scope, { type: `error`, value: timeout_message });
+    reports.addToReport( scope, { type: `error`, value: reload_failure_msg });
+    // reports.addToReport( scope, { type: `error`, value: timeout_message });
     // clearTimeout( server_timeout_id );
     throw( reload_failure_msg );
   },  // Ends scope.throw_server_was_reloading_error()
@@ -673,7 +541,7 @@ module.exports = {
     let elem = await scope.page.$(`#${tab_id}`); 
     let error_msg = `Couldn't find the tab with id "#${tab_id}" on the page, is there a typo?`;
     if ( !elem ) {
-      await scope.addToReport(scope, { type: `error`, value: error_msg }); 
+      reports.addToReport(scope, { type: `error`, value: error_msg }); 
     }
     expect( elem, error_msg ).to.exist;
     await scope.tapElement( scope, elem, tab_id );
@@ -683,7 +551,7 @@ module.exports = {
     /** Return if there were any violations and the file containing aXe output with more details of the checks */
     let { id } = await scope.examinePageID( scope, 'none to match' );
     if ( scope.page_id !== id ) {
-      await scope.addToReport(scope, { type: `page_id`, value: `${ id }`, });
+      reports.addToReport(scope, { type: `page_id`, value: `${ id }`, });
       scope.page_id = id;
     }
     const axe = new AxePuppeteer(scope.page);
@@ -701,16 +569,15 @@ module.exports = {
     let axe_path = null;
     const passed = axe_result.violations.length == 0;
     if (!passed) {
-      let axe_filename = await scope.getSafeScenarioFilename( scope, {prefix: `aXe_failure`}) + '.json';
-      axe_path = `${ scope.paths.scenario }/${ axe_filename }`;
+      axe_path = await scope.getA11yPath( scope );
       fs.writeFileSync( axe_path, JSON.stringify( axe_result, null, 2 ));
       let msg = `Found potential accessibility issues on the ${ scope.page_id } screen. Details in ${ axe_path }.`;
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `error`,
         value: msg
       });
     } else {
-      await scope.addToReport(scope, { type: `info`, value: `Accessibility on ${ scope.page_id } passed!`})
+      reports.addToReport(scope, { type: `info`, value: `Accessibility on ${ scope.page_id } passed!`})
     }
     return {
       passed: axe_result.violations.length == 0,
@@ -722,7 +589,7 @@ module.exports = {
     let elem = await scope.page.$(`${selector}`);
     let error_msg = `Couldn't find "${selector}" on the page, is there a typo?`;
     if ( !elem ) {
-      await scope.addToReport(scope, { type: `error`, value: error_msg });
+      reports.addToReport(scope, { type: `error`, value: error_msg });
     }
     expect( elem, error_msg ).to.exist;
     await scope.tapElement( scope, elem, '', waitForTimeout );
@@ -748,7 +615,7 @@ module.exports = {
       interview_url = `${ base_url }${ file_name }`;
     }
 
-    await scope.addToReport(scope,{
+    reports.addToReport(scope,{
       type: `row`,
       value: `Trying to load the interview at "${interview_url}"`
     });
@@ -772,7 +639,7 @@ module.exports = {
       } else {
 
         // Throw any non-timeout error
-        scope.addToReport( scope, {
+        reports.addToReport( scope, {
           type: `error`,
           value: `Error occurred when ALKiln tried to go to the interview url`
         });
@@ -784,7 +651,7 @@ module.exports = {
     let load_result = await scope.getLoadData( scope );
     // Errors, or reaching timeout, causes an exception and is shown to the developer at the end
     if ( load_result.error ) {
-      await scope.addToReport(scope, { type: `error`, value: `On final attempt to load interview, got "${ load_result.error }"` });
+      reports.addToReport(scope, { type: `error`, value: `On final attempt to load interview, got "${ load_result.error }"` });
       expect( load_result.error ).to.equal( '' );
     }
 
@@ -854,7 +721,7 @@ module.exports = {
     if (!var_data && ! raw_var_data) {
       let no_data_msg = `ALKiln Internal error: normalizeTable called w/o var_data or raw_var_data:` +
           ` open an issue at https://github.com/SuffolkLITLab/ALKiln/issues/new if you see this!`;
-      await scope.addToReport( scope, { type: `error`, value: no_data_msg });
+      reports.addToReport( scope, { type: `error`, value: no_data_msg });
       throw Error(no_data_msg);
     }
 
@@ -869,7 +736,7 @@ module.exports = {
             let bad_table_msg = `Your Story Table definition needs to be changed. The best practice`
                 + ` is to make your table 3 columns wide with a first row in this format: "| var | value | trigger |".`
                 + ` The first row of your table is actually "${ raw_array[0] }"`;
-            await scope.addToReport( scope, { type: `error`, value: bad_table_msg})
+            reports.addToReport( scope, { type: `error`, value: bad_table_msg})
             throw Error(bad_table_msg);
           } else if (row_raw.length == 2) {
             var_data.push({var: row_raw[0], value: row_raw[1]});
@@ -889,7 +756,7 @@ module.exports = {
         actual_var_value = process.env[row.flags.secret_name];
         if ( actual_var_value === undefined ) {
           let not_in_env_msg = `The GitHub SECRET "${ row.flags.secret_name }" doesn't exist. Check for a typo and check your workflow file.`;
-          await scope.addToReport( scope, { type: `error`, value: not_in_env_msg })
+          reports.addToReport( scope, { type: `error`, value: not_in_env_msg })
           throw ReferenceError(`Could not find "${row.flags.secret_name }": ${not_in_env_msg}.`)
         }
       }
@@ -941,7 +808,7 @@ module.exports = {
     let encoded_trigger_var = $( trigger_elem ).data( `variable` );  // This should come out to 'None' at the very least
     if ( !encoded_trigger_var ) {
       // TODO: Add documentation and then link to it in this message.
-      await scope.addToReport(scope, { type: `warning`, value: `You are missing an element containing the data for the page's trigger variable. Some fields, like loops with index variables, may not work correctly in these tests. See https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#a-missing-trigger-variable.` });
+      reports.addToReport(scope, { type: `warning`, value: `You are missing an element containing the data for the page's trigger variable. Some fields, like loops with index variables, may not work correctly in these tests. See https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#a-missing-trigger-variable.` });
       encoded_trigger_var = toBase64( `None` );
     }
     let trigger = fromBase64( encoded_trigger_var );
@@ -1080,7 +947,7 @@ module.exports = {
     // Exceptions: Some specific fields can be duplicates without being dangerous
     if ( matching_nodes.length > 1 && !selector.includes(`[class="file-caption-name"]`) ) {
       let msg = `${ matching_nodes.length } items on this page matched the base64 encoded name '${ selector }'. You may be setting the same variable in multiple places on this page.`;
-      await scope.addToReport(scope, { type: `warning`, value: msg });
+      reports.addToReport(scope, { type: `warning`, value: msg });
     }
 
     return selector;
@@ -1359,7 +1226,7 @@ module.exports = {
           + `variable or a generic object, but the page does not `
           + `contain the trigger variable's name in its HTML. See `
           + `https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#a-missing-trigger-variable.`;
-        await scope.addToReport(scope, { type: `warning`, value: page_trigger_warning });
+        reports.addToReport(scope, { type: `warning`, value: page_trigger_warning });
       }
 
       for ( let row_that_matches_this_field of rows_that_match_field_names ) {
@@ -1373,13 +1240,13 @@ module.exports = {
           // a useful message.
 
           // First, make the row data more readable to humans.
-          let row_as_story_table = await scope.convertToOriginalStoryTableRow( scope, { row_data: row_that_matches_this_field });
+          let row_as_story_table = reports.convertToOriginalStoryTableRow( scope, { row_data: row_that_matches_this_field });
           let row_trigger_warning = `This story table row needs a value in the "trigger" `
             + `column, but it is empty. The test might get unexpected results. See `
             + `https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#trigger.`
             + `\nThe row's data:\n${ row_as_story_table }`
             + `\nThis field's data:\n${ JSON.stringify( field )}`;
-          await scope.addToReport(scope, { type: `warning`, value: row_trigger_warning });
+          reports.addToReport(scope, { type: `warning`, value: row_trigger_warning });
           rows_without_a_trigger_name.push( row_that_matches_this_field );
         } else {
           let triggers_match = await scope.matchNeutralizedQuotes( scope, { value1: page_trigger_name, value2: table_row_trigger_name });
@@ -1405,7 +1272,7 @@ module.exports = {
         // First, make the row data more readable to humans.
         let conflicts_as_story_table_rows = [];
         for ( let row of rows_with_conflicting_trigger_names ) {
-          let table_str = await scope.convertToOriginalStoryTableRow( scope, { row_data: row });
+          let table_str = reports.convertToOriginalStoryTableRow( scope, { row_data: row });
           conflicts_as_story_table_rows.push( table_str );
         }
 
@@ -1416,7 +1283,7 @@ module.exports = {
           + `https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#trigger.`
           + `\nThe data of row(s) with conflicting trigger name(s):\n${ conflicts_as_story_table_rows.join( `\n` )}`
           + `\nThis field's data:\n${ JSON.stringify( field )}`;
-        await scope.addToReport(scope, {
+        reports.addToReport(scope, {
           type: `warning`,
           value: all_trigger_names_conflict_msg
         });
@@ -1433,7 +1300,7 @@ module.exports = {
       // First, make the row data more readable to humans.
       let rows_as_story_table = [];
       for ( let row of final_matching_rows ) {
-        let table_str = await scope.convertToOriginalStoryTableRow( scope, { row_data: row });
+        let table_str = reports.convertToOriginalStoryTableRow( scope, { row_data: row });
         rows_as_story_table.push( table_str );
       }
 
@@ -1443,7 +1310,7 @@ module.exports = {
         + `https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/#trigger.`
         + `\nThe matching rows:\n${ rows_as_story_table.join(`\n`) }`
         + `\nThis field's data:\n${ JSON.stringify( field )}`;
-      await scope.addToReport(scope, { type: `warning`, value: multiple_matches_msg });
+      reports.addToReport(scope, { type: `warning`, value: multiple_matches_msg });
     }
 
     if ( session_vars.get_debug() ) { 
@@ -1617,7 +1484,7 @@ module.exports = {
       // If it doesn't have the right format
       if ( answer === null || !answer.includes(`/`) ) {
         // Return field unused.
-        await scope.addToReport( scope, {
+        reports.addToReport( scope, {
           type: `warning`,
           value: `Invalid date. Value given: "${ answer }".`
         });
@@ -1823,7 +1690,7 @@ module.exports = {
     // allow a typo where someone forgot to include a space after a comma.
     let uploads_names = answer.split(`,`);
     if ( uploads_names.length === 0 ) {
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `error`,
         value: `Your list of files to upload is empty.`
       });
@@ -1855,7 +1722,7 @@ module.exports = {
 
     // Make sure sources directory exists and has files in it
     if ( dirs.length === 0 ) {
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `warning`,
         value: `The test files need to be in the "sources" directory, `
           + `but ALKiln couldn't find the directory or it was empty.`
@@ -1887,7 +1754,7 @@ module.exports = {
     }  // ends for each file
 
     if ( missing_files.length > 0 ) {
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `warning`,
         value: `ALKiln could not find "${ missing_files.join('", "') }" in any of these folders: "${ dirs.join('", "') }".`
       });
@@ -1944,7 +1811,7 @@ module.exports = {
     * takes a while to load. */
     let canvas = await scope.page.waitForSelector( `#dasigcanvas` );
     let msg = `Could not find a signature field.`;
-    if ( !canvas ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+    if ( !canvas ) { reports.addToReport(scope, { type: `error`, value: msg }); }
     expect( canvas, msg ).to.exist;
 
     // Provides functionality to draw name arg on canvas
@@ -2017,7 +1884,7 @@ module.exports = {
     // Only record the page id once for each page to which we navigate
     let { id } = await scope.examinePageID( scope, 'none to match' );
     if ( scope.page_id !== id ) {
-      await scope.addToReport(scope, { type: `page_id`, value: `${ id }`, });
+      reports.addToReport(scope, { type: `page_id`, value: `${ id }`, });
       scope.page_id = id;
     }
 
@@ -2047,9 +1914,9 @@ module.exports = {
 
           something_new_was_set = true;
           field.was_set = true;
-          await scope.addToReport(scope, {
+          reports.addToReport(scope, {
             type: `row`,
-            value: `${ await scope.convertToOriginalStoryTableRow(scope, { row_data: row_used }) }`
+            value: `${ reports.convertToOriginalStoryTableRow(scope, { row_data: row_used }) }`
           });
 
           // Accumulate number of times this row was used. Esp important for `.target_number`
@@ -2072,9 +1939,9 @@ module.exports = {
         let row_used = await scope.setVariable( scope, { field, var_data, });
         if ( row_used ) {
 
-          await scope.addToReport(scope, {
+          reports.addToReport(scope, {
             type: `row`,
-            value: `${ await scope.convertToOriginalStoryTableRow(scope, { row_data: row_used }) }`
+            value: `${ reports.convertToOriginalStoryTableRow(scope, { row_data: row_used }) }`
           });
 
           // Esp important for `.target_number`
@@ -2116,13 +1983,13 @@ module.exports = {
           if ( row.source ) { source_row = row.source; }
           let original_row = source_row.original;
           let msg = `Did not find a field on this page for the variable "${ original_row.var }" that could be set to "${ original_row.value }"`;
-          await scope.addToReport(scope, { type: `warning`, value: msg });
+          reports.addToReport(scope, { type: `warning`, value: msg });
         }
       }
       if ( num_unused_rows > 0 ) {
         let { id } = await scope.examinePageID( scope, 'none to match' );
         let msg = `Missing variable or variables on page "${id}"`;
-        await scope.addToReport(scope, { type: `error`, value: msg });
+        reports.addToReport(scope, { type: `error`, value: msg });
         expect( num_unused_rows, msg ).to.equal( 0 );
       }
     }
@@ -2312,7 +2179,7 @@ module.exports = {
         // probably add a confusing message to the report, so mutating seems, sadly, more useful.
         var_datum.value = `False`;  // The original value is still preserved
         // Warn the developer not to use `.there_is_another`.
-        await scope.addToReport( scope, { type: `warning`, 
+        reports.addToReport( scope, { type: `warning`, 
             value: 'The attribute `.there_is_another` is invalid in story table tests. Replace it with '
               + '`.target_number` in your `var` and `trigger` columns. Set the `value` to the number of items in '
               + `that list. This test will now set this row's \`value\` to \`False\`. See `
@@ -2346,10 +2213,10 @@ module.exports = {
     } else if ( value.match( /(true|yes|checked|check|selected|select)/i )) {
       if ( from_story_table && !row.artificial ) {
         if ( row.var.match( /\.there_is_another$/ ) ) {
-          await scope.addToReport( scope, { type: `warning`, value: `The attribute \`.there_is_another\` is invalid in story table tests. Replace it with \`.target_number\` in your \`var\` column. Set the \`value\` to the number of items in that list. This test will now set this row's \`value\` to \`False\`. See https://suffolklitlab.github.io/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing#there_is_another-loop. The row data is\n${ JSON.stringify( row.original )}` });
+          reports.addToReport( scope, { type: `warning`, value: `The attribute \`.there_is_another\` is invalid in story table tests. Replace it with \`.target_number\` in your \`var\` column. Set the \`value\` to the number of items in that list. This test will now set this row's \`value\` to \`False\`. See https://suffolklitlab.github.io/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing#there_is_another-loop. The row data is\n${ JSON.stringify( row.original )}` });
         } else {
           // If `target_number` is set to True instead of a number
-          await scope.addToReport( scope, { type: `warning`, value: `${ val_to_print } value is not a valid value for ${ row.var } here. This test will default to \`False\` to avoid problems.` });
+          reports.addToReport( scope, { type: `warning`, value: `${ val_to_print } value is not a valid value for ${ row.var } here. This test will default to \`False\` to avoid problems.` });
         }
         return `False`;
       } else { return `True`; }
@@ -2357,7 +2224,7 @@ module.exports = {
     } else if ( isNaN( desired_num )) {  // Story table loop will end
       // I can't figure out how to trigger this warning, so maybe we're
       // already handling everything we need to
-      await scope.addToReport( scope, { type: `warning`, value: `${ val_to_print } value is not a valid value for ${ row.var } here. This test will default to \`False\` to avoid problems.` });
+      reports.addToReport( scope, { type: `warning`, value: `${ val_to_print } value is not a valid value for ${ row.var } here. This test will default to \`False\` to avoid problems.` });
       return `False`;
 
 	    // Check that the number of elements in the list hits our desired number.
@@ -2408,7 +2275,7 @@ module.exports = {
     }
 
     let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. Check the picture of this page. If a secret variable was used, there will be no picture.`
-    if ( timedOut ) { await scope.addToReport(scope, { type: `error`, value: timeout_err_message }); }
+    if ( timedOut ) { reports.addToReport(scope, { type: `error`, value: timeout_err_message }); }
     expect( timedOut, timeout_err_message ).to.be.false;
 
     clearTimeout( cont_timeout );
@@ -2452,13 +2319,13 @@ module.exports = {
     /* Throw a cucumber error with the error that appeared on the page. */
     if ( error_handlers.system_error !== undefined ) {
       let sys_err_message = `The test tried to continue to the next page after ${ id }, but there was a system error. Check the picture of this page.`
-      await scope.addToReport(scope, { type: `error`, value: sys_err_message });
+      reports.addToReport(scope, { type: `error`, value: sys_err_message });
       expect( was_system_error, sys_err_message ).to.be.false;
 
     } else {
       // If it wasn't a system error, it was a user error
       let user_err_message = `The test tried to continue to the next page after ${ id }, but the user got an error. Check the picture of this page.`
-      await scope.addToReport(scope, { type: `error`, value: user_err_message });
+      reports.addToReport(scope, { type: `error`, value: user_err_message });
       expect( true, user_err_message ).to.be.false;
     }
 
@@ -2480,6 +2347,22 @@ module.exports = {
       fullPage: fullPage,
     });
 
+    let html_path = path;
+    if (path.endsWith(".jpg")) {
+      html_path = html_path.substring(0, html_path.length - 4) + ".html";
+    } else {
+      html_path = html_path + ".html";
+    }
+
+    // Also save the HTML of the page
+    await scope.page.content().then(content => {
+      let server_url = session_vars.get_da_server_url();
+      content = content.replaceAll(/"(\/static\/.*\.css\?v=[^"]+")/g, server_url + "$1");
+      content = content.replaceAll(/"(\/static\/.*\.js\?v=[^"]+")/g, server_url + "$1");
+      content = content.replaceAll(/"(\/packagestatic\/.*\.css\?v=[^"]+")/g, server_url + "$1");
+      content = content.replaceAll(/"(\/packagestatic\/.*\.js\?v=[^"]+")/g, server_url + "$1");
+      fs.writeFileSync(html_path, content)
+    });
   },  // Ends scope.take_a_screenshot()
 
 
@@ -2519,7 +2402,7 @@ module.exports = {
       });
 
       // In future, may be saved to artifact file
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `json`,
         value: JSON.stringify( data, null, 2 )
       });
@@ -2535,7 +2418,7 @@ module.exports = {
       let response = await linkPage.goto( actual_url, { waitUntil: 'domcontentloaded' });
 
       let msg = `The "${ linkText }" link is broken.`;
-      if ( !response.ok() ) { await scope.addToReport( scope, { type: `error`, value: msg }); }
+      if ( !response.ok() ) { reports.addToReport( scope, { type: `error`, value: msg }); }
       expect( response.ok(), msg ).to.be.true;
 
       // This may cause problems if the linked page is the interview page
@@ -2584,7 +2467,7 @@ module.exports = {
       const [link] = await scope.page.$x(`//a[contains(text(), "${phrase}")]`);
 
       let msg = `The term "${ phrase }" seems to be missing.`;
-      if ( !link ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+      if ( !link ) { reports.addToReport(scope, { type: `error`, value: msg }); }
       expect( link, msg ).to.exist;
 
       await scope.tapElement( scope, link );
@@ -2611,7 +2494,7 @@ module.exports = {
       let [elem] = await scope.page.$x(`//a[contains(@href, "${ filename }")]`);
 
       let msg = `"${ filename }" seems to be missing. Cannot find a link to that document.`;
-      if ( !elem ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+      if ( !elem ) { reports.addToReport(scope, { type: `error`, value: msg }); }
       expect( elem, msg ).to.exist;
 
       scope.toDownload = filename;
@@ -2647,7 +2530,7 @@ module.exports = {
         name_parts_slice.push(name_parts_last);
         name_parts = name_parts_slice;
       
-        await scope.addToReport(scope, {
+        reports.addToReport(scope, {
           type: `warning`,
           value: `The name "${name_str}" has more than 4 parts, but 4 is the maximum allowed. The test will set the name to "${name_parts.join(" ")}".`
         });
@@ -2733,7 +2616,7 @@ module.exports = {
           await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
         } else {
           // Throw any non-timeout error
-          scope.addToReport( scope, { type: `error`, value: err_msg });
+          reports.addToReport( scope, { type: `error`, value: err_msg });
           throw error;
         }  // ends if error is timeout error
 
@@ -2747,11 +2630,11 @@ module.exports = {
       // this is complex
       let email_msg = `The email address GitHub SECRET "${ email_secret_name }" doesn't exist. Check for a typo and check your workflow file.`
       if ( email === undefined ) {
-        await scope.addToReport( scope, { type: `error`, value: email_msg })
+        reports.addToReport( scope, { type: `error`, value: email_msg })
       }
       let password_msg = `The password GitHub SECRET "${ password_secret_name }" doesn't exist. Check for a typo and check your workflow file.`
       if ( password === undefined ) {
-        await scope.addToReport( scope, { type: `error`, value: password_msg })
+        reports.addToReport( scope, { type: `error`, value: password_msg })
       }
 
       expect( email, email_msg ).to.not.equal( undefined );
@@ -2768,7 +2651,7 @@ module.exports = {
       // TODO: use a Promise.race to fail faster.
       await scope.page.waitForSelector( `.datopcenter .alert-success` );
 
-      await scope.addToReport( scope, {
+      reports.addToReport( scope, {
         type: `row`,
         value: `Signed into ${ session_vars.get_da_server_url() }/user/sign-in successfully.`
       });
@@ -2910,7 +2793,7 @@ module.exports = {
         if ( type !== `` && type !== `hidden` ) {
           // Field types that we can't yet handle, but plan to in future
           if ( type === `file` ) {
-            await scope.addToReport( scope, {
+            reports.addToReport( scope, {
               type: `warning`,
               value: `Sorry, ALKiln's random input tests cannot yet handle "${ type }" fields.`
             });
@@ -3016,7 +2899,7 @@ module.exports = {
     */
     let current_report_obj = scope.report.get( scope.scenario_id );
     let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
-    let report = await scope.getPrintableScenario( scope, { scenario });
+    let report = reports.getPrintableScenario( scenario );
     let all_are_included = true;
     for ( let one_expectation of expected ) {
       if ( !report.includes( one_expectation )) {
@@ -3036,7 +2919,7 @@ module.exports = {
     */
     let current_report_obj = scope.report.get( scope.scenario_id );
     let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
-    let report = await scope.getPrintableScenario( scope, { scenario });
+    let report = reports.getPrintableScenario( scenario );
     let none_included = true;
     for ( let one_expectation of not_expected ) {
       if ( report.includes( one_expectation )) {

package/lib/steps.js

@@ -21,6 +21,7 @@ const scope = require('./scope');
 const log = require('./utils/log');
 const session_vars = require('./utils/session_vars');
 const files = require('./utils/files' );
+const reports = require(`./utils/reports`);
 
 /* Of Note:
 - We're using `*=` for selectors because sometimes da text has funny characters in it that are hard to anticipate
@@ -125,7 +126,7 @@ Before(async (scenario) => {
   scope.expected_not_in_report = null;
   scope.server_reload_promise = null;
 
-  await scope.addReportHeading(scope, {scenario});
+  reports.addReportHeading(scope, {scenario});
   // Make folder for this Scenario in the all-tests artifacts folder
   scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario});
   // Add a date for uniqueness in case dev has accidentally copied a Scenario description
@@ -353,7 +354,7 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
           resolve();
         } catch ( err ) {
           let err_msg = `Error occurred when tried to answer fields on question "${ id }".`
-          scope.addToReport( scope, { type: `error`, value: err_msg });
+          reports.addToReport( scope, { type: `error`, value: err_msg });
           // Throw error instead?
           reject( err );
         } finally {
@@ -372,7 +373,7 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
 
   // Add report strings from supported table, not an altered one
   for ( let row of supported_table ) {
-    row.report_str = `${ await scope.convertToOriginalStoryTableRow(scope, { row_data: row }) }`;
+    row.report_str = `${ reports.convertToOriginalStoryTableRow(scope, { row_data: row }) }`;
   }
   // Sort into alphabetical order, as is recommended for writing story tables
   let sorted_rows = supported_table.sort( function ( row1, row2 ) {
@@ -383,13 +384,13 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
 
   // Add full used table to report so it can be copied if needed.
   // TODO: Should we/How do we get this in the final scenario report instead of here?
-  await scope.addToReport( scope, { type: `note`, value: `\n${ ' '.repeat(2) }Rows that got set:` });
-  await scope.addToReport( scope, { type: `step`, value: `${ ' '.repeat(4) }And I get to the question id "${ target_id }" with this data:` });
-  await scope.addToReport(scope, { type: `row`, value: `${ ' '.repeat(6) }| var | value | trigger |`, });
+  reports.addToReport( scope, { type: `note`, value: `\n${ ' '.repeat(2) }Rows that got set:` });
+  reports.addToReport( scope, { type: `step`, value: `${ ' '.repeat(4) }And I get to the question id "${ target_id }" with this data:` });
+  reports.addToReport(scope, { type: `row`, value: `${ ' '.repeat(6) }| var | value | trigger |`, });
   let at_least_one_row_was_NOT_used = false;
   for ( let row of sorted_rows ) {
     if ( row.times_used > 0 ) {
-      await scope.addToReport(scope, { type: `row`, value: row.report_str, });
+      reports.addToReport(scope, { type: `row`, value: row.report_str, });
     } else {
       at_least_one_row_was_NOT_used = true;
     }
@@ -397,17 +398,17 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
 
   // Add unused rows if needed
   if ( at_least_one_row_was_NOT_used ) {
-    await scope.addToReport( scope, { type: `note`, value: `${ ' '.repeat(2) }Unused rows:` });
+    reports.addToReport( scope, { type: `note`, value: `${ ' '.repeat(2) }Unused rows:` });
     for ( let row of sorted_rows ) {
       if ( row.times_used === 0 ) {
-        await scope.addToReport(scope, { type: `row`, value: row.report_str, });
+        reports.addToReport(scope, { type: `row`, value: row.report_str, });
       }
     }
   } else {
-    await scope.addToReport(scope, { type: `note`, value: `${ ' '.repeat(2) }All rows were used` });
+    reports.addToReport(scope, { type: `note`, value: `${ ' '.repeat(2) }All rows were used` });
   }
 
-  await scope.addToReport(scope, { type: `note`, value: `` });
+  reports.addToReport(scope, { type: `note`, value: `` });
 });
 
 //#####################################
@@ -457,7 +458,7 @@ Then(/the (?:question|page|screen) id should be "([^"]+)"/i, { timeout: -1 }, as
   // No question id exists
   let no_id_msg = `Did not find any question id.`;
   if ( !question_has_id ) {
-    await scope.addToReport( scope, { type: `error`, value: no_id_msg });
+    reports.addToReport( scope, { type: `error`, value: no_id_msg });
   }
   expect( question_has_id , no_id_msg ).to.be.true;
 
@@ -465,7 +466,7 @@ Then(/the (?:question|page|screen) id should be "([^"]+)"/i, { timeout: -1 }, as
   let no_match_msg = `The question id was supposed to be "${ question_id }"`
     + `, but it's actually "${ id }".`;
   if ( !id_matches ) {
-    await scope.addToReport( scope, { type: `error`, value: no_match_msg });
+    reports.addToReport( scope, { type: `error`, value: no_match_msg });
   }
   expect( id_matches, no_match_msg ).to.be.true;
 
@@ -478,7 +479,7 @@ Then(/I ?(?:should)? see the phrase "([^"]+)"/i, async ( phrase ) => {  // √
   const bodyText = await scope.page.$eval('body', elem => elem.innerText);
 
   let msg = `The text "${ phrase }" SHOULD be on this page, but it's NOT.`;
-  if ( !bodyText.includes( phrase )) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !bodyText.includes( phrase )) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( bodyText, msg ).to.contain( phrase );
 
   await scope.afterStep(scope);
@@ -492,7 +493,7 @@ Then(/I should not see the phrase "([^"]+)"/i, async ( phrase ) => {  // √
   const bodyText = await scope.page.$eval('body', elem => elem.innerText);
 
   let msg = `The text "${ phrase }" should NOT be on this page, but it IS here.`;
-  if ( bodyText.includes( phrase )) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( bodyText.includes( phrase )) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( bodyText, msg ).not.to.contain( phrase );
 
   await scope.afterStep(scope);
@@ -502,7 +503,7 @@ Then('an element should have the id {string}', async ( id ) => {  // √
   const element = await scope.page.$('#' + id);
 
   let msg = `No element on this page has the ID "${ id }".`;
-  if ( !element ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !element ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( element, msg ).to.exist;
 
   await scope.afterStep(scope);
@@ -557,7 +558,7 @@ Then(/I (don't|can't|don’t|can’t|cannot|do not) continue/, async (unused) =>
   *    Can other things trigger navigation? Re: other inputs things
   *    like 'Enter' acting like a click is a test for docassemble */
   let msg = `The page should have stopped the user from continuing, but the user was able to continue.`;
-  if ( scope.navigated ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( scope.navigated ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( scope.navigated, msg ).to.be.false;
 
   await scope.afterStep(scope);
@@ -568,12 +569,12 @@ Then('I will be told an answer is invalid', async () => {  // √
   let { was_found, error_handlers } = await scope.checkForError( scope );
 
   let no_error_msg = `No error message was found on the page`;
-  if ( !was_found ) { await scope.addToReport(scope, { type: `error`, value: no_error_msg }); }
+  if ( !was_found ) { reports.addToReport(scope, { type: `error`, value: no_error_msg }); }
   expect( was_found, no_error_msg ).to.be.true;
 
   let was_user_error = error_handlers.complex_user_error !== undefined || error_handlers.simple_user_error !== undefined;
   let not_user_error_msg = `The error was a system error, not an invalid user answer. Check for the picture of the page's error if the step did not use any secret variables.`;
-  if ( !was_user_error ) { await scope.addToReport(scope, { type: `error`, value: not_user_error_msg }); }
+  if ( !was_user_error ) { reports.addToReport(scope, { type: `error`, value: not_user_error_msg }); }
   expect( was_user_error, not_user_error_msg ).to.be.true;
 
   await scope.afterStep(scope);
@@ -584,7 +585,7 @@ Then('I arrive at the next page', async () => {  // √
   *    Can other things trigger navigation? Re: other inputs things
   *    like 'Enter' acting like a click is a test for docassemble */
   let msg = `User did not arrive at the next page.`
-  if ( !scope.navigated ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !scope.navigated ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( scope.navigated, msg ).to.be.true;
   await scope.afterStep(scope);
 });
@@ -595,7 +596,7 @@ Then('I should see the link {string}', async ( linkText ) => {  // √
   let [link] = await scope.page.$x(`//a[contains(text(), "${linkText}")]`);
 
   let msg = `Cannot find a link with the text "${ linkText }".`;
-  if ( !link ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !link ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( link, msg ).to.exist;
 
   await scope.afterStep(scope);
@@ -606,7 +607,7 @@ Then('I should see the link to {string}', async ( url ) => {  // √
   let link = await scope.page.$(`*[href="${ url }"]`);
 
   let msg = `Cannot find a link to "${ url }".`;
-  if ( !link ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !link ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect(link, msg ).to.exist;
   
   await scope.afterStep(scope);
@@ -622,7 +623,7 @@ Then(/the "([^"]+)" link leads to "([^"]+)"/, async (linkText, expected_url) =>
   let actual_url = await prop_obj.jsonValue();
 
   let msg = `Cannot find a link with the text "${ linkText }" leading to ${ expected_url }.`;
-  if ( actual_url !== expected_url ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( actual_url !== expected_url ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( actual_url ).to.equal( expected_url );
 
   await scope.afterStep(scope);
@@ -641,7 +642,7 @@ Then(/the "([^"]+)" link opens in (a new window|the same window)/, async (linkTe
     || ( !should_open_a_new_window && !opens_a_new_window );
 
   let msg = `The link "${ linkText }" does NOT open in ${ which_window }.`;
-  if ( !hasCorrectWindowTarget ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
+  if ( !hasCorrectWindowTarget ) { reports.addToReport(scope, { type: `error`, value: msg }); }
   expect( hasCorrectWindowTarget, msg ).to.be.true;
 
   await scope.afterStep(scope);
@@ -674,7 +675,7 @@ Then(/(?:the )?text(?: in)?(?: the)?(?: JSON)?(?: var)?(?:iable)?(?: )?"([^"]+)"
 
   // Add warning to report if value is not as expected
   if ( actual_value !== expected_value ) {
-    await scope.addToReport( scope, {
+    reports.addToReport( scope, {
       type: `error`,
       value: `The variable "${ var_name }" was not equal to the expected `
         + `value on the "${ scope.page_id }" screen. Check `
@@ -781,7 +782,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
   let random_input_path = `${ scope.paths.scenario }/random${ timestamp }`
   scope.paths.random_screenshots = random_input_path;
   fs.mkdirSync( scope.paths.random_screenshots );
-  await scope.addToReport( scope, {
+  reports.addToReport( scope, {
     type: 'row',
     value: `Testing interview with random input (in ${ random_input_path })`
   });
@@ -816,7 +817,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
         let has_content = await scope.has_interview_content( scope );
 
         if ( !has_content ) {
-          await scope.addToReport( scope, {
+          reports.addToReport( scope, {
             type: `error`,
             value: no_content_err
           });
@@ -825,7 +826,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
         } else {
           let timeout_message = `The page with an id of "${ id }" took too long`
             + ` to fill with random answers (over ${ scope.timeout/1000/60 } min).`
-          await scope.addToReport( scope, { type: `error`, value: timeout_message });
+          reports.addToReport( scope, { type: `error`, value: timeout_message });
           reject( timeout_message );
         }
 
@@ -866,7 +867,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
             // Make sure to only throw once
             if ( !has_thrown ) {
               has_thrown = true;
-              await scope.addToReport( scope, {
+              reports.addToReport( scope, {
                 type: `error`,
                 value: no_content_err
               });
@@ -890,7 +891,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
           // Throw a specific error if there's no interview content
           let has_content = await scope.has_interview_content( scope );
           if ( !has_content ) {
-            await scope.addToReport( scope, {
+            reports.addToReport( scope, {
               type: `error`,
               value: no_content_err
             });
@@ -900,7 +901,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
           } else {
             let err_msg = `The page with the question id "${ id }" errored when trying to fill`
               + ` it with random answers: ${ error.name }`
-            await scope.addToReport( scope, { type: `error`, value: err_msg });
+            reports.addToReport( scope, { type: `error`, value: err_msg });
             reject( error );
           }  // ends if !has_content
 
@@ -1098,24 +1099,29 @@ Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected
 
 After(async function(scenario) {
 
+  let changeable_test_status = scenario.result.status;
   // The accessibility failures don't happen til the end, so the rest of the test can continue
   if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
     // The fact that the scenario failed has already been noted earlier in the report
-    scenario.result.status = `FAILED`;
+    changeable_test_status = `FAILED`;
+    scope.addToReport(scope, {
+      type: `error`,
+      value: `Accessibility standards weren't met. See information above.`
+    });
   }
 
   // Record status before internal report tests can interfere with values
-  await scope.setReportScenarioStatus( scope, { status: scenario.result.status });
+  reports.setReportScenarioStatus( scope, { status: changeable_test_status });
 
-  if (scenario.result.status !== `PASSED`) {
+  if (changeable_test_status !== `PASSED`) {
     if ( session_vars.get_debug() ) {
-      console.log( `Non-passed status occurred while running test:`, scenario.result.status );
+      console.log( `Non-passed status occurred while running test:`, changeable_test_status );
     }
 
     if ( scope.page ) {
 
       if ( !!scope.disable_error_screenshot ) {
-        scope.addToReport(scope, {
+        reports.addToReport(scope, {
           type: `row`,
           value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen.`
         });
@@ -1139,21 +1145,19 @@ After(async function(scenario) {
     }  // ends if scope.page exists
 
     // This has to come after the security message or security message doesn't print. Not sure why.
-    if (scenario.result.status === `FAILED`) {
-      await scope.addToReport(scope, { type: `outcome`, value: `**-- Scenario Failed --**` });
+    if (changeable_test_status === `FAILED`) {
+      reports.addToReport(scope, { type: `outcome`, value: `**-- Scenario Failed --**` });
     }
 
   }  // ends if not passed
 
-  let original_scenario_status = scenario.result.status;
-
   // Internal tests of this framework
   if ( scope.expected_status ) {
     let expected = scope.expected_status;
     scope.expected_status = null;  // reset for next Scenario
-    expect( scenario.result.status ).to.equal( expected );
+    expect( changeable_test_status ).to.equal( expected );
     // If the status was as expected, pass this criteria
-    scenario.result.status = `PASSED`;
+    changeable_test_status = `PASSED`;
   }
   
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
@@ -1162,8 +1166,8 @@ After(async function(scenario) {
     scope.expected_in_report = null;
     let all_were_included = await scope.reportIncludesAllExpected( scope, { expected });
 
-    if ( all_were_included ) { scenario.result.status = `PASSED`; }
-    else { scenario.result.status = `FAILED`; }
+    if ( all_were_included ) { changeable_test_status = `PASSED`; }
+    else { changeable_test_status = `FAILED`; }
   }
   
   if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) {
@@ -1173,9 +1177,9 @@ After(async function(scenario) {
     let all_were_not_there = await scope.reportDoesNotInclude( scope, { not_expected });
 
     if ( all_were_not_there ) {
-      scenario.result.status = `PASSED`;
+      changeable_test_status = `PASSED`;
     } else { 
-      scenario.result.status = `FAILED`;
+      changeable_test_status = `FAILED`;
     }
 
   }
@@ -1199,7 +1203,7 @@ After(async function(scenario) {
       } else {
         // Throw any non-timeout error
         let err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
-        scope.addToReport( scope, { type: `error`, value: err_msg });
+        reports.addToReport( scope, { type: `error`, value: err_msg });
         throw error;
       }  // ends if error is timeout error
 
@@ -1210,16 +1214,29 @@ After(async function(scenario) {
   // Add report text to Scenario folder after everything has been added to the report
   let current_report_obj = scope.report.get( scope.scenario_id );
   let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
-  let report = await scope.getPrintableScenario( scope, { scenario: scenario_report_obj });
+  let report = reports.getPrintableScenario( scenario_report_obj );
   // Save the report as in the Scenario folder
   fs.writeFileSync( scope.paths.scenario_report, report );
+
+  if ( changeable_test_status === `FAILED` ) {
+    // If the result object status wasn't failure and we still want
+    // the tests to fail, as with a11y tests, we can't mess with the
+    // cucumber result.status object because it will cause unwanted
+    // errors. We have to throw an error here instead.
+    if ( scenario.result.status !== `FAILED` ) {
+      throw new Error(`The test failed. See the report for more information.`);
+    }
+  } else {
+    // Other types of errors should be ok
+    scenario.result.status = changeable_test_status;
+  }
 });
 
 AfterAll(async function() {
   // Stop collecting server response statuses
   scope.stop_tracking_server_status = true;
   
-  let report = await scope.getPrintableReport( scope );
+  let report = reports.getPrintableReport( scope );
   // Log the report. Can't get this.attach() or this.log() to work in `After()` or here.
   // May need newest version of cucumberjs
   console.log( `\n\n${ report }` );

package/lib/utils/reports.js

@@ -0,0 +1,147 @@
+const fs = require(`fs`);
+
+// Ours
+const session_vars = require(`./session_vars`);
+const files = require(`./files` );
+
+
+module.exports = reports = {};
+
+reports.addReportHeading = function (scope, { scenario }) {
+  /** Return printable heading based on Scenario name and tags.
+  *    Allow characters from other languages.
+  */
+  if ( session_vars.get_debug() ) { console.log(`reports.addReportHeading(): ${scenario.pickle.name} \n`); }
+
+  // Collect all tag names
+  let tag_names = [];
+  for ( let tag of scenario.pickle.tags ) {
+    tag_names.push( tag.name );
+  }
+
+  let heading = `\n---------------\n`
+    + `Scenario: ${ scenario.pickle.name }`;
+
+  if ( tag_names.length > 0 ) {
+    heading += `\nTags: ${ tag_names.join(' ') }`;
+  }
+
+  heading += `\n---------------`;
+
+  reports.addToReport( scope, {
+    type: `heading`,
+    value: heading,
+  });
+};  // Ends reports.addReportHeading()
+
+reports.addToReport = function( scope, { type, value }) {
+  /** Add an item to a specific list in a scenario's report.
+  * Create report if necessary.
+  * 
+  * The report has a key for each scenario and each of those
+  * keys should correspond to a list containing the relevant data. */
+  if ( session_vars.get_debug() ) { console.log(`reports.addToReport():\ntype: ${ type }, value: ${ value }\n`) }
+  let scenario = makeSureReportPartsExist( scope );
+  scenario.lines.push({ type, value });
+
+  const MISC_ARTIFACTS_DIR = `_alkiln-misc_artifacts`;
+  // Ensure filepath for saving local unit test logs
+  if ( !scope.paths ) {
+    // TODO: would it be useful to put in existing artifacts folder if possible.
+    // Can this be usefully abstracted?
+    scope.paths = { debug_log: `${ MISC_ARTIFACTS_DIR }/unit_tests_logs-${ files.readable_date() }.txt` };
+  }
+
+  try {
+    fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
+  } catch (err) {
+    fs.mkdirSync( MISC_ARTIFACTS_DIR );
+    fs.appendFileSync(scope.paths.debug_log, `\nReport:\ntype: ${ type }, value: ${ value }`);
+  }
+
+  return scenario;
+};  // Ends reports.addToReport()
+
+let makeSureReportPartsExist = function( scope ) {
+  /** If a part of the report needs to exist right now, but doesn't exist,
+  * create it. */
+  if ( !scope.report ) { scope.report = new Map(); }
+  if ( !scope.scenario_id ) {
+    scope.scenario_id = `_report_${ Date.now() }`;
+  }
+  if ( !scope.report.get( scope.scenario_id )) {
+    scope.report.set( scope.scenario_id, {} )
+    scope.report.get( scope.scenario_id ).status = `in progress`;
+  }
+
+  let scenario = scope.report.get( scope.scenario_id );
+  if ( !scenario.lines ) { scenario.lines = []; }
+
+  return scenario;
+};  // Ends makeSureReportPartsExist()
+
+reports.setReportScenarioStatus = function( scope, { status=`PASSED` }) {
+  /** Add final status of scenario to scenario report object - passed, failed, etc. */
+  scope.report.get( scope.scenario_id ).status = status;
+};  // Ends reports.setReportScenarioStatus()
+
+reports.convertToOriginalStoryTableRow = function( scope, { row_data }) {
+  /** Returns the original data of row into a string formatted as a story table row.
+  * Should this show the actual row data that was used instead? That could
+  * be pretty confusing to most devs. */
+  let source_row = row_data.source || row_data;
+  let original = source_row.original;
+  let var_str = trimJSON( original.var );
+  let value_str = trimJSON( original.value );
+  let trigger_str = trimJSON( original.trigger );
+  if ( trigger_str === scope.trigger_not_needed_flag ) {
+    // This text was added by ALKiln itself. It's not part of the user's data
+    trigger_str = ``;
+  }
+  return `${ ' '.repeat(6) }| ${ var_str } | ${ value_str } | ${ trigger_str } |`;
+};  // Ends reports.convertToOriginalStoryTableRows()
+
+let trimJSON = function( str=`` ) {
+  /** Turn into JSON string then trim quotes from the start and end. */
+  let json = JSON.stringify( str );
+  let trimmed = json.replace( /^"/, `` ).replace( /"$/, `` ).replace( /\\\\/g, `\\`);
+  return trimmed;
+};  // Ends trimJSON()
+
+reports.getPrintableReport = function( scope ) {
+  /** Return a string generated from test report data. */
+  let report = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
+
+  // First list failed scenarios, then non-passed scenarios, then passed scenarios
+  let failed_reports = ``;
+  let passed_reports = ``;
+  let other_reports = ``;
+  for ( let scenario of scope.report ) {
+
+    let report = reports.getPrintableScenario( scenario );
+    let [ scenario_id, data ] = scenario;
+    if ( data.status === `FAILED` ) { failed_reports += report; }
+    else if ( data.status === `PASSED` ) { passed_reports += report; }
+    else { other_reports += report; }
+  };  // ends for every scenario
+
+  if ( failed_reports !== `` ) { report += `\n\n===============================\n===============================\nFailed scenarios:\n${ failed_reports }`; }
+  if ( other_reports !== `` ) { report += `\n\n===============================\n===============================\nScenarios that did not pass:\n${ other_reports }`; }
+  if ( passed_reports !== `` ) { report += `\n\n===============================\n===============================\nPassed scenarios:\n${ passed_reports }`; }
+
+  return report;
+};  // Ends reports.getPrintableReport()
+
+reports.getPrintableScenario = function( scenario ) {
+  /** Return a string generated from scenario list of report items. */
+  let [ scenario_id, data ] = scenario;
+  let report = ``;
+  for ( let line of data.lines ) {
+    if ( line.type === `page_id` ) { report += `screen id: `; }
+    else if ( line.type === `outcome` ) { report += `\n`; }
+    else if ( line.type === `warning` ) { report += `WARNING: `; }
+    else if ( line.type === `error` ) { report += `\nERROR: ` }
+    report += `${ line.value }\n`;
+  }
+  return report;
+};  // Ends reports.getPrintableScenario()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.2.1",
+  "version": "5.3.0",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {