@suffolklitlab/alkiln 5.11.1 → 5.11.2-feat-output-2

package/.github/workflows/github_server.yml

@@ -70,11 +70,11 @@ jobs:
       - run: echo "ALKiln finished starting the isolated GitHub docassemble server"
         shell: bash
 
-      #### Developer note: You can probably leave this out
-      # Optional debugging to explore things like docker issues
-      - name: Docker debug tmate session
-        if: ${{ github.event.inputs.enable_tmate }}
-        uses: mxschmitt/action-tmate@v3
+      # #### Developer note: You can probably leave this out
+      # # Optional debugging to explore things like docker issues
+      # - name: Docker debug tmate session
+      #   if: ${{ failure() && github.event.inputs.enable_tmate == 'true' }}
+      #   uses: mxschmitt/action-tmate@v3
 
       #### Developer note: You can probably leave this out
       - name: Run ALKiln - Env vars for our own tests from server output
@@ -93,25 +93,50 @@ jobs:
 
       - name: "Run ALKiln tests"
         if: ${{ success() }}
+        id: alkiln
         #### Developer note: you'll need to replace `uses: ./`
         #### with the path to ALKiln’s repository name and branch name. For example,
         #### for version 5, use the branch name `v5`: `uses: suffolkLITLab/ALKiln@v5`
         uses: ./
-        #### Developer note: You can probably leave this out
+        #### Developer note: You can probably leave out this `env` section
         env:
-          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || 'not @a11y' }}"
+          # Internal: docassemble changes sometimes break @a11y tests. That is
+          # a da problem, not ours. We will trigger these tests manually when we
+          # want to check up on this.
+          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || '(not @a11y) and (not @error)' }}"
         with:
           #### Developer note: Required inputs. See
           #### https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#sandbox-inputs
           SERVER_URL: "${{ steps.github_server.outputs.SERVER_URL }}"
           DOCASSEMBLE_DEVELOPER_API_KEY: "${{ steps.github_server.outputs.DOCASSEMBLE_DEVELOPER_API_KEY }}"
           INSTALL_METHOD: "server"
-          #### Developer note: See https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#optional-inputs
-          #### You can probably leave this out
-          # Internal: docassemble changes sometimes break @a11y tests. That is
-          # a da problem, not ours. We will trigger these tests manually when we
-          # want to check up on this.
-          ALKILN_TAG_EXPRESSION: $ALKILN_TAG_EXPRESSION
-          #### Developer note: You can probably leave this out
-          # ALKILN_VERSION: url
+          #### Developer note: You can probably leave the rest out
+          #### To learn more, see https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#optional-inputs
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
+          ALKILN_VERSION: output
+
+      #### Developer note: Example of making an issue when tests fail, part 1
+      - name: Get ALKiln output files
+        run: |
+          echo "REPORT=${{ steps.alkiln.outputs.REPORT }}" >> $GITHUB_ENV
+          echo "UNEXPECTED_RESULTS=${{ steps.alkiln.outputs.UNEXPECTED_RESULTS }}" >> $GITHUB_ENV
+          echo "DEBUG_LOG=${{ steps.alkiln.outputs.DEBUG_LOG }}" >> $GITHUB_ENV
+
+      #### Developer note: Example of making an issue when tests fail, part 2
+      #### You can do this with any type of GitHub test
+      #### Change the name, title, body, and labels
+      #### To learn more, search online for actions-ecosystem/action-create-issue
+      - name: "If tests failed, create an issue"
+        if: ${{ failure() && '$UNEXPECTED_RESULTS' != 'false' }}
+        uses: actions-ecosystem/action-create-issue@v1
+        with:
+          github_token: ${{ secrets.github_token }}
+          title: "ALKiln tests failed"
+          body: |
+            An ALKiln test run failed. See the action at ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}.
+
+            ### Report of unexpected results:
 
+            ${{ env.UNEXPECTED_RESULTS }}
+          labels: |
+            testing

package/.github/workflows/playground.yml

@@ -55,18 +55,13 @@ jobs:
       # Place the root directory in this branch to access relative paths to action files
       - uses: actions/checkout@v4
 
-      - name: Pause between triggers
-        if: (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork)
-        shell: bash
-        run: sleep 60
-
       #### Developer note: you'll need to replace `uses: ./`
       #### with the path to ALKiln’s repository name and branch name. For example,
       #### for version 5, use the branch name `v5`: `uses: suffolkLITLab/ALKiln@v5`
       - uses: ./
         #### Developer note: You can probably leave this out
         env:
-          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || 'not @a11y' }}"
+          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || '(not @a11y) and (not @error)' }}"
         with:
           #### Developer note: Required ALKiln inputs. You can read more at
           #### https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#githubNyou-inputs
@@ -77,6 +72,6 @@ jobs:
           # Internal: docassemble changes sometimes break @a11y tests. That is
           # a da problem, not ours. We will trigger these tests manually when we
           # want to check up on this.
-          ALKILN_TAG_EXPRESSION: $ALKILN_TAG_EXPRESSION
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
           #### Developer note: You can probably leave this out
           # ALKILN_VERSION: url

package/CHANGELOG.md

@@ -43,7 +43,44 @@ Format:
 - 
 -->
 
-<!-- ## [Unreleased] -->
+## [Unreleased]
+
+### Added
+
+- New outputs from the ALKiln action - `REPORT`, `UNEXPECTED_RESULTS_PATH`, and `DEBUG_LOG_PATH`. Authors can choose to use these as they wish. For example, they can put the contents of these in issues they create from their workflow. Closes [#701](https://github.com/SuffolkLITLab/ALKiln/issues/701).
+- New file with just failure results, including cucumber results when possible. It accumulates Scenario info as the tests run. Even if a test run stops early, there will still be some results.
+
+### Changed
+
+- report.txt should accumulate over time instead of being created all at once at the end. Even if a test run stops early, there will still be some results.
+- report.txt includes the cucumber output at the bottom as well.
+
+### Fixed
+
+- Tag expression not being parsed correctly. Surrounded `ALKILN_TAG_EXPRESSION` with `"`.
+
+### Internal
+
+- Removed our own workflow's pause between triggers
+- Improved our use of ALKIN_TAG_EXPRESSION env var
+- 
+
+## [5.11.2] - 2024-05-17
+
+### Added
+
+- Added new debug log/code to reflect difference between a expected status mismatch and a cucumber status mismatch.
+
+### Changed
+
+- Changed text of error to make it more clear.
+- Changed logic of error finding to use expected status for users as well as internal tests. That needs discussion.
+- Refactored code in that section.
+- Wait till the end to throw some errors so the user can get as much info as possible about the failures.
+
+### Internal
+
+- Exclude another file from code checking (cucumber-report.txt)
 
 ## [5.11.1] - 2024-04-06
 

package/action.yml

@@ -34,6 +34,18 @@ inputs:
     required: false
     default: "0.0.17"
 
+
+outputs:
+  REPORT:
+    description: 'report.txt contents or "false" if the file is missing'
+    value: ${{ steps.files.outputs.REPORT }}
+  UNEXPECTED_RESULTS:
+    description: 'unexpected_results.txt contents or "false" if the file is missing'
+    value: ${{ steps.files.outputs.UNEXPECTED_RESULTS }}
+  DEBUG_LOG:
+    description: 'debug_log.txt contents or "false" if the file is missing'
+    value: ${{ steps.files.outputs.DEBUG_LOG }}
+
 runs:
   using: "composite"
   steps:
@@ -121,7 +133,7 @@ runs:
       env:
         ALKILN_TAG_EXPRESSION: ${{ inputs.ALKILN_TAG_EXPRESSION || github.event.inputs.tags && format('{0}', github.event.inputs.tags) }}
       shell: bash
-      run: alkiln-run $ALKILN_TAG_EXPRESSION
+      run: alkiln-run "$ALKILN_TAG_EXPRESSION"
     
     # on playground, delete project
     - name: "💡 ALK0032 INFO: Delete the Project from the Playground"
@@ -137,6 +149,43 @@ runs:
         name: ${{ env.ARTIFACT_NAME }}
         path: ./alkiln-*
 
+    # Download artifacts folder internally to create action output
+    - name: "💡 ALK0182 INFO: Get uploaded artifacts folder"
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+
+    # Create output
+    - shell: bash
+      id: files
+      run: |
+        # 💡 ALK0183 INFO: Give file contents or 'false' to interview authors
+        REPORT_PATH="${ARTIFACT_NAME}/report.txt"
+        UNEXPECTED_RESULTS_PATH="${ARTIFACT_NAME}/unexpected_results.txt"
+        DEBUG_LOG_PATH="${ARTIFACT_NAME}/debug_log.txt"
+
+        # Get contents of files if they exist
+        if [ -f "$REPORT_PATH" ]; then
+          REPORT=$(cat "$REPORT_PATH")
+        else
+          REPORT="report.txt does not exist"
+        fi
+        if [ -f "$UNEXPECTED_RESULTS_PATH" ]; then
+          UNEXPECTED_RESULTS_PATH=$(cat "$UNEXPECTED_RESULTS_PATH")
+        else
+          UNEXPECTED_RESULTS_PATH="unexpected_results.txt does not exist"
+        fi
+        if [ -f "$DEBUG_LOG_PATH" ]; then
+          DEBUG_LOG_PATH=$(cat "$DEBUG_LOG_PATH")
+        else
+          DEBUG_LOG_PATH="debug_log.txt does not exist"
+        fi
+
+        # Create the outputs
+        echo "REPORT=$REPORT" >> "$GITHUB_OUTPUT"
+        echo "UNEXPECTED_RESULTS=$UNEXPECTED_RESULTS" >> "$GITHUB_OUTPUT"
+        echo "DEBUG_LOG=$DEBUG_LOG" >> "$GITHUB_OUTPUT"
+
     - shell: bash
       run: |
         echo "💡 ALK0034 INFO: ALkiln finished running tests"

package/lib/run_cucumber.js

@@ -3,9 +3,11 @@
 const fs = require('fs');
 const cucumber_api = require('@cucumber/cucumber/api');
 
-const log = require('./utils/log');
+const log = require('./utils/log');  // These always log in the console
 const session_vars = require('./utils/session_vars');
 
+const SUMMARY_FILENAME = `cucumber-report.txt`;
+
 function build_config( command_args ) {
   // Turn all the paths to the sources folders into paths to the feature files
   let sources_paths = session_vars.get_sources_paths();
@@ -17,13 +19,15 @@ function build_config( command_args ) {
   const config = {
     publishQuiet: true,
     require: `./lib/index.js`,
-    format: [`progress`, `summary:cucumber-report.txt`],
+    format: [`progress`, `summary:${ SUMMARY_FILENAME }`],
     paths: feature_paths,
     retry: 1,
   }
 
   // Normalize tag expressions, if there are any
-  let tags = command_args[`_`];
+
+  // Joining and splitting handles tags surrounded with `""` or those without `""`
+  let tags = command_args[`_`].join(` `).split(` `);
   if (tags.length > 0) {
     let joined_tags = '';
     // Cucumber expects tags in boolean expressions, like (@a0 or @a1) and @random.
@@ -105,19 +109,20 @@ function build_config( command_args ) {
 
   const { success } = await cucumber_api.runCucumber({...runConfiguration, support});
   if ( success ) {
-    log.info({ type: `run`, code: `ALK0054`, pre: `Cucumber executed correctly` });
+    log.success({ type: `run`, code: `ALK0054`, pre: `Tests succeeded!` });
   } else {
-    log.error({ type: `run`, code: `ALK0055`, pre: `Cucumber ran into an error` });
+    log.error({ type: `run`, code: `ALK0055`, pre: `One or more tests got unexpected results` });
   }
-  const exitCode = success ? 0 : 1;
+  // Code is our custom code for proper completion with failed tests
+  const exitCode = success ? 0 : 2;
   process.exitCode = exitCode;
 
   // This is hardcoded information from `provide_config` above.
   // This code isn't used for anything other than detecting and removing color codes.
-  let summary_file_name = 'cucumber-report.txt';
+  let summary_file_name = SUMMARY_FILENAME;
   fs.readFile(summary_file_name, 'utf8', (err, text) => {
     if (err) {
-      log.error({ type: `run`, code: `ALK0056`, pre: `cucumber report`, data: err });
+      log.error({ type: `internal run`, code: `ALK0056`, pre: `Failed to read cucumber report`, data: err });
       return;
     }
 
@@ -128,5 +133,16 @@ function build_config( command_args ) {
     // Put it in the correct file in the correct folder
     let artifacts_path = session_vars.get_artifacts_path_name();
     fs.appendFileSync( `${ artifacts_path }/${ log.debug_log_file }`, `\n\n${text}`);
+    try {
+      // Add to a unexpected results file (that should exist already) if needed
+      if ( exitCode === 2 ) {
+        fs.appendFileSync(
+          `${ artifacts_path }/${ log.unexpected_filename }`,
+          `\n\n${ text }`
+        );
+      }
+    } catch ( error ) {
+      log.error({ type: `internal run`, code: `ALK0181`, pre: `This is sometimes fine. ALKiln was unable to read ${ log.unexpected_filename }, which might just mean there was an unrelated failure above.`, data: error });
+    }
   });
 })();

package/lib/scope.js

@@ -3228,8 +3228,7 @@ module.exports = {
     * 
     * @param expected {array} Arr of strs that should appear in the report.
     */
-    let current_report_obj = scope.report.get( scope.scenario_id );
-    let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+    let scenario = scope.report.get( scope.scenario_id );
     let report = reports.getPrintableScenario( scenario );
     let all_are_included = true;
     for ( let one_expectation of expected ) {
@@ -3248,8 +3247,7 @@ module.exports = {
     * 
     * @param not_expected {array} Arr of strs that should not appear in the report.
     */
-    let current_report_obj = scope.report.get( scope.scenario_id );
-    let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+    let scenario = scope.report.get( scope.scenario_id );
     let report = reports.getPrintableScenario( scenario );
     let none_included = true;
     for ( let one_expectation of not_expected ) {

package/lib/steps.js

@@ -101,8 +101,11 @@ BeforeAll(async () => {
   session_vars.save_artifacts_path_name( scope.paths.artifacts );
 
   scope.paths.report = `${ scope.paths.artifacts }/report.txt`;
+  scope.paths.failures = `${ scope.paths.artifacts }/${ log.unexpected_filename }`;
   scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_file }`;
   fs.writeFileSync( scope.paths.debug_log, "" );
+
+  reports.create( scope );
 });
 
 Before(async (scenario) => {
@@ -1212,7 +1215,7 @@ After(async function(scenario) {
   }
 
   // Record status before internal report tests can interfere with values
-  reports.setReportScenarioStatus( scope, { status: changeable_test_status });
+  reports.saveFinalScenarioData( scope, { status: changeable_test_status });
 
   if (changeable_test_status !== `PASSED`) {
     log.debug({
@@ -1256,13 +1259,13 @@ After(async function(scenario) {
   }  // ends if not passed
 
   // Add report text to Scenario folder after everything has been added to the report
-  let current_report_obj = scope.report.get( scope.scenario_id );
-  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+  let scenario_report_obj = scope.report.get( scope.scenario_id );
   let report = reports.getPrintableScenario( scenario_report_obj );
   // Save the report as in the Scenario folder
   fs.writeFileSync( scope.paths.scenario_report, report );
+  // TODO: Save any cucumber failure message for the Scenario here (instead of waiting till run_cucumber.js)
 
-  // ---------------- Check internal test results of this framework ----------------
+  // ---------------- Check internal test results ----------------
 
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
     let report_includes_all_expected_strings = await scope.reportIncludesAllExpected(
@@ -1284,74 +1287,120 @@ After(async function(scenario) {
     scope.expected_not_in_report = null;
   }
 
-  // The default is 'passed', which won't need to get in here
-  if ( scope.expected_status ) {
-    // When internal tests should fail (to test our failure conditions), passing _is_ failing
-    expect( changeable_test_status ).to.equal( scope.expected_status );
-    // If the status was as expected, pass this Scenario
-    changeable_test_status = `PASSED`;
-    scope.expected_status = null;  // reset for next Scenario
+  // ---------------- Ends internal results check ----------------
+
+  if ( scope.expected_status === null ) {
+    scope.expected_status = `PASSED`;  // default expected status
   }
 
-  // ---------------- Ends check internal test results of this framework ----------------
+  let expected_status_was_met = true;
+  let expected_status_err_msg = ``;
+  // Avoid manipulating `UNDEFINED` and such
+  if ( changeable_test_status === `PASSED`
+       || changeable_test_status === `FAILED` ) {
+
+    if ( changeable_test_status === scope.expected_status ) {
+      // If the status was as expected, pass this Scenario. E.g. an internal
+      // test was supposed to fail and it did. Or a test was suppose to pass
+      // and it passed.
+      changeable_test_status = `PASSED`;
+    } else {
+      // If a test that was supposed to fail passed (e.g. internal
+      // test), it failed. If is was supposed to pass and failed, it also
+      // failed. E.g. a11y failures let the Scenario pass initially, but
+      // should cause a failure here.
+      changeable_test_status = `FAILED`;
+      // Flag this failure to trigger an error later, so cucumber shows
+      // it as failed too.
+      expected_status_was_met = false;
+      // Keep it general. Users may see this message, e.g. for a11y tests.
+      let msg = `Test failed. Expected status: ${scope.expected_status}. `
+          + `ALKiln's assessed status: ${ changeable_test_status }. `
+          + `See the report for details.`;
+      let code = `ALK0099`;
+      log.debug({ type: `error`, code: code, pre: msg, })
+      expected_status_err_msg = `🤕 ${ code } ERROR: ${ msg }`
+    }
+  }
+  scope.expected_status = null;  // reset for next Scenario
+
+  let need_to_force_failed_status = false;
+  let force_failure_msg = `ALKiln assessed that the test failed. See the report for details.`;
+  let force_failure_code = `ALK0180`;
+  if ( changeable_test_status !== `FAILED` ) {
+    // Other than `FAILED`, it's fine to set the scenario status to make
+    // sure it matches up with ALKiln's assessed status.
+    // WARNING: This may be impossible in future cucumber versions
+    scenario.result.status = changeable_test_status;
+    // TODO: Should we add (an info) debug log here too?
 
+  // If assessed status is FAILED, but result.status is something else
+  } else if ( scenario.result.status !== `FAILED` ) {
+    // Make sure that tests which ALKiln has calculated as failed
+    // cause a cucumber error later. Just changing the status to `FAILED`
+    // causes unhelpful cucumber errors. TODO: Add to internal docs.
+    log.debug({
+      type: `error`,
+      code: force_failure_code,
+      pre: force_failure_msg
+    });
+    // TODO: Look into setDefinitionFunctionWrapper to catch errors
+    // of failing tests that should indeed be failing:
+    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
+    need_to_force_failed_status = true;
+  }
+
+  let signout_succeeded = true;
+  let signout_error_msg = ``;
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
     // Catch a possible reload error.
-
-    // TODO: implement and use scope.handle_possible_timeout()
     try {
       // puppeteer will ensure proper timeout.
       await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});  
 
     } catch ( error ) {
+      signout_succeeded = false;
 
       if ( error.name === `TimeoutError` ) {
         // TODO: Should a more specific message about signing out get into the report no matter what?
         // If so, what should the message be in order to not get confused with these other messages?
         let non_reload_report_msg = `It took too long to sign out in preparation for the next test.`;
-        await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
-          code: `ALK0097`,
-          message: non_reload_report_msg,
-        }, error });
+        let code = `ALK0097`;
+        try {
+          await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
+            code: code,
+            message: non_reload_report_msg,
+          }, error });
+        } catch {
+          signout_error_msg = `🤕 ${ code } ERROR: ${ non_reload_report_msg }`;
+        }
       } else {
         // Throw any non-timeout error
-        let err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
-        reports.addToReport( scope, { type: `error`, code: `ALK0098`, value: err_msg });
-        throw error;
+        let other_err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
+        let code = `ALK0098`;
+        reports.addToReport( scope, { type: `error`, code: code, value: other_err_msg });
+        signout_error_msg = `🤕 ${ code } ERROR: ${ other_err_msg }`;
       }  // ends if error is timeout error
 
     }  // ends try/catch
 
   }  // ends if scope.page
 
-  // INTERNAL WARNING: TODO: A console.log here prevents the try/catch from
-  // throwing an error. Why is that? Does it somehow make sure that scope.page
-  // gets destroyed?
-
-  // console.log('This stops try/catch from throwing an error in some cases.');
-
-  // More internal testing stuff
-  if ( changeable_test_status === `FAILED` ) {
-    // If the result object status wasn't failure and we still want
-    // the tests to fail, as with a11y tests, we can't mess with the
-    // cucumber result.status object because it will cause unwanted
-    // errors. We have to throw an error here instead.
-    if ( scenario.result.status !== `FAILED` ) {
-      let msg = `The test unexpectedly didn't fail. Scenario status: ${scenario.result.status}.`
-      reports.addToReport( scope, { type: `error`, code: `ALK0099`, value: msg });
-      throw new Error(msg);
-    }
-  } else {
-    // Some internal test failures should be ok
-    // TODO: Look into setDefinitionFunctionWrapper to catch errors
-    // of failing tests that should indeed be failing:
-    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
-    scenario.result.status = changeable_test_status;
+  let errors = []
+  if ( !expected_status_was_met ) { errors.push( expected_status_err_msg ); }
+  if ( !signout_succeeded ) { errors.push( signout_error_msg ); }
+  if ( !expected_status_was_met || !signout_succeeded ) {
+    throw new Error( `${errors.join( errors, '\n')}` );
   }
-  // Ends more internal testing stuff
 
-  // WARNING: TODO: console.log can prevent a sign-out error. What race condition is going on?
+  // Theoretically, we should never get here:
+  // If everything else worked, if we need to force an internal test failure
+  if ( need_to_force_failed_status ) {
+    throw new Error( `🤕 ${force_failure_code} unreachable ERROR: ${force_failure_msg}` );
+  }
+
+  // Find the race condition.
   log.debug({ type: `cucumber`, code: `ALK0100`, verbose: true, pre: `Scenario AfterStep() message`, data: scenario.result.message });
 
 });
@@ -1365,9 +1414,6 @@ AfterAll(async function() {
   // May need newest version of cucumberjs
   console.log( `\n\n${ report }` );
 
-  // Save/download the report as an artifact
-  fs.writeFileSync( scope.paths.report, report );
-
   // If there is a browser open, then close it
   if (scope.browser) {
     await scope.browser.close();

package/lib/utils/log.js

@@ -2,6 +2,9 @@ const fs = require('fs');
 const session_vars = require( './session_vars' );
 
 // ? 🌈 💡 🔎 🤕 🐞
+// Alternatives: 🩹 ❤️‍🩹 🚑
+// TODO: return full message so it can be passed to manually
+// thrown errors if needed.
 
 let log = {};
 module.exports = log;
@@ -89,4 +92,5 @@ log.add_to_debug_log = function(value) {
   fs.appendFileSync(log.debug_log_file, value);
 };
 
-log.debug_log_file = 'debug_log.txt'
+log.debug_log_file = `debug_log.txt`;
+log.unexpected_filename = `unexpected_results.txt`;

package/lib/utils/reports.js

@@ -7,6 +7,13 @@ const files = require(`./files` );
 
 module.exports = reports = {};
 
+reports.title = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
+reports.failures = [ `\n\n===============================\n===============================\n🤕 Failed scenarios:\n` ];
+reports.successes = [ `\n\n===============================\n===============================\n🌈 Passed scenarios:\n` ];
+// reports.warnings = [ `\n\n===============================\n===============================\n🔎 Scenarios to check:\n` ];
+// reports.undefineds = [ `\n\n===============================\n===============================\n[langs icon] Scenarios with typos:\n` ];
+reports.others = [ `\n\n===============================\n===============================\n🔎 Other scenarios that did not pass:\n` ];
+
 reports.addReportHeading = function (scope, { scenario }) {
   /** Return printable heading based on Scenario name and tags.
   *    Allow characters from other languages.
@@ -46,6 +53,9 @@ reports.addToReport = function( scope, { type, code, value }) {
   * @params code {string} (Optional) Error or warning code. Default
   *    is unspecified error or warning.
   * */
+  // TODO: return full message so it can be passed to manually
+  // thrown errors if needed.
+
   // Debugging logs
   let debug_line = getDebugLine({ type, code, value });
   if ( session_vars.get_debug() ) {
@@ -68,21 +78,30 @@ reports.addToReport = function( scope, { type, code, value }) {
   }
 
   // Report stuff
-  let scenario = makeSureReportPartsExist( scope );
+  let scenario = reports.makeSureScenarioExists( scope );
   scenario.lines.push({ type, code, value });
 
   return scenario;
 };  // Ends reports.addToReport()
 
-let makeSureReportPartsExist = function( scope ) {
+reports.create = function( scope ) {
+  reports.makeSureReportExists( scope );
+}
+
+reports.makeSureReportExists = function( scope ) {
   /** If a part of the report needs to exist right now, but doesn't exist,
   * create it. */
   if ( !scope.report ) { scope.report = new Map(); }
+  return this;
+};  // Ends reports.makeSureReportExists()
+
+reports.makeSureScenarioExists = function ( scope ) {
+  reports.makeSureReportExists( scope );
   if ( !scope.scenario_id ) {
     scope.scenario_id = `_report_${ Date.now() }`;
   }
   if ( !scope.report.get( scope.scenario_id )) {
-    scope.report.set( scope.scenario_id, {} )
+    scope.report.set( scope.scenario_id, {} );
     scope.report.get( scope.scenario_id ).status = `in progress`;
   }
 
@@ -90,11 +109,36 @@ let makeSureReportPartsExist = function( scope ) {
   if ( !scenario.lines ) { scenario.lines = []; }
 
   return scenario;
-};  // Ends makeSureReportPartsExist()
+};  // Ends reports.makeSureScenarioExists()
+
+reports.saveFinalScenarioData = function( scope, { status=`in progress` }) {
+  this.setReportScenarioStatus( scope, { status });
+  let scenario = scope.report.get( scope.scenario_id );
+  let report = this.getPrintableScenario( scenario );
+  if ( scenario.status === `FAILED` ) { this.failures.push( report ); }
+  else if ( scenario.status === `PASSED` ) { this.successes.push( report ); }
+  // else if ( scenario.status === `WARNING` ) { passed_reports += report; this.warnings.push( report ); }
+  // else if ( scenario.status === `UNDEFINED` ) { passed_reports += report; this.undefineds.push( report ); }
+  else { this.others.push( report ); }
+
+  // Save to failures.txt (worry about optimization later)
+  if ( this.failures.length > 1 || this.others.length > 1 ) {
+    fs.writeFileSync(
+      scope.paths.failures,
+      this.title + this.getPrintableUnexpected( scope )
+    );
+  }
+  // Save to reports.txt (worry about optimization later)
+  fs.writeFileSync( scope.paths.report, this.getPrintableReport( scope ) );
+
+  return scenario;
+};  // Ends reports.saveFinalScenarioData()
+
 
 reports.setReportScenarioStatus = function( scope, { status=`PASSED` }) {
   /** Add final status of scenario to scenario report object - passed, failed, etc. */
   scope.report.get( scope.scenario_id ).status = status;
+  return scope.report.get( scope.scenario_id );
 };  // Ends reports.setReportScenarioStatus()
 
 reports.convertToOriginalStoryTableRow = function( scope, { row_data }) {
@@ -121,34 +165,38 @@ let trimJSON = function( str=`` ) {
 };  // Ends trimJSON()
 
 reports.getPrintableReport = function( scope ) {
-  /** Return a string generated from test report data. */
-  let report = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
-
-  // First list failed scenarios, then non-passed scenarios, then passed scenarios
-  let failed_reports = ``;
-  let passed_reports = ``;
-  let other_reports = ``;
-  for ( let scenario of scope.report ) {
-
-    let report = reports.getPrintableScenario( scenario );
-    let [ scenario_id, data ] = scenario;
-    if ( data.status === `FAILED` ) { failed_reports += report; }
-    else if ( data.status === `PASSED` ) { passed_reports += report; }
-    else { other_reports += report; }
-  };  // ends for every scenario
-
-  if ( failed_reports !== `` ) { report += `\n\n===============================\n===============================\n🤕 Failed scenarios:\n${ failed_reports }`; }
-  if ( other_reports !== `` ) { report += `\n\n===============================\n===============================\n🔎 Scenarios that did not pass:\n${ other_reports }`; }
-  if ( passed_reports !== `` ) { report += `\n\n===============================\n===============================\n🌈 Passed scenarios:\n${ passed_reports }`; }
+  /** Return a string generated from test report data.
+  *
+  * First list failed scenarios, then non-passed scenarios, then passed scenarios. */
+  let report = this.getTitle( scope );
+
+  if ( this.failures.length > 1 ) { report += this.failures.join( `` ); }
+  if ( this.successes.length > 1 ) { report += this.successes.join( `` ); }
+  // if ( this.warnings.length > 1 ) {}
+  // if ( this.undefineds.length > 1 ) {}
+  if ( this.others.length > 1 ) { report += this.others.join( `` ); }
 
   return report;
 };  // Ends reports.getPrintableReport()
 
+reports.getTitle = function( scope ) { return this.title; }
+
+reports.getPrintableUnexpected = function( scope ) {
+  // Note about `> 1`: The first line of the report section always exists
+  let output = ``;
+  if ( this.failures.length > 1 ) {
+    output += this.failures.join(``);
+  }
+  if ( this.others.length > 1 ) {
+    output += this.others.join(``);
+  }
+  return output;
+};  // Ends reports.getPrintableUnexpected()
+
 reports.getPrintableScenario = function( scenario ) {
   /** Return a string generated from scenario list of report items. */
-  let [ scenario_id, data ] = scenario;
   let report = ``;
-  for ( let line of data.lines ) {
+  for ( let line of scenario.lines ) {
     let { type, code, value } = line;
     report += `${ getLineText({ type, code, value }) }\n`;
   }

package/lib/utils/session_vars.js

@@ -120,7 +120,7 @@ session_vars.save_project_name = function ( project_name ) {
   // Warn local developers if file already exists
   if ( fs.existsSync( runtime_config_path ) ) {
     let old_project_name = session_vars.get_project_name();
-    let session_project_already_exists_msg = `🔎 ALK0035 WARNING: `
+    let session_project_already_exists_msg = `🔎 ALK0035 setup WARNING: `
       + `This session had a project name already: ${ old_project_name }. `
       + `A new project has now been created called ${ project_name }. `
       + `Remember to delete ${ old_project_name }`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.11.1",
+  "version": "5.11.2-feat-output-2",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {

package/tests/log_codes/check_codes.sh

@@ -16,7 +16,7 @@ fi
 # The grepped files will include the list of all the codes that have been
 # removed and are no longer used. Syntax used works with GitHub cli ([[:digit:]])
 # https://stackoverflow.com/a/6901221
-lines=$(grep -roh --exclude="tests/log_codes/check_codes.sh" --exclude="debug_log.txt" --exclude-dir="node_modules" --exclude-dir='alkiln-*' --exclude-dir='_alkiln*' 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' "$directory")
+lines=$(grep -roh --exclude="tests/log_codes/check_codes.sh" --exclude="debug_log.txt" --exclude="cucumber-report.txt" --exclude-dir="node_modules" --exclude-dir='alkiln-*' --exclude-dir='_alkiln*' 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' "$directory")
 sorted=$(echo "$lines" | sort -n)