@suffolklitlab/alkiln 5.11.2 → 5.12.0

package/.github/workflows/github_server.yml

@@ -23,8 +23,8 @@ on:
 jobs:
 
   github-server-tests:
-    #### Developer note: If under `on` (above) you have both `push` and
-    #### `pull_request`, this avoids running double tests when someone
+    #### Developer note: If you have both `push` and `pull_request` in the
+    #### `on` section (above), this avoids running double tests when someone
     #### on the team makes a pull request
     # 1: Trigger job on push or dispatch from this repository
     # 2: Trigger job on pull request from a fork
@@ -71,10 +71,10 @@ jobs:
         shell: bash
 
       #### Developer note: You can probably leave this out
-      # Optional debugging to explore things like docker issues
-      - name: Docker debug tmate session
-        if: ${{ github.event.inputs.enable_tmate }}
-        uses: mxschmitt/action-tmate@v3
+      # # Optional debugging to explore things like docker issues
+      # - name: Docker debug tmate session
+      #   if: ${{ failure() && github.event.inputs.enable_tmate == 'true' }}
+      #   uses: mxschmitt/action-tmate@v3
 
       #### Developer note: You can probably leave this out
       - name: Run ALKiln - Env vars for our own tests from server output
@@ -93,25 +93,49 @@ jobs:
 
       - name: "Run ALKiln tests"
         if: ${{ success() }}
+        id: alkiln
         #### Developer note: you'll need to replace `uses: ./`
         #### with the path to ALKiln’s repository name and branch name. For example,
         #### for version 5, use the branch name `v5`: `uses: suffolkLITLab/ALKiln@v5`
         uses: ./
-        #### Developer note: You can probably leave this out
+        #### Developer note: You can probably leave out this `env` section
         env:
-          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || 'not @a11y' }}"
+          # Internal: docassemble changes sometimes break @a11y tests. That is
+          # a da problem, not ours. We will trigger these tests manually when we
+          # want to check up on this.
+          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || '(not @a11y) and (not @error)' }}"
         with:
           #### Developer note: Required inputs. See
           #### https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#sandbox-inputs
           SERVER_URL: "${{ steps.github_server.outputs.SERVER_URL }}"
           DOCASSEMBLE_DEVELOPER_API_KEY: "${{ steps.github_server.outputs.DOCASSEMBLE_DEVELOPER_API_KEY }}"
           INSTALL_METHOD: "server"
-          #### Developer note: See https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#optional-inputs
-          #### You can probably leave this out
-          # Internal: docassemble changes sometimes break @a11y tests. That is
-          # a da problem, not ours. We will trigger these tests manually when we
-          # want to check up on this.
-          ALKILN_TAG_EXPRESSION: $ALKILN_TAG_EXPRESSION
-          #### Developer note: You can probably leave this out
-          # ALKILN_VERSION: url
+          #### Developer note: You can probably leave the rest out
+          #### To learn more, see https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#optional-inputs
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
+          ALKILN_VERSION: output
+
+      #### Developer note: Example of making an issue when tests fail
+      #### that includes the text of the failure output file
+      #### See https://cli.github.com/manual/gh_issue_create for more details
+      - shell: bash
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PATH_TO_REPORT_FILE: "${{ steps.alkiln.outputs.PATH_TO_REPORT_FILE }}"
+          PATH_TO_UNEXPECTED_RESULTS_FILE: "${{ steps.alkiln.outputs.PATH_TO_UNEXPECTED_RESULTS_FILE }}"
+          PATH_TO_DEBUG_LOG_FILE: "${{ steps.alkiln.outputs.PATH_TO_DEBUG_LOG_FILE }}"
+        run: |
+          # Make an issue with file contents or 'no file' message as part of the body
+
+          if [ -f "${{ env.PATH_TO_UNEXPECTED_RESULTS_FILE }}" ]; then
+            UNEXPECTED_2=$(cat "${{ env.PATH_TO_UNEXPECTED_RESULTS_FILE }}")
+          else
+            UNEXPECTED_2="unexpected_results.txt does not exist. Something other than the interview tests may have failed."
+          fi
+
+          gh issue create --body "An ALKiln test run failed. See the action at ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}.
+
+          ### Results:
 
+          $UNEXPECTED_2" --repo "$GITHUB_REPOSITORY" --label "testing" --title "ALKiln tests failed"

package/.github/workflows/playground.yml

@@ -13,8 +13,8 @@ on:
 jobs:
 
   puppeteer-tests:
-    #### Developer note: If under `on` (above) you have both `push` and
-    #### `pull_request`, this avoids running double tests when someone
+    #### Developer note: If you have both `push` and `pull_request` in the
+    #### `on` section (above), this avoids running double tests when someone
     #### editing the repository makes a pull request
     # 1: Trigger job on push or dispatch from this repository
     # 2: Trigger job on pull request from a fork
@@ -55,18 +55,13 @@ jobs:
       # Place the root directory in this branch to access relative paths to action files
       - uses: actions/checkout@v4
 
-      - name: Pause between triggers
-        if: (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork)
-        shell: bash
-        run: sleep 60
-
       #### Developer note: you'll need to replace `uses: ./`
       #### with the path to ALKiln’s repository name and branch name. For example,
       #### for version 5, use the branch name `v5`: `uses: suffolkLITLab/ALKiln@v5`
       - uses: ./
         #### Developer note: You can probably leave this out
         env:
-          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || 'not @a11y' }}"
+          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) || '(not @a11y) and (not @error)' }}"
         with:
           #### Developer note: Required ALKiln inputs. You can read more at
           #### https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/alkiln/writing/#githubNyou-inputs
@@ -77,6 +72,6 @@ jobs:
           # Internal: docassemble changes sometimes break @a11y tests. That is
           # a da problem, not ours. We will trigger these tests manually when we
           # want to check up on this.
-          ALKILN_TAG_EXPRESSION: $ALKILN_TAG_EXPRESSION
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
           #### Developer note: You can probably leave this out
-          # ALKILN_VERSION: url
+          ALKILN_VERSION: output

package/CHANGELOG.md

@@ -45,6 +45,31 @@ Format:
 
 <!-- ## [Unreleased] -->
 
+## [5.12.0] - 2024-06-06
+
+### Added
+
+- New file with just failure results, including cucumber results when possible. It accumulates Scenario info as the tests run. Even if a test run stops early, there will still be some results.
+- New output from ALKiln action - `PATH_TO_ARTIFACTS`. The value is `""` or the path to the folder that contains the output of the test run, if the test run actually happened. Sometimes this folder doesn't get created. For example, if there was a typo in one or more `.feature` file. For example, `"alkiln-2024-06-03 at 02h29m44sUTC"`.
+- New outputs from the ALKiln action - `PATH_TO_REPORT_FILE`, `PATH_TO_UNEXPECTED_RESULTS_FILE`, and `PATH_TO_DEBUG_LOG_FILE`.  For example, `"alkiln-2024-06-03 at 02h29m44sUTC/report.txt"`. Authors can choose to use these as they wish. For example, they can put the contents of these files in issues they create from their workflow. Closes [#701](https://github.com/SuffolkLITLab/ALKiln/issues/701).
+- Added cucumber output to report.txt.
+- Added more explanation for the logs that appear when test runs error in a way that prevents debug_log.txt and such files from being created. We will watch for feedback about the error data itself. It may still be too confusing.
+
+### Changed
+
+- report.txt accumulates over time instead of being created all at once at the end. Even if a test run stops early, there will still be some results.
+
+### Fixed
+
+- Tag expression not being parsed correctly. Surrounded `ALKILN_TAG_EXPRESSION` with `"`.
+
+### Internal
+
+- Download ALKiln's artifacts back into the running job.
+- Removed our own workflow's pause between triggers.
+- Improved our use of ALKIN_TAG_EXPRESSION env var.
+- Exclude tests that we are aware fail. We will test those manually.
+
 ## [5.11.2] - 2024-05-17
 
 ### Added

package/action.yml

@@ -34,6 +34,20 @@ inputs:
     required: false
     default: "0.0.17"
 
+outputs:
+  PATH_TO_ARTIFACTS:
+    description: 'An empty string ("") or the path to the artifacts folder that contains the test run output. Remember to check for the existence of the folder before using it.'
+    value: ${{ steps.alkiln_outputs.outputs.PATH_TO_ARTIFACTS }}
+  PATH_TO_REPORT_FILE:
+    description: 'An empty string ("") or the path to report.txt. Remember to check for the existence of the file before using it.'
+    value: ${{ steps.alkiln_outputs.outputs.PATH_TO_REPORT_FILE }}
+  PATH_TO_UNEXPECTED_RESULTS_FILE:
+    description: 'An empty string ("") or the path to unexpected_results.txt. Remember to check for the existence of the file before using it.'
+    value: ${{ steps.alkiln_outputs.outputs.PATH_TO_UNEXPECTED_RESULTS_FILE }}
+  PATH_TO_DEBUG_LOG_FILE:
+    description: 'An empty string ("") or the path to debug_log.txt. Remember to check for the existence of the file before using it.'
+    value: ${{ steps.alkiln_outputs.outputs.PATH_TO_DEBUG_LOG_FILE }}
+
 runs:
   using: "composite"
   steps:
@@ -102,6 +116,7 @@ runs:
       if: ${{ inputs.INSTALL_METHOD == 'playground' && failure() }}
       shell: bash
       run: |
+        # 😞 ALK0028 ERROR: Unable to create a Project
         echo -e "\n\n====\n😞 ALK0028 ERROR: Unable to create a Project on your server's testing account or pull your package into it. Check the messages above this line.\n\n"
 
     # Server installations - find folders and save session vars
@@ -121,7 +136,7 @@ runs:
       env:
         ALKILN_TAG_EXPRESSION: ${{ inputs.ALKILN_TAG_EXPRESSION || github.event.inputs.tags && format('{0}', github.event.inputs.tags) }}
       shell: bash
-      run: alkiln-run $ALKILN_TAG_EXPRESSION
+      run: alkiln-run "$ALKILN_TAG_EXPRESSION"
     
     # on playground, delete project
     - name: "💡 ALK0032 INFO: Delete the Project from the Playground"
@@ -137,6 +152,64 @@ runs:
         name: ${{ env.ARTIFACT_NAME }}
         path: ./alkiln-*
 
+    # Download artifacts folder internally to create action output
+    - name: "💡 ALK0183 INFO: Get uploaded artifacts folder"
+      if: ${{ always() }}
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+
+    - name: "💡 ALK0184 INFO: Prepare ALKiln output file paths"
+      if: ${{ always() }}
+      id: alkiln_outputs
+      shell: bash
+      run: |
+        # 💡 ALK0184 INFO: Prepare ALKiln output file paths
+
+        FOLDER=$(ls -d */ | grep -E '^alkiln-*' || true)
+
+        echo "Test results artifacts folder is $FOLDER"
+
+        if [[ "$FOLDER" = "" ]]; then
+          echo "Artifacts folder is missing. The path is an empty string: '$FOLDER'"
+
+        else
+          REPORT_PATH="${FOLDER}report.txt"
+          UNEXPECTED_RESULTS_PATH="${FOLDER}unexpected_results.txt"
+          DEBUG_LOG_PATH="${FOLDER}debug_log.txt"
+
+          if [ -f "$REPORT_PATH" ]; then
+            echo "$REPORT_PATH exists"
+          else
+            echo "$REPORT_PATH is missing"
+            REPORT_PATH=""
+          fi
+
+          if [ -f "$UNEXPECTED_RESULTS_PATH" ]; then
+            echo "$UNEXPECTED_RESULTS_PATH exists"
+          else
+            echo "$UNEXPECTED_RESULTS_PATH is missing"
+            UNEXPECTED_RESULTS_PATH=""
+          fi
+
+          if [ -f "$DEBUG_LOG_PATH" ]; then
+            echo "$DEBUG_LOG_PATH exists"
+          else
+            echo "$DEBUG_LOG_PATH is missing"
+            DEBUG_LOG_PATH=""
+          fi
+
+        fi
+
+        echo "PATH_TO_ARTIFACTS=$FOLDER" >> "$GITHUB_OUTPUT"
+        echo "PATH_TO_REPORT_FILE=$REPORT_PATH" >> "$GITHUB_OUTPUT"
+        echo "PATH_TO_UNEXPECTED_RESULTS_FILE=$UNEXPECTED_RESULTS_PATH" >> "$GITHUB_OUTPUT"
+        echo "PATH_TO_DEBUG_LOG_FILE=$DEBUG_LOG_PATH" >> "$GITHUB_OUTPUT"
+
+    # Avoid creating output from text in files
+    # Cannot properly create outputs with multi-line text. Collapses onto one line.
+
     - shell: bash
+      if: ${{ always() }}
       run: |
         echo "💡 ALK0034 INFO: ALkiln finished running tests"

package/action_for_github_server/action.yml

@@ -109,12 +109,14 @@ runs:
         if: ${{ env.OUTPUT_DOCKER == 'false' }}
         shell: bash
         run: |
-          echo '===\n💡 ALK0007 INFO: ALKiln will now hide docker output to avoid possibly showing details about your config. If you want to get output to see more of what is going on, use "with:" and set the input "SHOW_DOCKER_OUTPUT" to "true". This output will only be visible to those who can already see the action output, like people with admin or write permissions.\n==='
+          # 💡 ALK0007 INFO: ALKiln will now hide docker output
+          echo -e '===\n💡 ALK0007 INFO: ALKiln will now hide docker output to avoid possibly showing details about your config. If you want to get output to see more of what is going on, use "with:" and set the input "SHOW_DOCKER_OUTPUT" to "true". This output will only be visible to those who can already see the action output, like people with admin or write permissions.\n==='
       - name: "💡 ALK0008 INFO: ALKiln will show docker output"
         if: ${{ env.OUTPUT_DOCKER == 'true' }}
         shell: bash
         run: |
-          echo '===\n💡 ALK0008 INFO: ALKiln will show docker output. ALKiln cannot control this output. If you want to prevent output, use "with:" and set the input "SHOW_DOCKER_OUTPUT" to "false". This output will only be visible to those who can already see the action output, like people with admin or write permissions.\n==='
+          # 💡 ALK0008 INFO: ALKiln will show docker output
+          echo -e '===\n💡 ALK0008 INFO: ALKiln will show docker output. ALKiln cannot control this output. If you want to prevent output, use "with:" and set the input "SHOW_DOCKER_OUTPUT" to "false". This output will only be visible to those who can already see the action output, like people with admin or write permissions.\n==='
 
       # TODO: Reduce these to 1 block by putting ` > /dev/null 2>&1` into an env var
       # No docker output
@@ -156,6 +158,7 @@ runs:
         if: ${{ env.OUTPUT_DOCKER == 'false' }}
         shell: bash
         run: |
+          # 💡 ALK0011 INFO: The following messages are not docker output
           echo "💡 ALK0011 INFO: The following messages are not docker output, just ALKiln waiting for the server to start"
 
       - name: "💡 ALK0012 INFO: Waiting for server to start"
@@ -245,4 +248,3 @@ runs:
         if: ${{ failure() }}
         run: |
           echo "🤕 ALK0022 ERROR: Something went wrong. See above for more details."
-

package/lib/run_cucumber.js

@@ -3,9 +3,11 @@
 const fs = require('fs');
 const cucumber_api = require('@cucumber/cucumber/api');
 
-const log = require('./utils/log');
+const log = require('./utils/log');  // These always log in the console
 const session_vars = require('./utils/session_vars');
 
+const SUMMARY_FILENAME = `cucumber-report.txt`;
+
 function build_config( command_args ) {
   // Turn all the paths to the sources folders into paths to the feature files
   let sources_paths = session_vars.get_sources_paths();
@@ -17,14 +19,17 @@ function build_config( command_args ) {
   const config = {
     publishQuiet: true,
     require: `./lib/index.js`,
-    format: [`progress`, `summary:cucumber-report.txt`],
+    format: [`progress`, `summary:${ SUMMARY_FILENAME }`],
     paths: feature_paths,
     retry: 1,
   }
 
   // Normalize tag expressions, if there are any
-  let tags = command_args[`_`];
-  if (tags.length > 0) {
+  // Joining and splitting handles both tags surrounded with `""` or those without `""`
+  let tags = command_args[`_`].join(` `).split(` `);
+  // If there was no tag expression, `tags` is ['']. Remove that item.
+  tags = tags.filter(function(entry) { return entry !== ""; })
+  if ( tags.length > 0 ) {
     let joined_tags = '';
     // Cucumber expects tags in boolean expressions, like (@a0 or @a1) and @random.
     // If we get a list of only tags, then "or" them all together
@@ -105,19 +110,21 @@ function build_config( command_args ) {
 
   const { success } = await cucumber_api.runCucumber({...runConfiguration, support});
   if ( success ) {
-    log.info({ type: `run`, code: `ALK0054`, pre: `Cucumber executed correctly` });
+    log.success({ type: `run`, code: `ALK0054`, pre: `Tests succeeded!` });
   } else {
-    log.error({ type: `run`, code: `ALK0055`, pre: `Cucumber ran into an error` });
+    log.error({ type: `run`, code: `ALK0055`, pre: `The test run ran into something unexpected. Look above for more details.` });
   }
-  const exitCode = success ? 0 : 1;
+  // Custom exit code. May overlap with other systems.
+  // Best info I've found via stack overflow: https://tldp.org/LDP/abs/html/exitcodes.html
+  const exitCode = success ? 0 : 50;
   process.exitCode = exitCode;
 
   // This is hardcoded information from `provide_config` above.
   // This code isn't used for anything other than detecting and removing color codes.
-  let summary_file_name = 'cucumber-report.txt';
+  let summary_file_name = SUMMARY_FILENAME;
   fs.readFile(summary_file_name, 'utf8', (err, text) => {
     if (err) {
-      log.error({ type: `run`, code: `ALK0056`, pre: `cucumber report`, data: err });
+      log.error({ type: `internal run`, code: `ALK0056`, pre: `Failed to read cucumber report`, data: err });
       return;
     }
 
@@ -127,6 +134,18 @@ function build_config( command_args ) {
 
     // Put it in the correct file in the correct folder
     let artifacts_path = session_vars.get_artifacts_path_name();
-    fs.appendFileSync( `${ artifacts_path }/${ log.debug_log_file }`, `\n\n${text}`);
+    try {
+      fs.appendFileSync( `${ artifacts_path }/${ log.debug_log_file }`, `\n\n${text}`);
+    } catch ( error ) {
+      log.info({ type: `internal run`, code: `ALK0182`, pre: `Something probably went wrong earlier. Look above for more details, but here's more info just in case: ALKiln was unable to write to ${ artifacts_path }/${ log.debug_log_file }.`, data: error });
+    }
+    try {
+      // Add to a unexpected results file when tests themselves fail
+      if ( exitCode === 50 ) {
+        fs.appendFileSync( `${ artifacts_path }/${ log.unexpected_filename }`, `\n\n${ text }` );
+      }
+    } catch ( error ) {
+      log.info({ type: `internal run`, code: `ALK0181`, pre: `Something probably went wrong earlier. Look above for more details, but here's more info just in case: ALKiln was unable to write to ${ artifacts_path }/${ log.unexpected_filename }.`, data: error });
+    }
   });
 })();

package/lib/scope.js

@@ -3228,8 +3228,7 @@ module.exports = {
     * 
     * @param expected {array} Arr of strs that should appear in the report.
     */
-    let current_report_obj = scope.report.get( scope.scenario_id );
-    let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+    let scenario = scope.report.get( scope.scenario_id );
     let report = reports.getPrintableScenario( scenario );
     let all_are_included = true;
     for ( let one_expectation of expected ) {
@@ -3248,8 +3247,7 @@ module.exports = {
     * 
     * @param not_expected {array} Arr of strs that should not appear in the report.
     */
-    let current_report_obj = scope.report.get( scope.scenario_id );
-    let scenario = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+    let scenario = scope.report.get( scope.scenario_id );
     let report = reports.getPrintableScenario( scenario );
     let none_included = true;
     for ( let one_expectation of not_expected ) {

package/lib/steps.js

@@ -101,8 +101,11 @@ BeforeAll(async () => {
   session_vars.save_artifacts_path_name( scope.paths.artifacts );
 
   scope.paths.report = `${ scope.paths.artifacts }/report.txt`;
+  scope.paths.failures = `${ scope.paths.artifacts }/${ log.unexpected_filename }`;
   scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_file }`;
   fs.writeFileSync( scope.paths.debug_log, "" );
+
+  reports.create( scope );
 });
 
 Before(async (scenario) => {
@@ -1202,17 +1205,17 @@ After(async function(scenario) {
   }
 
   if ( scope.failed_pdf_compares.length > 0) {
-    let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n---\n${ new_msg }`)
+    let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n―――\n${ new_msg }`)
     changeable_test_status = `FAILED`;
     reports.addToReport(scope, {
       type: `error`,
       code: `ALK0093`,
-      value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\n---\n${ msg }\n---\n`
+      value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\n―――\n${ msg }\n―――\n`
     });
   }
 
   // Record status before internal report tests can interfere with values
-  reports.setReportScenarioStatus( scope, { status: changeable_test_status });
+  reports.saveFinalScenarioData( scope, { status: changeable_test_status });
 
   if (changeable_test_status !== `PASSED`) {
     log.debug({
@@ -1256,11 +1259,11 @@ After(async function(scenario) {
   }  // ends if not passed
 
   // Add report text to Scenario folder after everything has been added to the report
-  let current_report_obj = scope.report.get( scope.scenario_id );
-  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+  let scenario_report_obj = scope.report.get( scope.scenario_id );
   let report = reports.getPrintableScenario( scenario_report_obj );
   // Save the report as in the Scenario folder
   fs.writeFileSync( scope.paths.scenario_report, report );
+  // TODO: Save any cucumber failure message for the Scenario here (instead of waiting till run_cucumber.js)
 
   // ---------------- Check internal test results ----------------
 
@@ -1273,7 +1276,7 @@ After(async function(scenario) {
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_in_report = null;
   }
-  
+
   if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) {
     let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude(
       scope,
@@ -1354,7 +1357,7 @@ After(async function(scenario) {
     // Catch a possible reload error.
     try {
       // puppeteer will ensure proper timeout.
-      await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});  
+      await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});
 
     } catch ( error ) {
       signout_succeeded = false;
@@ -1405,17 +1408,14 @@ After(async function(scenario) {
 AfterAll(async function() {
   // Stop collecting server response statuses
   scope.stop_tracking_server_status = true;
-  
+
   let report = reports.getPrintableReport( scope );
   // Log the report. Can't get this.attach() or this.log() to work in `After()` or here.
   // May need newest version of cucumberjs
   console.log( `\n\n${ report }` );
 
-  // Save/download the report as an artifact
-  fs.writeFileSync( scope.paths.report, report );
-
   // If there is a browser open, then close it
   if (scope.browser) {
     await scope.browser.close();
   }
-});
+});

package/lib/utils/log.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const session_vars = require( './session_vars' );
 
 // ? 🌈 💡 🔎 🤕 🐞
+// Alternatives: 🩹 ❤️‍🩹 🚑
 // TODO: return full message so it can be passed to manually
 // thrown errors if needed.
 
@@ -91,4 +92,5 @@ log.add_to_debug_log = function(value) {
   fs.appendFileSync(log.debug_log_file, value);
 };
 
-log.debug_log_file = 'debug_log.txt'
+log.debug_log_file = `debug_log.txt`;
+log.unexpected_filename = `unexpected_results.txt`;

package/lib/utils/reports.js

@@ -7,6 +7,13 @@ const files = require(`./files` );
 
 module.exports = reports = {};
 
+reports.title = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
+reports.failures = [ `\n\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n🤕 Failed scenarios\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n` ];
+reports.successes = [ `\n\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n🌈 Passed scenarios\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n` ];
+// reports.warnings = [ `\n\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n🔎 Scenarios to check\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n` ];
+// reports.undefineds = [ `\n\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n[langs icon] Scenarios with typos\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n` ];
+reports.others = [ `\n\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n🔎 Other scenarios that did not pass\n≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n` ];
+
 reports.addReportHeading = function (scope, { scenario }) {
   /** Return printable heading based on Scenario name and tags.
   *    Allow characters from other languages.
@@ -19,14 +26,14 @@ reports.addReportHeading = function (scope, { scenario }) {
     tag_names.push( tag.name );
   }
 
-  let heading = `\n---------------\n`
+  let heading = `\n―――――――――――――――\n`
     + `Scenario: ${ scenario.pickle.name }`;
 
   if ( tag_names.length > 0 ) {
     heading += `\nTags: ${ tag_names.join(' ') }`;
   }
 
-  heading += `\n---------------`;
+  heading += `\n―――――――――――――――`;
 
   reports.addToReport( scope, {
     code: `ALK0176`,
@@ -71,21 +78,30 @@ reports.addToReport = function( scope, { type, code, value }) {
   }
 
   // Report stuff
-  let scenario = makeSureReportPartsExist( scope );
+  let scenario = reports.makeSureScenarioExists( scope );
   scenario.lines.push({ type, code, value });
 
   return scenario;
 };  // Ends reports.addToReport()
 
-let makeSureReportPartsExist = function( scope ) {
+reports.create = function( scope ) {
+  reports.makeSureReportExists( scope );
+}
+
+reports.makeSureReportExists = function( scope ) {
   /** If a part of the report needs to exist right now, but doesn't exist,
   * create it. */
   if ( !scope.report ) { scope.report = new Map(); }
+  return this;
+};  // Ends reports.makeSureReportExists()
+
+reports.makeSureScenarioExists = function ( scope ) {
+  reports.makeSureReportExists( scope );
   if ( !scope.scenario_id ) {
     scope.scenario_id = `_report_${ Date.now() }`;
   }
   if ( !scope.report.get( scope.scenario_id )) {
-    scope.report.set( scope.scenario_id, {} )
+    scope.report.set( scope.scenario_id, {} );
     scope.report.get( scope.scenario_id ).status = `in progress`;
   }
 
@@ -93,11 +109,36 @@ let makeSureReportPartsExist = function( scope ) {
   if ( !scenario.lines ) { scenario.lines = []; }
 
   return scenario;
-};  // Ends makeSureReportPartsExist()
+};  // Ends reports.makeSureScenarioExists()
+
+reports.saveFinalScenarioData = function( scope, { status=`in progress` }) {
+  this.setReportScenarioStatus( scope, { status });
+  let scenario = scope.report.get( scope.scenario_id );
+  let report = this.getPrintableScenario( scenario );
+  if ( scenario.status === `FAILED` ) { this.failures.push( report ); }
+  else if ( scenario.status === `PASSED` ) { this.successes.push( report ); }
+  // else if ( scenario.status === `WARNING` ) { passed_reports += report; this.warnings.push( report ); }
+  // else if ( scenario.status === `UNDEFINED` ) { passed_reports += report; this.undefineds.push( report ); }
+  else { this.others.push( report ); }
+
+  // Save to failures.txt (worry about optimization later)
+  if ( this.failures.length > 1 || this.others.length > 1 ) {
+    fs.writeFileSync(
+      scope.paths.failures,
+      this.title + this.getPrintableUnexpected( scope )
+    );
+  }
+  // Save to reports.txt (worry about optimization later)
+  fs.writeFileSync( scope.paths.report, this.getPrintableReport( scope ) );
+
+  return scenario;
+};  // Ends reports.saveFinalScenarioData()
+
 
 reports.setReportScenarioStatus = function( scope, { status=`PASSED` }) {
   /** Add final status of scenario to scenario report object - passed, failed, etc. */
   scope.report.get( scope.scenario_id ).status = status;
+  return scope.report.get( scope.scenario_id );
 };  // Ends reports.setReportScenarioStatus()
 
 reports.convertToOriginalStoryTableRow = function( scope, { row_data }) {
@@ -124,34 +165,38 @@ let trimJSON = function( str=`` ) {
 };  // Ends trimJSON()
 
 reports.getPrintableReport = function( scope ) {
-  /** Return a string generated from test report data. */
-  let report = `Assembly Line Kiln Automated Testing Report - ${ (new Date()).toUTCString() }\n`;
-
-  // First list failed scenarios, then non-passed scenarios, then passed scenarios
-  let failed_reports = ``;
-  let passed_reports = ``;
-  let other_reports = ``;
-  for ( let scenario of scope.report ) {
-
-    let report = reports.getPrintableScenario( scenario );
-    let [ scenario_id, data ] = scenario;
-    if ( data.status === `FAILED` ) { failed_reports += report; }
-    else if ( data.status === `PASSED` ) { passed_reports += report; }
-    else { other_reports += report; }
-  };  // ends for every scenario
-
-  if ( failed_reports !== `` ) { report += `\n\n===============================\n===============================\n🤕 Failed scenarios:\n${ failed_reports }`; }
-  if ( other_reports !== `` ) { report += `\n\n===============================\n===============================\n🔎 Scenarios that did not pass:\n${ other_reports }`; }
-  if ( passed_reports !== `` ) { report += `\n\n===============================\n===============================\n🌈 Passed scenarios:\n${ passed_reports }`; }
+  /** Return a string generated from test report data.
+  *
+  * First list failed scenarios, then non-passed scenarios, then passed scenarios. */
+  let report = this.getTitle( scope );
+
+  if ( this.failures.length > 1 ) { report += this.failures.join( `` ); }
+  if ( this.successes.length > 1 ) { report += this.successes.join( `` ); }
+  // if ( this.warnings.length > 1 ) {}
+  // if ( this.undefineds.length > 1 ) {}
+  if ( this.others.length > 1 ) { report += this.others.join( `` ); }
 
   return report;
 };  // Ends reports.getPrintableReport()
 
+reports.getTitle = function( scope ) { return this.title; }
+
+reports.getPrintableUnexpected = function( scope ) {
+  // Note about `> 1`: The first line of the report section always exists
+  let output = ``;
+  if ( this.failures.length > 1 ) {
+    output += this.failures.join(``);
+  }
+  if ( this.others.length > 1 ) {
+    output += this.others.join(``);
+  }
+  return output;
+};  // Ends reports.getPrintableUnexpected()
+
 reports.getPrintableScenario = function( scenario ) {
   /** Return a string generated from scenario list of report items. */
-  let [ scenario_id, data ] = scenario;
   let report = ``;
-  for ( let line of data.lines ) {
+  for ( let line of scenario.lines ) {
     let { type, code, value } = line;
     report += `${ getLineText({ type, code, value }) }\n`;
   }

package/package.json

@@ -1,12 +1,14 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.11.2",
+  "version": "5.12.0",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
     "test": "npm run unit && npm run codes && npm run cucumber",
     "cucumber": "./lib/run_cucumber.js",
     "local": "npm run setup && npm run cucumber; npm run takedown",
+    "pass": "npm run cucumber 'not @error'",
+    "fail": "npm run cucumber '@error'",
     "setup": "./lib/docassemble/setup.js",
     "takedown": "./lib/docassemble/takedown.js",
     "server_install": "./lib/docassemble/server_install.js",

package/tests/log_codes/check_codes.sh

@@ -106,7 +106,8 @@ ALK0026
 ALK0027
 ALK0028
 ALK0030
-ALK0049"
+ALK0049
+ALK0184"
 
 # Use a sorted list. `uniq` only detects consecutive duplicates
 duplicates=$(echo "$sorted" | uniq -d)