@suffolklitlab/alkiln 5.11.1 → 5.11.2

package/CHANGELOG.md

@@ -45,6 +45,23 @@ Format:
 
 <!-- ## [Unreleased] -->
 
+## [5.11.2] - 2024-05-17
+
+### Added
+
+- Added new debug log/code to reflect difference between a expected status mismatch and a cucumber status mismatch.
+
+### Changed
+
+- Changed text of error to make it more clear.
+- Changed logic of error finding to use expected status for users as well as internal tests. That needs discussion.
+- Refactored code in that section.
+- Wait till the end to throw some errors so the user can get as much info as possible about the failures.
+
+### Internal
+
+- Exclude another file from code checking (cucumber-report.txt)
+
 ## [5.11.1] - 2024-04-06
 
 ### Fixed

package/lib/steps.js

@@ -1262,7 +1262,7 @@ After(async function(scenario) {
   // Save the report as in the Scenario folder
   fs.writeFileSync( scope.paths.scenario_report, report );
 
-  // ---------------- Check internal test results of this framework ----------------
+  // ---------------- Check internal test results ----------------
 
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
     let report_includes_all_expected_strings = await scope.reportIncludesAllExpected(
@@ -1284,74 +1284,120 @@ After(async function(scenario) {
     scope.expected_not_in_report = null;
   }
 
-  // The default is 'passed', which won't need to get in here
-  if ( scope.expected_status ) {
-    // When internal tests should fail (to test our failure conditions), passing _is_ failing
-    expect( changeable_test_status ).to.equal( scope.expected_status );
-    // If the status was as expected, pass this Scenario
-    changeable_test_status = `PASSED`;
-    scope.expected_status = null;  // reset for next Scenario
+  // ---------------- Ends internal results check ----------------
+
+  if ( scope.expected_status === null ) {
+    scope.expected_status = `PASSED`;  // default expected status
   }
 
-  // ---------------- Ends check internal test results of this framework ----------------
+  let expected_status_was_met = true;
+  let expected_status_err_msg = ``;
+  // Avoid manipulating `UNDEFINED` and such
+  if ( changeable_test_status === `PASSED`
+       || changeable_test_status === `FAILED` ) {
+
+    if ( changeable_test_status === scope.expected_status ) {
+      // If the status was as expected, pass this Scenario. E.g. an internal
+      // test was supposed to fail and it did. Or a test was suppose to pass
+      // and it passed.
+      changeable_test_status = `PASSED`;
+    } else {
+      // If a test that was supposed to fail passed (e.g. internal
+      // test), it failed. If is was supposed to pass and failed, it also
+      // failed. E.g. a11y failures let the Scenario pass initially, but
+      // should cause a failure here.
+      changeable_test_status = `FAILED`;
+      // Flag this failure to trigger an error later, so cucumber shows
+      // it as failed too.
+      expected_status_was_met = false;
+      // Keep it general. Users may see this message, e.g. for a11y tests.
+      let msg = `Test failed. Expected status: ${scope.expected_status}. `
+          + `ALKiln's assessed status: ${ changeable_test_status }. `
+          + `See the report for details.`;
+      let code = `ALK0099`;
+      log.debug({ type: `error`, code: code, pre: msg, })
+      expected_status_err_msg = `🤕 ${ code } ERROR: ${ msg }`
+    }
+  }
+  scope.expected_status = null;  // reset for next Scenario
+
+  let need_to_force_failed_status = false;
+  let force_failure_msg = `ALKiln assessed that the test failed. See the report for details.`;
+  let force_failure_code = `ALK0180`;
+  if ( changeable_test_status !== `FAILED` ) {
+    // Other than `FAILED`, it's fine to set the scenario status to make
+    // sure it matches up with ALKiln's assessed status.
+    // WARNING: This may be impossible in future cucumber versions
+    scenario.result.status = changeable_test_status;
+    // TODO: Should we add (an info) debug log here too?
+
+  // If assessed status is FAILED, but result.status is something else
+  } else if ( scenario.result.status !== `FAILED` ) {
+    // Make sure that tests which ALKiln has calculated as failed
+    // cause a cucumber error later. Just changing the status to `FAILED`
+    // causes unhelpful cucumber errors. TODO: Add to internal docs.
+    log.debug({
+      type: `error`,
+      code: force_failure_code,
+      pre: force_failure_msg
+    });
+    // TODO: Look into setDefinitionFunctionWrapper to catch errors
+    // of failing tests that should indeed be failing:
+    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
+    need_to_force_failed_status = true;
+  }
 
+  let signout_succeeded = true;
+  let signout_error_msg = ``;
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
     // Catch a possible reload error.
-
-    // TODO: implement and use scope.handle_possible_timeout()
     try {
       // puppeteer will ensure proper timeout.
       await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});  
 
     } catch ( error ) {
+      signout_succeeded = false;
 
       if ( error.name === `TimeoutError` ) {
         // TODO: Should a more specific message about signing out get into the report no matter what?
         // If so, what should the message be in order to not get confused with these other messages?
         let non_reload_report_msg = `It took too long to sign out in preparation for the next test.`;
-        await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
-          code: `ALK0097`,
-          message: non_reload_report_msg,
-        }, error });
+        let code = `ALK0097`;
+        try {
+          await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
+            code: code,
+            message: non_reload_report_msg,
+          }, error });
+        } catch {
+          signout_error_msg = `🤕 ${ code } ERROR: ${ non_reload_report_msg }`;
+        }
       } else {
         // Throw any non-timeout error
-        let err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
-        reports.addToReport( scope, { type: `error`, code: `ALK0098`, value: err_msg });
-        throw error;
+        let other_err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
+        let code = `ALK0098`;
+        reports.addToReport( scope, { type: `error`, code: code, value: other_err_msg });
+        signout_error_msg = `🤕 ${ code } ERROR: ${ other_err_msg }`;
       }  // ends if error is timeout error
 
     }  // ends try/catch
 
   }  // ends if scope.page
 
-  // INTERNAL WARNING: TODO: A console.log here prevents the try/catch from
-  // throwing an error. Why is that? Does it somehow make sure that scope.page
-  // gets destroyed?
-
-  // console.log('This stops try/catch from throwing an error in some cases.');
-
-  // More internal testing stuff
-  if ( changeable_test_status === `FAILED` ) {
-    // If the result object status wasn't failure and we still want
-    // the tests to fail, as with a11y tests, we can't mess with the
-    // cucumber result.status object because it will cause unwanted
-    // errors. We have to throw an error here instead.
-    if ( scenario.result.status !== `FAILED` ) {
-      let msg = `The test unexpectedly didn't fail. Scenario status: ${scenario.result.status}.`
-      reports.addToReport( scope, { type: `error`, code: `ALK0099`, value: msg });
-      throw new Error(msg);
-    }
-  } else {
-    // Some internal test failures should be ok
-    // TODO: Look into setDefinitionFunctionWrapper to catch errors
-    // of failing tests that should indeed be failing:
-    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
-    scenario.result.status = changeable_test_status;
+  let errors = []
+  if ( !expected_status_was_met ) { errors.push( expected_status_err_msg ); }
+  if ( !signout_succeeded ) { errors.push( signout_error_msg ); }
+  if ( !expected_status_was_met || !signout_succeeded ) {
+    throw new Error( `${errors.join( errors, '\n')}` );
+  }
+
+  // Theoretically, we should never get here:
+  // If everything else worked, if we need to force an internal test failure
+  if ( need_to_force_failed_status ) {
+    throw new Error( `🤕 ${force_failure_code} unreachable ERROR: ${force_failure_msg}` );
   }
-  // Ends more internal testing stuff
 
-  // WARNING: TODO: console.log can prevent a sign-out error. What race condition is going on?
+  // Find the race condition.
   log.debug({ type: `cucumber`, code: `ALK0100`, verbose: true, pre: `Scenario AfterStep() message`, data: scenario.result.message });
 
 });

package/lib/utils/log.js

@@ -2,6 +2,8 @@ const fs = require('fs');
 const session_vars = require( './session_vars' );
 
 // ? 🌈 💡 🔎 🤕 🐞
+// TODO: return full message so it can be passed to manually
+// thrown errors if needed.
 
 let log = {};
 module.exports = log;

package/lib/utils/reports.js

@@ -46,6 +46,9 @@ reports.addToReport = function( scope, { type, code, value }) {
   * @params code {string} (Optional) Error or warning code. Default
   *    is unspecified error or warning.
   * */
+  // TODO: return full message so it can be passed to manually
+  // thrown errors if needed.
+
   // Debugging logs
   let debug_line = getDebugLine({ type, code, value });
   if ( session_vars.get_debug() ) {

package/lib/utils/session_vars.js

@@ -120,7 +120,7 @@ session_vars.save_project_name = function ( project_name ) {
   // Warn local developers if file already exists
   if ( fs.existsSync( runtime_config_path ) ) {
     let old_project_name = session_vars.get_project_name();
-    let session_project_already_exists_msg = `🔎 ALK0035 WARNING: `
+    let session_project_already_exists_msg = `🔎 ALK0035 setup WARNING: `
       + `This session had a project name already: ${ old_project_name }. `
       + `A new project has now been created called ${ project_name }. `
       + `Remember to delete ${ old_project_name }`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.11.1",
+  "version": "5.11.2",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {

package/tests/log_codes/check_codes.sh

@@ -16,7 +16,7 @@ fi
 # The grepped files will include the list of all the codes that have been
 # removed and are no longer used. Syntax used works with GitHub cli ([[:digit:]])
 # https://stackoverflow.com/a/6901221
-lines=$(grep -roh --exclude="tests/log_codes/check_codes.sh" --exclude="debug_log.txt" --exclude-dir="node_modules" --exclude-dir='alkiln-*' --exclude-dir='_alkiln*' 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' "$directory")
+lines=$(grep -roh --exclude="tests/log_codes/check_codes.sh" --exclude="debug_log.txt" --exclude="cucumber-report.txt" --exclude-dir="node_modules" --exclude-dir='alkiln-*' --exclude-dir='_alkiln*' 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' "$directory")
 sorted=$(echo "$lines" | sort -n)