@suffolklitlab/alkiln 5.8.1-feat2columns3 → 5.8.1-feat2columns5

package/.github/workflows/github_server.yml

@@ -88,5 +88,5 @@ jobs:
           #### Developer note: Useful for temporarily shortening ALKiln tests
           #### to debug the docker install step
           # ALKILN_TAG_EXPRESSION: @a_tag_expression
-          ALKILN_VERSION: 5.8.1-feat2columns3
+          ALKILN_VERSION: 2columns
 

package/.github/workflows/playground.yml

@@ -55,4 +55,4 @@ jobs:
           #### to debug the docker install step
           # ALKILN_TAG_EXPRESSION: @a_tag_expression
           #### Developer note: you probalby don't need this
-          ALKILN_VERSION: 5.8.1-feat2columns3
+          ALKILN_VERSION: 2columns

package/docs/expected_inernal_results.md

@@ -0,0 +1,15 @@
+# What results are we expecting in our own tests?
+
+Our internal test results look a little strange. For example, they sometimes initially look like they're failing, but eventually pass. That's because we're testing that our tests respond appropriately to authors' failures as well as successees.
+
+These are the results that we expect:
+
+## Cucumber feature tests {#cucumber}
+
+- Many red `F`s in our cucumber "progress bar"
+- Final results to show all Scenarios passing
+- Cucumber seems to be unable to print the name of the first scenario
+
+## Unit tests {#unit}
+
+All these must pass

package/lib/steps.js

@@ -106,6 +106,12 @@ BeforeAll(async () => {
 });
 
 Before(async (scenario) => {
+
+  // Useful for those with a lot of tests
+  // Is unable to print the first Scenario name for some reason
+  console.log('');
+  process.stdout.write(`Scenario: ${ scenario.pickle.name }: `);
+
   // Create browser, which can't be created in BeforeAll() where .driver doesn't exist for some reason
   if (!scope.browser) {
     if (session_vars.get_origin() === 'playground') {
@@ -1244,8 +1250,8 @@ Given(/the Scenario report should not include/i, async ( report_text ) => {
 
 });
 
-Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected_status ) => {
-  /* WARNING: If a Step will fail, this Step must come first.
+Given(/the final Scenario status should be "([^"]+)"/i, async ( expected_status ) => {
+  /** WARNING: If a Step will fail, this Step must come first.
   * 
   * Prepare to test that the Scenario completes with the correct status. */
   scope.expected_status = expected_status.toUpperCase();
@@ -1327,39 +1333,46 @@ After(async function(scenario) {
 
   }  // ends if not passed
 
-  // Internal tests of this framework
-  if ( scope.expected_status ) {
-    let expected = scope.expected_status;
-    scope.expected_status = null;  // reset for next Scenario
-    expect( changeable_test_status ).to.equal( expected );
-    // If the status was as expected, pass this criteria
-    changeable_test_status = `PASSED`;
-  }
-  
+  // Add report text to Scenario folder after everything has been added to the report
+  let current_report_obj = scope.report.get( scope.scenario_id );
+  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+  let report = reports.getPrintableScenario( scenario_report_obj );
+  // Save the report as in the Scenario folder
+  fs.writeFileSync( scope.paths.scenario_report, report );
+
+  // ---------------- Check internal test results of this framework ----------------
+
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
-    let expected = scope.expected_in_report;
+    let report_includes_all_expected_strings = await scope.reportIncludesAllExpected(
+      scope,
+      { expected: scope.expected_in_report }
+    );
+    if ( !report_includes_all_expected_strings ) { changeable_test_status = `FAILED`; }
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_in_report = null;
-    let all_were_included = await scope.reportIncludesAllExpected( scope, { expected });
-
-    if ( all_were_included ) { changeable_test_status = `PASSED`; }
-    else { changeable_test_status = `FAILED`; }
   }
   
   if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) {
-    let not_expected = scope.expected_not_in_report;
+    let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude(
+      scope,
+      { not_expected: scope.expected_not_in_report }
+    );
+    if ( !all_prohibited_strings_were_absent ) { changeable_test_status = `FAILED`;}
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_not_in_report = null;
-    let all_were_not_there = await scope.reportDoesNotInclude( scope, { not_expected });
-
-    if ( all_were_not_there ) {
-      changeable_test_status = `PASSED`;
-    } else { 
-      changeable_test_status = `FAILED`;
-    }
+  }
 
+  // The default is 'passed', which won't need to get in here
+  if ( scope.expected_status ) {
+    // When internal tests should fail (to test our failure conditions), passing _is_ failing
+    expect( changeable_test_status ).to.equal( scope.expected_status );
+    // If the status was as expected, pass this Scenario
+    changeable_test_status = `PASSED`;
+    scope.expected_status = null;  // reset for next Scenario
   }
 
+  // ---------------- Ends check internal test results of this framework ----------------
+
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
     // Catch a possible reload error.
@@ -1387,23 +1400,19 @@ After(async function(scenario) {
 
   }  // ends if scope.page
 
-  // Add report text to Scenario folder after everything has been added to the report
-  let current_report_obj = scope.report.get( scope.scenario_id );
-  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
-  let report = reports.getPrintableScenario( scenario_report_obj );
-  // Save the report as in the Scenario folder
-  fs.writeFileSync( scope.paths.scenario_report, report );
-
   if ( changeable_test_status === `FAILED` ) {
     // If the result object status wasn't failure and we still want
     // the tests to fail, as with a11y tests, we can't mess with the
     // cucumber result.status object because it will cause unwanted
     // errors. We have to throw an error here instead.
     if ( scenario.result.status !== `FAILED` ) {
-      throw new Error(`The test failed. See the report for more information.`);
+      throw new Error(`The test failed. See the report above for more information.`);
     }
   } else {
-    // Other types of errors should be ok
+    // Some internal test failures should be ok
+    // TODO: Look into setDefinitionFunctionWrapper to catch errors
+    // of failing tests that should indeed be failing:
+    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
     scenario.result.status = changeable_test_status;
   }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.8.1-feat2columns3",
+  "version": "5.8.1-feat2columns5",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {