@suffolklitlab/alkiln 5.8.1 → 5.9.0

package/lib/steps.js

@@ -106,6 +106,12 @@ BeforeAll(async () => {
 });
 
 Before(async (scenario) => {
+
+  // Useful for those with a lot of tests
+  // Is unable to print the first Scenario name for some reason
+  console.log('');
+  process.stdout.write(`Scenario: ${ scenario.pickle.name }: `);
+
   // Create browser, which can't be created in BeforeAll() where .driver doesn't exist for some reason
   if (!scope.browser) {
     if (session_vars.get_origin() === 'playground') {
@@ -297,7 +303,9 @@ Given(/the max(?:imum)? sec(?:ond)?s for each step (?:in this scenario )?is (\d+
 // Story
 //#####################################
 //#####################################
-
+// TODO: Consider more permissive sentences:
+// with(?: this data| these answers)?:?
+// with(?:.*):?
 When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)" with this data:?/, {timeout: -1}, async (target_id, raw_var_data) => {
   /* NO CUCUMBER TIMEOUT. Must handle timeout ourselves:
   * https://github.com/cucumber/cucumber-js/blob/master/docs/support_files/timeouts.md#disable-timeouts
@@ -341,6 +349,7 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
         try {
           let { question_has_id, id, id_matches } = await scope.examinePageID( scope );
 
+          // THIS FUNCTION ACTUALLY MATTERS
           // Add special rows. Basically, do weirder things.
           // TODO: Remove from_story_table - no longer needed if only called from in here
           let ensured_var_data = await scope.ensureSpecialRows( scope, {
@@ -352,8 +361,10 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
             console.log( `Rows, including special rows:`, JSON.stringify( ensured_var_data ));
           }
 
+          // THIS FUNCTION ACTUALLY MATTERS
           // Some errors happen in there to be closer to the relevant code. We'd have to try/catch anyway.
           let error = await scope.setFields(scope, { var_data: ensured_var_data, id, ensure_navigation: true });
+
           if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
           resolve();
         } catch ( err ) {
@@ -415,6 +426,123 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
   reports.addToReport(scope, { type: `note`, value: `` });
 });
 
+// When(/^foo id "([^"]+)"/, {timeout: -1}, async (target_id, raw_var_data) => {
+//   /* NO CUCUMBER TIMEOUT. Must handle timeout ourselves:
+//   * https://github.com/cucumber/cucumber-js/blob/master/docs/support_files/timeouts.md#disable-timeouts
+//   * Iterate in here to keep timeout reasonable */
+
+//   // Note: Why do we not just stop when we've used all the vars in the table or
+//   // reach the last var? Vars are not required to be given in sequence and pages that
+//   // have non-button fields don't usually have a var assigned to the 'continue' button.
+//   // Alternative: require vars even for 'continue' pages. Then in loop require at least one var
+//   // to be found on a page before hitting 'continue'. That requires detecting navigation because
+//   // pages with fields will not have a var for the continue button, so there will
+//   // still have to be some automatic continuing.
+//   // It also seems harsh, but more reliable.
+//   // Ask developers what they'd like.
+
+
+//   let supported_table = await scope.normalizeTable( scope, { raw_var_data });
+
+//   let { question_has_id, id, id_matches } = await scope.examinePageID( scope, target_id );
+//   // Until we reach the target page, hit an error, or time out (take too long)
+//   while ( !id_matches ) {
+//     let custom_timeout = new Promise(function( resolve, reject ) {
+
+//       // Set up the timeout that avoids infinite loops
+//       let on_page_timeout = async function() {
+
+//         let non_reload_report_msg = `The target id was "${ target_id }", but it took `
+//         + `too long to try to reach it (over ${ scope.timeout/1000/60 } min). The `
+//         + `test got stuck on "${ id }".`
+//         let error = non_reload_report_msg;
+//         await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+//       };  // Ends on_page_timeout()
+//       let page_timeout_id = setTimeout(on_page_timeout, scope.timeout);
+
+//       // self-invoke an anonymous async function so we don't have to
+//       // abstract the details of resolving and rejecting.
+//       // If we need to call recursively, signature function might look like:
+//       // func( scope, { resolve, reject, supported_table, page_timeout_id })
+//       // Maybe list of timeout and/or interval ids in state that can all be cleared at once?
+//       (async function() {
+//         try {
+//           let { question_has_id, id, id_matches } = await scope.examinePageID( scope );
+
+//           // Add special rows. Basically, do weirder things.
+//           // TODO: Remove from_story_table - no longer needed if only called from in here
+//           let ensured_var_data = await scope.ensureSpecialRows( scope, {
+//             var_data: supported_table,
+//             from_story_table: true,
+//           });
+
+//           if ( session_vars.get_debug() ) {
+//             console.log( `Rows, including special rows:`, JSON.stringify( ensured_var_data ));
+//           }
+
+//           // Some errors happen in there to be closer to the relevant code. We'd have to try/catch anyway.
+//           let error = await scope.setFields(scope, { var_data: ensured_var_data, id, ensure_navigation: true });
+//           if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
+//           resolve();
+//         } catch ( err ) {
+//           let err_msg = `Error occurred when tried to answer fields on question "${ id }".`
+//           reports.addToReport( scope, { type: `error`, value: err_msg });
+//           // Throw error instead?
+//           reject( err );
+//         } finally {
+//           clearTimeout( page_timeout_id );  // prevent temporary hang at end of tests
+//         }
+//       })();
+//     });  // ends custom_timeout for setFields
+
+//     await custom_timeout;
+
+//     // Prep for next loop
+//     ({ question_has_id, id, id_matches } = await scope.examinePageID( scope, target_id ));
+
+//   }  // ends while !id_matches
+//   // Can anything unsatisfactory get through here?
+
+//   // Add report strings from supported table, not an altered one
+//   for ( let row of supported_table ) {
+//     row.report_str = `${ reports.convertToOriginalStoryTableRow(scope, { row_data: row }) }`;
+//   }
+//   // Sort into alphabetical order, as is recommended for writing story tables
+//   let sorted_rows = supported_table.sort( function ( row1, row2 ) {
+//     if( row1.report_str < row2.report_str ) { return -1; }
+//     if( row1.report_str > row2.report_str ) { return 1; }
+//     return 0;
+//   });
+
+//   // Add full used table to report so it can be copied if needed.
+//   // TODO: Should we/How do we get this in the final scenario report instead of here?
+//   reports.addToReport( scope, { type: `note`, value: `\n${ ' '.repeat(2) }Rows that got set:` });
+//   reports.addToReport( scope, { type: `step`, value: `${ ' '.repeat(4) }And I get to the question id "${ target_id }" with this data:` });
+//   reports.addToReport(scope, { type: `row`, value: `${ ' '.repeat(6) }| var | value | trigger |`, });
+//   let at_least_one_row_was_NOT_used = false;
+//   for ( let row of sorted_rows ) {
+//     if ( row.times_used > 0 ) {
+//       reports.addToReport(scope, { type: `row`, value: row.report_str, });
+//     } else {
+//       at_least_one_row_was_NOT_used = true;
+//     }
+//   }
+
+//   // Add unused rows if needed
+//   if ( at_least_one_row_was_NOT_used ) {
+//     reports.addToReport( scope, { type: `note`, value: `${ ' '.repeat(2) }Unused rows:` });
+//     for ( let row of sorted_rows ) {
+//       if ( row.times_used === 0 ) {
+//         reports.addToReport(scope, { type: `row`, value: row.report_str, });
+//       }
+//     }
+//   } else {
+//     reports.addToReport(scope, { type: `note`, value: `${ ' '.repeat(2) }All rows were used` });
+//   }
+
+//   reports.addToReport(scope, { type: `note`, value: `` });
+// });
+
 //#####################################
 //#####################################
 // Passive/Observational
@@ -1122,8 +1250,8 @@ Given(/the Scenario report should not include/i, async ( report_text ) => {
 
 });
 
-Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected_status ) => {
-  /* WARNING: If a Step will fail, this Step must come first.
+Given(/the final Scenario status should be "([^"]+)"/i, async ( expected_status ) => {
+  /** WARNING: If a Step will fail, this Step must come first.
   * 
   * Prepare to test that the Scenario completes with the correct status. */
   scope.expected_status = expected_status.toUpperCase();
@@ -1138,6 +1266,12 @@ Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected
 
 After(async function(scenario) {
 
+  // Log errors
+  if ( scenario.result.message ) {
+    log.debug({ type: `error`, pre: scenario.result.message });
+    // log.debug({ type: `error`, pre: scenario.result.message, verbose: true });
+  }
+
   let changeable_test_status = scenario.result.status;
   // The accessibility failures don't happen til the end, so the rest of the test can continue
   if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
@@ -1199,39 +1333,46 @@ After(async function(scenario) {
 
   }  // ends if not passed
 
-  // Internal tests of this framework
-  if ( scope.expected_status ) {
-    let expected = scope.expected_status;
-    scope.expected_status = null;  // reset for next Scenario
-    expect( changeable_test_status ).to.equal( expected );
-    // If the status was as expected, pass this criteria
-    changeable_test_status = `PASSED`;
-  }
-  
+  // Add report text to Scenario folder after everything has been added to the report
+  let current_report_obj = scope.report.get( scope.scenario_id );
+  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
+  let report = reports.getPrintableScenario( scenario_report_obj );
+  // Save the report as in the Scenario folder
+  fs.writeFileSync( scope.paths.scenario_report, report );
+
+  // ---------------- Check internal test results of this framework ----------------
+
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
-    let expected = scope.expected_in_report;
+    let report_includes_all_expected_strings = await scope.reportIncludesAllExpected(
+      scope,
+      { expected: scope.expected_in_report }
+    );
+    if ( !report_includes_all_expected_strings ) { changeable_test_status = `FAILED`; }
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_in_report = null;
-    let all_were_included = await scope.reportIncludesAllExpected( scope, { expected });
-
-    if ( all_were_included ) { changeable_test_status = `PASSED`; }
-    else { changeable_test_status = `FAILED`; }
   }
   
   if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) {
-    let not_expected = scope.expected_not_in_report;
+    let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude(
+      scope,
+      { not_expected: scope.expected_not_in_report }
+    );
+    if ( !all_prohibited_strings_were_absent ) { changeable_test_status = `FAILED`;}
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_not_in_report = null;
-    let all_were_not_there = await scope.reportDoesNotInclude( scope, { not_expected });
-
-    if ( all_were_not_there ) {
-      changeable_test_status = `PASSED`;
-    } else { 
-      changeable_test_status = `FAILED`;
-    }
+  }
 
+  // The default is 'passed', which won't need to get in here
+  if ( scope.expected_status ) {
+    // When internal tests should fail (to test our failure conditions), passing _is_ failing
+    expect( changeable_test_status ).to.equal( scope.expected_status );
+    // If the status was as expected, pass this Scenario
+    changeable_test_status = `PASSED`;
+    scope.expected_status = null;  // reset for next Scenario
   }
 
+  // ---------------- Ends check internal test results of this framework ----------------
+
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
     // Catch a possible reload error.
@@ -1259,23 +1400,19 @@ After(async function(scenario) {
 
   }  // ends if scope.page
 
-  // Add report text to Scenario folder after everything has been added to the report
-  let current_report_obj = scope.report.get( scope.scenario_id );
-  let scenario_report_obj = [ scope.scenario_id, current_report_obj ];  // Mimic how loop works with `Map`
-  let report = reports.getPrintableScenario( scenario_report_obj );
-  // Save the report as in the Scenario folder
-  fs.writeFileSync( scope.paths.scenario_report, report );
-
   if ( changeable_test_status === `FAILED` ) {
     // If the result object status wasn't failure and we still want
     // the tests to fail, as with a11y tests, we can't mess with the
     // cucumber result.status object because it will cause unwanted
     // errors. We have to throw an error here instead.
     if ( scenario.result.status !== `FAILED` ) {
-      throw new Error(`The test failed. See the report for more information.`);
+      throw new Error(`The test failed. See the report above for more information.`);
     }
   } else {
-    // Other types of errors should be ok
+    // Some internal test failures should be ok
+    // TODO: Look into setDefinitionFunctionWrapper to catch errors
+    // of failing tests that should indeed be failing:
+    // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
     scenario.result.status = changeable_test_status;
   }
 });

package/lib/utils/log.js

@@ -40,14 +40,25 @@ log.error = function ({ type='', pre='', data='', post='' }) {
   if ( post ) { console.error( post, `\n` ); }
 };
 
-log.debug = function ({ type='', pre='', data='', post='' }) {
+log.debug = function ({ type='', pre='', data='', post='', verbose=false }) {
   /* If debugging is turned on, log a message that includes `ALKiln` and `debug` */
-  if ( session_vars.get_debug() ) {
+  if ( session_vars.get_debug() || verbose ) {
     let start = create_msg_start([ type, 'debug' ], pre );
+    let full = `\n${ start }`;
     // Log them each on a separate line
-    console.log( `ALKiln ${ type } debug: ${ pre }` );
-    if ( data ) { console.log( data ); }
-    if ( post ) { console.log( post ); }
+    console.log( `\n${ start }` );
+    if ( data ) {
+      full += `\n${ data }`;
+      console.log( data );
+    }
+    if ( post ) {
+      full += `\n${ post }`;
+      console.log( post );
+    }
+    // TODO: Make a verbose log instead of adding to the debug one and always add to it
+    if ( verbose ) {
+      log.add_to_debug_log( full );
+    }
   }
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.8.1",
+  "version": "5.9.0",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {