@suffolklitlab/alkiln 4.7.0 → 4.7.1

package/CHANGELOG.md

@@ -40,12 +40,25 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [4.7.1] - 2022-08-07
+### Changed
+- Ensure unique folder names for random input screenshots in case developer uses multiple random steps in one Scenario.
+- Ensure random input detects unexpected interview errors.
+
+### Fixed
+- Fix failure when signature page is the first screen.
+- Internal tests failing even when failure status is as expected.
+- Double printing in report of scenario failure, one of them even when Scenario was possibly non-failure, like skipped.
+
 ## [4.7.0] - 2022-07-27
 ### Added
-- Generate random input to fill out a simple form automatically
+- Generate random input to fill out a simple form automatically. Does not yet ensure that random screenshot folder names are unique, or that error screens are detected as errors. 4.7.1 implements those.
 
 ### Changed
 - Add page id to the "Missing Variable or variables on page" error report
+- Allows Story Tables to not include a header row, as long as they have 2 or 3 columns.
+  This isn't the suggested way at the moment, since including headers is more readable,
+  but can prevent confusing and unnecessary errors.
 
 ## [4.6.2] - 2022-06-28
 ### Changed

package/lib/scope.js

@@ -191,6 +191,7 @@ module.exports = {
     let report = ``;
     for ( let line of data.lines ) {
       if ( line.type === `page_id` ) { report += `screen id: `; }
+      else if ( line.type === `outcome` ) { report += `\n`; }
       else if ( line.type === `warning` ) { report += `WARNING: `; }
       else if ( line.type === `error` ) { report += `\nERROR: ` }
       report += `${ line.value }\n`;
@@ -356,7 +357,7 @@ module.exports = {
     // This check might never be needed now as we try to match fields on the page
     // first in user-triggered cases - users used to be able to access this almost
     // directly, but they can't anymore. Our functions should know to only pass good things.
-    let msg = `Cannot find that element.`;
+    let msg = `Cannot find the element to tap.`;
     if ( !elem ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
 
     expect( elem, msg ).to.exist;
@@ -617,6 +618,9 @@ module.exports = {
     let result = { error: ``, elem: null };
 
     let winner = await Promise.race([
+      // Signature page
+      scope.page.waitForSelector( `#dasigpage`, { timeout: scope.timeout }),
+      // Any other valid page
       scope.page.waitForSelector( `#daMainQuestion`, { timeout: scope.timeout }),
       scope.page.waitForSelector( `#da-retry`, { timeout: scope.timeout }),
     ]);
@@ -626,10 +630,14 @@ module.exports = {
     let error_id_elem = await scope.page.$( `#da-retry` );
     if ( error_id_elem ) {
       let error_elem = await scope.page.$( `blockquote` );
-      let error_handle = await error_elem.getProperty( `textContent` );
-      let system_error_text = await error_handle.jsonValue();
+      if (error_elem !== null) {
+        let error_handle = await error_elem.getProperty( `textContent` );
+        let system_error_text = await error_handle.jsonValue();
 
-      result.error = system_error_text;
+        result.error = system_error_text;
+      } else {
+        result.error = "Docassemble ran into an error on the page, but ALKiln does not know what the error is. If a screenshot was taken, check the screenshot";
+      }
     }
 
     return result;
@@ -650,9 +658,40 @@ module.exports = {
     }
   },  // Ends scope.setLanguage()
 
-  normalizeTable: async function ( scope, { var_data, from_story_table=true }) {
-    /* Return data derived from cucumber variable-setting data.
-     * Table must have headers. Arrays are not supported. */
+  normalizeTable: async function ( scope, { var_data=null, raw_var_data=null }) {
+    /* Return data derived from cucumber variable-setting data, from either an object format
+     * (var_data) or an array format (raw_var_data). `var_data` only is used when calling
+     * normalizeTable internally.
+     */
+
+    if (!var_data && ! raw_var_data) {
+      let no_data_msg = `ALKiln Internal error: normalizeTable called w/o var_data or raw_var_data:` +
+          ` open an issue at https://github.com/SuffolkLITLab/ALKiln/issues/new if you see this!`;
+      await scope.addToReport( scope, { type: `error`, value: no_data_msg });
+      throw Error(no_data_msg);
+    }
+
+    // Always use the raw var data if present; it has the full first row
+    if (raw_var_data) {
+      var_data = raw_var_data.hashes();
+      if (!('var' in var_data[0]) || !('value' in var_data[0])) {
+        let raw_array = raw_var_data.raw();
+        var_data = [];
+        for (let row_raw of raw_array) {
+          if (row_raw.length < 2) {
+            let bad_table_msg = `Your Story Table definition needs to be changed. The best practice`
+                + ` is to make your table 3 columns wide with a first row in this format: "| var | value | trigger |".`
+                + ` The first row of your table is actually "${ raw_array[0] }"`;
+            await scope.addToReport( scope, { type: `error`, value: bad_table_msg})
+            throw Error(bad_table_msg);
+          } else if (row_raw.length == 2) {
+            var_data.push({var: row_raw[0], value: row_raw[1]});
+          } else {
+            var_data.push({var: row_raw[0], value: row_raw[1], trigger: row_raw[2]});
+          }
+        }
+      }
+    }
 
     // Support tables with no 'trigger' column
     let supported_table = [];
@@ -1934,6 +1973,20 @@ module.exports = {
     }
   },  // Ends scope.getThereIsAnotherValue()
 
+  // throw_if_no_interview_content()?
+  has_interview_content: async function ( scope ) {
+    /** Return false if ther's no element indicating a da interview question,
+    *    like a signature page, other fields, or interview text. Assumes that page has already
+    *    finished navigating. */
+    let elem_da_question = await scope.page.$( `#daMainQuestion` );
+    let elem_da_signature = await scope.page.$( `#dasigpage` );
+    if ( elem_da_question === null && elem_da_signature === null ) {
+      return false;
+    } else {
+      return true;
+    }
+  },  // Ends scope.has_interview_content()
+
   waitUntilContinued: async function ( scope, { url = '', id = '' }) {
     /** Given a url, attempt to continue until the page url changes or an error
         is found. Return the error object no matter what. */
@@ -2279,7 +2332,6 @@ module.exports = {
 
       // For every field on the page
       for ( let field of fields ) {
-
         // fields[n].tag is HTML tag of node, as a string
 
         // If the field is a button, save it in a list of buttons that we'll deal with later
@@ -2406,7 +2458,7 @@ module.exports = {
           if ( type === `file` ) {
             await scope.addToReport( scope, {
               type: `warning`,
-              value: `Sorry, ALKiln's random-input cannot yet handle "${ type }" fields.`
+              value: `Sorry, ALKiln's random input tests cannot yet handle "${ type }" fields.`
             });
           } else {
             let answer = await scope.get_random_input_for[ type ]( scope );

package/lib/steps.js

@@ -334,7 +334,8 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
   // It also seems harsh, but more reliable.
   // Ask developers what they'd like.
 
-  let supported_table = await scope.normalizeTable( scope, { var_data: raw_var_data.hashes() });
+
+  let supported_table = await scope.normalizeTable( scope, { raw_var_data });
 
   let { question_has_id, id, id_matches } = await scope.examinePageID( scope, target_id );
   // Until we reach the final page or hit an error
@@ -641,7 +642,7 @@ Then(/the "([^"]+)" link opens in (a new window|the same window)/, async (linkTe
   let [link] = await scope.page.$x(`//a[contains(text(), "${linkText}")]`);
 
   let prop_obj = await link.getProperty('target');
-  let target = await await prop_obj.jsonValue();
+  let target = await prop_obj.jsonValue();
 
   let should_open_a_new_window = which_window === 'a new window';
   let opens_a_new_window = target === '_blank';
@@ -790,35 +791,61 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
   * I answer randomly for at most 10 pages
   * I answer randomly for at most 101 screens
   */
-  scope.paths.random_screenshots = `${ scope.paths.scenario }/random_answers_screenshots`;
+  // Give each screenshots folder a time-stamped name in case the dev gets clever
+  // and use the same Scenario to run multiple randomized tests.
+  let timestamp = files.readable_date();
+  let random_input_path = `${ scope.paths.scenario }/random_answers_screenshots-${ timestamp }`
+  scope.paths.random_screenshots = random_input_path;
   fs.mkdirSync( scope.paths.random_screenshots );
+  await scope.addToReport( scope, {
+    type: 'row',
+    value: `Testing interview with random input (in ${ random_input_path })`
+  });
 
   // Force the dev to prevent infinite loops
   let max_pages = parseInt( max_pages_str );
   let curr_page_num = 0;
 
-  // While a continue button is on the current page, fill out
+  // While a continue-type button is on the current page, fill out
   // the fields and then try to continue
-  // let continue_exists = true;
-  // while ( continue_exists && curr_page_num < max_pages ) {
   let buttons_exist = true;
   while ( buttons_exist && curr_page_num < max_pages ) {
 
     curr_page_num = curr_page_num + 1;
     let { id } = await scope.examinePageID( scope, `no target id` );
 
+    let no_content_err = `The interview seemed to run into an error page`
+      + ` after the page with the question id "${ id }".`
+      + ` See the screenshot in "${ random_input_path }".`;
+
+    let has_thrown = false;
     // Ensure no infinite loop, but each page has a full timeout to run
     let custom_timeout = new Promise(async function( resolve, reject ) {
 
       // Set up timeout to stop infinite loops
       let page_timeout_id = setTimeout(async function() {
         
-        let timeout_message = `The page with an id of "${ id }" took too long `
-          + `to fill it with random answers (over ${ scope.timeout/1000/60 } min).`
-        await scope.addToReport( scope, { type: `error`, value: timeout_message });
-        throw( timeout_message );
+        // Make sure to only throw once
+        if ( has_thrown ) { return; }
+        else { has_thrown = true; }
 
-      }, scope.timeout);
+        let has_content = await scope.has_interview_content( scope );
+
+        if ( !has_content ) {
+          await scope.addToReport( scope, {
+            type: `error`,
+            value: no_content_err
+          });
+          reject( no_content_err );
+
+        } else {
+          let timeout_message = `The page with an id of "${ id }" took too long`
+            + ` to fill with random answers (over ${ scope.timeout/1000/60 } min).`
+          await scope.addToReport( scope, { type: `error`, value: timeout_message });
+          reject( timeout_message );
+        }
+
+      }, scope.timeout );
 
       // self-invoke an anonymous async function so we don't have to
       // abstract the details of resolving and rejecting.
@@ -830,7 +857,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
 
           // Take a screenshot of the page before continuing or ending
           await scope.page.screenshot({
-            path: `${ scope.paths.random_screenshots }/${ Date.now() }.jpg`,
+            path: `${ scope.paths.random_screenshots }/random-${ Date.now() }.jpg`,
             type: `jpeg`,
             fullPage: true
           });
@@ -848,7 +875,23 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
             if ( buttons_exist ) {
               await scope.continue( scope );
             }  // ends if continue exists
-          }
+          }  // ends if buttons_exist
+
+          let has_content = await scope.has_interview_content( scope );
+          if ( !has_content ) {
+
+            // Throw an error if there's no interview content
+            // Make sure to only throw once
+            if ( !has_thrown ) {
+              has_thrown = true;
+              await scope.addToReport( scope, {
+                type: `error`,
+                value: no_content_err
+              });
+              reject( no_content_err );
+            }
+
+          }  // ends if !has_content
 
           if ( buttons_exist ) {
             process.stdout.write(`\x1b[36m${ 'v' }\x1b[0m`);  // pressed button
@@ -856,11 +899,28 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
 
           resolve();
 
-        } catch ( err ) {
-          let err_msg = `The page with the "${ id }" errored when trying to fill `
-            + `it with random answers: ${ err.name }`
-          await scope.addToReport( scope, { type: `error`, value: err_msg });
-          throw( err );
+        } catch ( error ) {
+
+          // Make sure to only throw once
+          if ( has_thrown ) { return; }
+          else { has_thrown = true; }
+
+          // Throw a specific error if there's no interview content
+          let has_content = await scope.has_interview_content( scope );
+          if ( !has_content ) {
+            await scope.addToReport( scope, {
+              type: `error`,
+              value: no_content_err
+            });
+            reject( no_content_err );
+
+          // Otherwise throw a generic error
+          } else {
+            let err_msg = `The page with the question id "${ id }" errored when trying to fill`
+              + ` it with random answers: ${ error.name }`
+            await scope.addToReport( scope, { type: `error`, value: err_msg });
+            reject( error );
+          }  // ends if !has_content
 
         } finally {
           // prevent temporary hang at end of tests
@@ -1051,9 +1111,9 @@ After(async function(scenario) {
       if ( scope.page ) { await scope.page.waitForTimeout( 60 * 1000 ); }
     }
 
-    await scope.addToReport(scope, { type: `outcome`, value: `**-- Scenario Failed --**` });
     if ( scope.page ) {
-      if ( scope.disable_error_screenshot ) {
+
+      if ( !!scope.disable_error_screenshot ) {
         scope.addToReport(scope, {
           type: `row`,
           value: `For security, ALKiln will not create a screenshot for this error. It's possible a secret is being used on this screen.`
@@ -1079,11 +1139,12 @@ After(async function(scenario) {
       }  // ends if scope.disable_error_screenshot
     }  // ends if scope.page exists
 
-  }  // ends if not passed
+    // This has to come after the security message or security message doesn't print. Not sure why.
+    if (scenario.result.status === `FAILED`) {
+      await scope.addToReport(scope, { type: `outcome`, value: `**-- Scenario Failed --**` });
+    }
 
-  if (scenario.result.status === `FAILED`) {
-    await scope.addToReport(scope, { type: `outcome`, value: `**-- Scenario Failed --**` });
-  }
+  }  // ends if not passed
 
   let original_scenario_status = scenario.result.status;
 
@@ -1092,13 +1153,17 @@ After(async function(scenario) {
     let expected = scope.expected_status;
     scope.expected_status = null;  // reset for next Scenario
     expect( scenario.result.status ).to.equal( expected );
+    // If the status was as expected, pass this criteria
+    scenario.result.status = `PASSED`;
   }
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
     let expected = scope.expected_in_report;
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_in_report = null;
     let all_were_included = await scope.reportIncludesAllExpected( scope, { expected });
+
     if ( all_were_included ) { scenario.result.status = `PASSED`; }
+    else { scenario.result.status = `FAILED`; }
   }
 
   // If there is a page open, then sign out and close it

package/lib/utils/files.js

@@ -8,6 +8,7 @@ files.readable_date = function() {
   *  match the workflow artifact date format. */
   let date = new Date();  // now
   let str = date.toLocaleString('en-GB', { timeZone: 'UTC' });
+  // To properly get the year/month/day format, look at https://stackoverflow.com/a/48729396/14144258
   let fancy_str = str.replace(/\//g, '-').replace(',', ' at').replace(':', 'hrs ').replace(':', 'mins ') + 'secs UTC';
   return fancy_str;
 };  // Ends files.readable_date()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "4.7.0",
+  "version": "4.7.1",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {

package/tests/features/interactive_steps.feature

@@ -132,8 +132,3 @@ Scenario: tap element with an error
   """
   Given I start the interview at "test_taps"
   And I tap the "Tests-not_there-tab" tab
-
-@i5 @random
-Scenario: I answer randomly till the end
-  Given I start the interview at "all_tests"
-  And I answer randomly for at most 50 pages

package/tests/features/random_input.feature

@@ -3,13 +3,29 @@ Feature: Assembly Line package-specific Steps
 
 # In tag names, 'ri' is for 'random input'
 
-@slow @ri1 @random
+@fast @ri1 @random
 Scenario: I fill in random input till the end
   Given I start the interview at "test_random_input"
   And I answer randomly for at most 20 pages
   Then the question id should be "end"
 
-@slow @ri2 @random
+@fast @ri2 @random
 Scenario: I fill in random input for 1 page
   Given I start the interview at "test_random_input"
   And I answer randomly for at most 1 page
+
+# Should create two unique folders for random step screenshots which
+# must be checked manually
+@fast @ri3 @random
+Scenario: I answer randomly till the end twice
+  Given I start the interview at "test_random_input"
+  And I answer randomly for at most 50 pages
+  Given I start the interview at "test_random_input"
+  And I answer randomly for at most 50 pages
+
+@fast @ri4 @random @error
+Scenario: Fail with error page from random input
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_missing_var_error_screen"
+  And the max seconds for each step in this scenario is 10
+  And I answer randomly for at most 3 pages

package/tests/features/story_table_formats.feature

@@ -3,7 +3,6 @@ Feature: Supported story table formats
 
 NOTE:
     Unsupported formats:
-    - Arrays (see https://github.com/plocket/docassemble-cucumber/issues/257)
     - Format 1 and 2 tables (permanent) (see https://github.com/plocket/docassemble-cucumber/issues/264)
 
 NOTE:
@@ -75,3 +74,52 @@ Scenario: Table MISSING trigger column
     | single_quote_dict['single_quote_key']['sq_two'] | true |
     | text_input | Regular text input field value |
     | textarea | Multiline text\narea value |
+
+@fast @stf5
+Scenario: Table has no header row, has trigger column
+  Given I start the interview at "all_tests"
+  And I get to "showifs" with this data:
+    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
+    | checkboxes_other['checkbox_other_opt_1'] | true |  |
+    | checkboxes_other['checkbox_other_opt_2'] | true |  |
+    | checkboxes_other['checkbox_other_opt_3'] | false |  |
+    | checkboxes_yesno | True |  |
+    | dropdown_test | dropdown_opt_2 |  |
+    | radio_other | radio_other_opt_3 |  |
+    | radio_yesno | False |  |
+    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
+    | text_input | Regular text input field value |  |
+    | textarea | Multiline text\narea value |  |
+
+@fast @stf6
+Scenario: Table has no header row, MISSING trigger column
+  Given I start the interview at "all_tests"
+  And I get to "showifs" with this data:
+    | double_quote_dict["double_quote_key"]['dq_two'] | true |
+    | checkboxes_other['checkbox_other_opt_2'] | true |
+    | checkboxes_yesno | True |
+    | dropdown_test | dropdown_opt_2 |
+    | radio_other | radio_other_opt_3 |
+    | radio_yesno | False |
+    | single_quote_dict['single_quote_key']['sq_two'] | true |
+    | text_input | Regular text input field value |
+    | textarea | Multiline text\narea value |
+
+@fast @stf7
+Scenario: Fails when table has no header row and rows have only one column
+  Given the final Scenario status should be "failed"
+  And the Scenario report should include:
+  """
+  Your Story Table definition needs to be changed.
+  """
+  Given I start the interview at "all_tests"
+  And I get to "showifs" with this data:
+    | double_quote_dict["double_quote_key"]['dq_two'] |
+    | checkboxes_other['checkbox_other_opt_2'] |
+    | checkboxes_yesno |
+    | dropdown_test |
+    | radio_other |
+    | radio_yesno |
+    | single_quote_dict['single_quote_key']['sq_two'] |
+    | text_input |
+    | textarea |