@suffolklitlab/alkiln 4.7.0 → 4.8.0

package/CHANGELOG.md

@@ -40,12 +40,35 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [4.8.0] - 2022-08-30
+### Changed
+- Date for filenames. Format date in y-m-d format instead of d-m-y that it had by default. Include ms.
+
+## [4.7.2] - 2022-08-09
+### Changed
+- Inform the developer when a test failure may have been caused by a server reload. See https://github.com/SuffolkLITLab/ALKiln/issues/392.
+- Retry all failing tests once.
+
+
+## [4.7.1] - 2022-08-07
+### Changed
+- Ensure unique folder names for random input screenshots in case developer uses multiple random steps in one Scenario.
+- Ensure random input detects unexpected interview errors.
+
+### Fixed
+- Fix failure when signature page is the first screen.
+- Internal tests failing even when failure status is as expected.
+- Double printing in report of scenario failure, one of them even when Scenario was possibly non-failure, like skipped.
+
 ## [4.7.0] - 2022-07-27
 ### Added
-- Generate random input to fill out a simple form automatically
+- Generate random input to fill out a simple form automatically. Does not yet ensure that random screenshot folder names are unique, or that error screens are detected as errors. 4.7.1 implements those.
 
 ### Changed
 - Add page id to the "Missing Variable or variables on page" error report
+- Allows Story Tables to not include a header row, as long as they have 2 or 3 columns.
+  This isn't the suggested way at the moment, since including headers is more readable,
+  but can prevent confusing and unnecessary errors.
 
 ## [4.6.2] - 2022-06-28
 ### Changed

package/lib/docassemble/docassemble_api_interface.js

@@ -120,6 +120,24 @@ da_i.throw_an_error_if_server_is_not_responding = async function ( options ) {
   }
 };  // Ends da_i.throw_an_error_if_server_is_not_responding()
 
+// node -e 'require("./lib/docassemble/docassemble_api_interface.js").is_server_responding()'
+da_i.is_server_responding = async function ( dev_id_options ) {
+  /** Use _da_REST.get_dev_id() to test if server is up.
+  *    See https://docassemble.org/docs/api.html#user. */
+
+  // Default timeouts
+  let { timeout } = dev_id_options ||  { timeout: DEFAULT_MAX_REQUEST_MS };
+  try {
+    let response = await _da_REST.get_dev_id( timeout );
+    if ( response && response.data && response.data.id ) {
+      return true;
+    } else {
+      return false;
+    }
+  } catch ( error ) {
+    return false;
+  }
+};  // Ends da_i.is_server_responding();
 
 // node -e 'require("./lib/docassemble/docassemble_api_interface.js").get_dev_id()'
 da_i.get_dev_id = async function ( dev_id_options ) {

package/lib/scope.js

@@ -14,6 +14,8 @@ const time = require('./utils/time');
 const log = require('./utils/log');
 const { waitForTimeout } = require('./utils/time');
 const files = require('./utils/files' );
+const server = require('./docassemble/docassemble_api_interface');
+const wrapPromiseWithTimeout = require('./cucumber_8_shim');
 
 
 let ordinal_to_integer = {
@@ -57,6 +59,7 @@ let misc_artifacts_dir = `_alkiln_test-misc_artifacts`;
 
 module.exports = {
   trigger_not_needed_flag: `ALKiln: no trigger variable needed`,
+
   addToReport: async function( scope, { type, value }) {
     /* Add an item to a specific list in a scenario's report.
     * Create report if necessary.
@@ -191,6 +194,7 @@ module.exports = {
     let report = ``;
     for ( let line of data.lines ) {
       if ( line.type === `page_id` ) { report += `screen id: `; }
+      else if ( line.type === `outcome` ) { report += `\n`; }
       else if ( line.type === `warning` ) { report += `WARNING: `; }
       else if ( line.type === `error` ) { report += `\nERROR: ` }
       report += `${ line.value }\n`;
@@ -356,41 +360,54 @@ module.exports = {
     // This check might never be needed now as we try to match fields on the page
     // first in user-triggered cases - users used to be able to access this almost
     // directly, but they can't anymore. Our functions should know to only pass good things.
-    let msg = `Cannot find that element.`;
+    let msg = `Cannot find the element to tap.`;
     if ( !elem ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
 
     expect( elem, msg ).to.exist;
 
+    // Get the text on the button or link for a potential error message later
+    let tapperText = await (await elem.getProperty('textContent')).jsonValue();
+
     let start_url = await scope.page.url();  // so we can later check for navigation
 
-    let clickWait;
-    // Can't use elem[ scope.activate ](). It works locally, but errors on GitHub
-    // with "Node is not visible or not an HTMLElement". We're not using a custom Chromium version
-    // Unfortunately, this won't catch an invisible element. Hopefully we'd catch it before now...?
-    // "Tap" on mobile is more complex to implement ourselves.
-    // See https://stackoverflow.com/a/56547605/14144258
-    // and other convos around it. We haven't made it public that the device
-    // can customized. Until we do that, we'll just use "click" all the time.
-    if (!tabToWaitFor) {
-      // Click with no navigation will end immediately
-      clickWait = Promise.all([
-        elem.evaluate( (el) => { return el.click(); }),
-        scope.page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
-      ]);
-    } else {
-      // Same thing as above, but the promise doesn't wait for navigation.
-      clickWait = elem.evaluate( (el) => {return el.click(); });
+    // We'll wait for the click itself at the very least
+    let click_promises = [ elem.evaluate( (el) => { return el.click(); }) ];
+    
+    // If we're possibly navigating, wait for navigation as well
+    if ( !tabToWaitFor ) {
+      click_promises.push( scope.page.waitForNavigation({ waitUntil: 'domcontentloaded' }) );
     }
-    // Error loads page, so no need to detect to keep things short
-    let winner = await Promise.race([
-      clickWait,
-      // This is meant to detect user-visible alerts/errors,
-      // not breaking errors in the testing code.
-      scope.page.waitForSelector('.alert-danger', { visible: true }),
-      scope.page.waitForSelector('.da-has-error', { visible: true }),
-    ])
-
-    // TODO: Not sure how to detect navigation landing at same page.
+    let clickWait = Promise.all( click_promises );
+
+    let winner = null;
+
+    // TODO: implement and use scope.handle_possible_timeout()
+    try {
+      // Error loads page, so no need to detect to keep things short
+      // Also, puppeteer will ensure proper timeout.
+      winner = await Promise.race([
+        clickWait,
+        // This is meant to detect user-visible alerts/errors,
+        // not breaking errors in the testing code.
+        scope.page.waitForSelector('.alert-danger', { visible: true }),
+        scope.page.waitForSelector('.da-has-error', { visible: true }),
+      ]);
+    } catch ( error ) {
+
+      if ( error.name === `TimeoutError` ) {
+        let non_reload_report_msg = `ALKiln tapped "${ tapperText }", but the interview took to long to finish.`;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+      } else {
+        // Throw any non-timeout error
+        let err_msg = `Error occurred when ALKiln tried to tap "${ tapperText }".`
+        scope.addToReport( scope, { type: `error`, value: err_msg });
+        throw error;
+      }  // ends if error is timeout error
+
+    }  // ends try/catch
+
+    // TODO: Not sure how to detect navigation landing at same page, such as
+    // with a user input validation error. Does that count as navigating?
     // Maybe url change + trigger var change? Maybe just trigger var?
     let end_url = await scope.page.url();
     let navigated = start_url !== end_url;
@@ -413,6 +430,156 @@ module.exports = {
     return winner;
   },  // Ends scope.tapElement()
 
+  // Handle server reload errors
+  server_statuses: [],
+  server_check_interval_ms: 1 * 1000,
+  stop_tracking_server_status: false,
+  track_server_status: async function( scope ) {
+    /** Updates server_statuses list by checking if server is responding until,
+    *    eventually, it is told to stop. */
+    let server_check_start_time = Date.now();
+
+    while ( !scope.stop_tracking_server_status ) {
+      // This code assumes a reasonable response time would be n seconds if the
+      // server is up and that a server reload/restart won't take less than n seconds.
+      // Also, it's non-blocking, so it doesn't slow down tests
+      let is_up = await server.is_server_responding({ timeout: scope.server_check_interval_ms });
+      // Add status to the beginning of the list to make truncating easier
+      scope.server_statuses.unshift( is_up );
+
+      // Ensure we wait longer if a full second hasn't yet passed
+      let ms_passed = Date.now() - server_check_start_time;
+      // Use Math.max to ensure no negative numbers
+      let ms_remaining = Math.max( 0, scope.server_check_interval_ms - ms_passed );
+      await time.waitForTimeout( ms_remaining );
+      // Prepare for next check
+      server_check_start_time = Date.now()
+
+      // To make it easier to check past server statuses, remove defunct server statuses
+      let max_length = Math.round( scope.timeout/scope.server_check_interval_ms );
+      // Trying to extract more items than the actual length is ok
+      scope.server_statuses = scope.server_statuses.slice( 0, max_length );
+    }  // ends while !scope.stop_tracking_server_status
+    
+  },  // Ends scope.track_server_status()
+
+  did_server_reload: async function( scope ) {
+    return scope.server_statuses.includes( false );
+  },  // Ends scope.did_server_reload()
+
+  // handle_possible_timeout: async function( scope, {
+  //   promise: ,
+  //   non_timeout_msg: ,
+  //   normal_timeout_msg: ,
+  //   actual_error: ,
+  // }) {},  // Ends scope.handle_possible_timeout()
+
+  handle_page_timeout_error: async function( scope, { non_reload_report_msg, error } ) {
+    /* For a page timeout error, handles server reload vs. other timeout errors
+    
+    @param options.non_reload_report_msg { str } Non-reload message to add to
+        the report as an error.
+    @param options.error { object | str } Error to throw. It can be the actual
+        error object or a string that will show up as a message in the error.
+    */
+    // If the server was reloading during that time, wait for the server to
+    // respond again so the next test has a better chance of passing. Then
+    // throw an error. The command-line script will re-try the Scenario after.
+    let server_reloaded_during_interval = await scope.did_server_reload( scope );
+    if ( server_reloaded_during_interval ) {
+      // Wait for the server to reload just to make sure the next test
+      // doesn't also fail because of this same server reload.
+      await scope.get_server_reload_promise( scope );
+    } else {
+      // Otherwise just throw the error
+      await scope.addToReport( scope, {
+        type: `error`,
+        value: non_reload_report_msg });
+      throw error;
+    }
+  },  // Ends scope.handle_page_timeout_error()
+
+  get_server_reload_promise: async function( scope ) {
+    /** If the promise already exists in state, return it because it can
+    *    be used anywhere with `await`. Otherwise, make the promise, store
+    *    it in state, and return that.
+    */
+    if ( scope.server_reload_promise !== null ) {
+      // Return the existing promise of the func that will check on reloading the server
+      return scope.server_reload_promise
+
+    } else {
+      // Create and return the promise of the func that will check on reloading the server
+      scope.server_reload_promise = scope.wait_for_server_response( scope );
+      return scope.server_reload_promise;
+    }
+  },  // Ends scope.get_server_reload_promise()
+
+  wait_for_server_response: async function( scope ) {
+    /** Wait for the server to respond using a multiple of scope.timeout
+    *    (or a custom timeout?) so the tests don't fail just because the
+    *    server isn't responding. Should this take an argument of an amount
+    *    of time to check or just assume we're using scope.timeout?
+    *
+    * This should only be called if the server failed to respond at some point.
+    * 
+    * This currently assumes we're either using scope.timeout as basis for
+    * the amount of time in which to detect a finished reload or that
+    * there's a test input that defines a different timeout.
+    */
+    
+    let ms_till_server_timeout;
+    if ( session_vars.get_server_reload_timeout() !== null ) {
+      // Should we require the expected value to be seconds instead of ms?
+      ms_till_server_timeout = parseInt( session_vars.get_server_reload_timeout() );
+    } else {
+      // Default server timeout will always be based on scope.timeout even if it changes
+      ms_till_server_timeout = 5 * scope.timeout;
+    }
+
+    let throw_after_server_reloads = async function () {
+
+      // Continue waiting until server has responded or until it's taken too long
+      let time_accumulator = 0;
+      let max = ms_till_server_timeout/scope.server_check_interval_ms
+
+      while ( scope.server_statuses[0] !== true && time_accumulator < max ) {
+        await time.waitForTimeout( scope.server_check_interval_ms );
+        time_accumulator += 1;
+      }
+
+      // If the server hasn't responded yet even now, let the developer know
+      if ( time_accumulator >= max ) {
+        let timeout_message = `The server took longer than `
+          + `${ Math.round( ms_till_server_timeout/1000 ) } seconds to respond. `
+          + `If the server was just reloading and needed more time, try setting the `
+          + `"SERVER_RELOAD_TIMEOUT_SECONDS" GitHub secret to a longer amount of time.`
+        await scope.addToReport( scope, { type: `error`, value: timeout_message });
+      }
+
+      // Once server has responded, throw an error to end the test. After the first
+      // failure, the whole Scenario should retry because of command line flag. Hopefully
+      // it'll get through the next time. We considered trying to redo the input on the
+      // page instead of re-doing the whole test, but that wouldn't work - we don't
+      // know how many Steps to back to do that and we can't redo Steps anyway.
+      await scope.throw_server_was_reloading_error( scope )
+    };
+
+    await throw_after_server_reloads();
+
+  },  // Ends scope.wait_for_server_response()
+
+  throw_server_was_reloading_error: async function( scope ) {//, { server_timeout_id, timeout_message } ) {
+    /* Throw the server reload error with appropriate side effects. */
+    let reload_failure_msg = `The test was unable to continue because the `
+      + `server was reloading. ALKiln will try this Scenario a total of 2 times. An `
+      + `error will still be printed for this attempt.`;
+    await scope.addToReport( scope, { type: `error`, value: reload_failure_msg });
+    // await scope.addToReport( scope, { type: `error`, value: timeout_message });
+    // clearTimeout( server_timeout_id );
+    throw( reload_failure_msg );
+  },  // Ends scope.throw_server_was_reloading_error()
+
   detectDownloadComplete: async function detectDownloadComplete(scope, endTime) {
     /* Resolve if a download flag has changed. Timeout after a bit less than
     *    the global permitted timeout
@@ -585,18 +752,36 @@ module.exports = {
     /* Try to load the page. Should we also pass a `timeout` arg or just
     *    use scope.timeout? */
 
-    // If there is no browser open, start a new one
-    if (!scope.browser) {
-      scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() });
-    }
-    if ( !scope.page ) { scope.page = await scope.browser.newPage(); }
-
     if ( !file_name.includes('.yml') ) { file_name = `${ file_name }.yml` }
     let base_url = await get_base_interview_url();
     let interview_url = `${ base_url }${ file_name }`;
     if ( session_vars.get_debug() ) { console.log( interview_url ); }
 
-    await scope.page.goto(interview_url, { waitUntil: 'domcontentloaded', timeout: scope.timeout })
+    if ( !scope.page ) { scope.page = await scope.browser.newPage(); }
+
+    // TODO: implement and use scope.handle_possible_timeout()
+    try {
+
+      await scope.page.goto(interview_url, { waitUntil: 'domcontentloaded', timeout: scope.timeout })
+
+    } catch ( error ) {
+
+      if ( error.name === `TimeoutError` ) {
+
+        let non_reload_report_msg = `Timeout error occurred when ALKiln tried to go to the interview url "${ interview_url }".`;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+
+      } else {
+
+        // Throw any non-timeout error
+        scope.addToReport( scope, {
+          type: `error`,
+          value: `Error occurred when ALKiln tried to go to the interview url "${ interview_url }".`
+        });
+        throw( error );
+      }  // ends if error is timeout error
+
+    }  // ends try/catch
 
     let load_result = await scope.getLoadData( scope );
     // Errors, or reaching timeout, causes an exception and is shown to the developer at the end
@@ -617,6 +802,9 @@ module.exports = {
     let result = { error: ``, elem: null };
 
     let winner = await Promise.race([
+      // Signature page
+      scope.page.waitForSelector( `#dasigpage`, { timeout: scope.timeout }),
+      // Any other valid page
       scope.page.waitForSelector( `#daMainQuestion`, { timeout: scope.timeout }),
       scope.page.waitForSelector( `#da-retry`, { timeout: scope.timeout }),
     ]);
@@ -626,10 +814,14 @@ module.exports = {
     let error_id_elem = await scope.page.$( `#da-retry` );
     if ( error_id_elem ) {
       let error_elem = await scope.page.$( `blockquote` );
-      let error_handle = await error_elem.getProperty( `textContent` );
-      let system_error_text = await error_handle.jsonValue();
+      if (error_elem !== null) {
+        let error_handle = await error_elem.getProperty( `textContent` );
+        let system_error_text = await error_handle.jsonValue();
 
-      result.error = system_error_text;
+        result.error = system_error_text;
+      } else {
+        result.error = "Docassemble ran into an error on the page, but ALKiln does not know what the error is. If a screenshot was taken, check the screenshot";
+      }
     }
 
     return result;
@@ -650,9 +842,40 @@ module.exports = {
     }
   },  // Ends scope.setLanguage()
 
-  normalizeTable: async function ( scope, { var_data, from_story_table=true }) {
-    /* Return data derived from cucumber variable-setting data.
-     * Table must have headers. Arrays are not supported. */
+  normalizeTable: async function ( scope, { var_data=null, raw_var_data=null }) {
+    /* Return data derived from cucumber variable-setting data, from either an object format
+     * (var_data) or an array format (raw_var_data). `var_data` only is used when calling
+     * normalizeTable internally.
+     */
+
+    if (!var_data && ! raw_var_data) {
+      let no_data_msg = `ALKiln Internal error: normalizeTable called w/o var_data or raw_var_data:` +
+          ` open an issue at https://github.com/SuffolkLITLab/ALKiln/issues/new if you see this!`;
+      await scope.addToReport( scope, { type: `error`, value: no_data_msg });
+      throw Error(no_data_msg);
+    }
+
+    // Always use the raw var data if present; it has the full first row
+    if (raw_var_data) {
+      var_data = raw_var_data.hashes();
+      if (!('var' in var_data[0]) || !('value' in var_data[0])) {
+        let raw_array = raw_var_data.raw();
+        var_data = [];
+        for (let row_raw of raw_array) {
+          if (row_raw.length < 2) {
+            let bad_table_msg = `Your Story Table definition needs to be changed. The best practice`
+                + ` is to make your table 3 columns wide with a first row in this format: "| var | value | trigger |".`
+                + ` The first row of your table is actually "${ raw_array[0] }"`;
+            await scope.addToReport( scope, { type: `error`, value: bad_table_msg})
+            throw Error(bad_table_msg);
+          } else if (row_raw.length == 2) {
+            var_data.push({var: row_raw[0], value: row_raw[1]});
+          } else {
+            var_data.push({var: row_raw[0], value: row_raw[1], trigger: row_raw[2]});
+          }
+        }
+      }
+    }
 
     // Support tables with no 'trigger' column
     let supported_table = [];
@@ -1518,7 +1741,9 @@ module.exports = {
   },  // ends scope.enter_answer
 
   uploadFiles: async function ( scope, { handle, answer }) {
-    /* Try to upload one or more files to a file-type input field. */
+    /* Try to upload one or more files to a file-type input field.
+    *    Does not submit, so no danger of timeout with server reload.
+    */
     
     // Tests could be in one of two places. Figure out which directory they're in
     // Makes an absolute path from the directory in which `npm run` was used plus our internal folders
@@ -1934,6 +2159,20 @@ module.exports = {
     }
   },  // Ends scope.getThereIsAnotherValue()
 
+  // throw_if_no_interview_content()?
+  has_interview_content: async function ( scope ) {
+    /** Return false if ther's no element indicating a da interview question,
+    *    like a signature page, other fields, or interview text. Assumes that page has already
+    *    finished navigating. */
+    let elem_da_question = await scope.page.$( `#daMainQuestion` );
+    let elem_da_signature = await scope.page.$( `#dasigpage` );
+    if ( elem_da_question === null && elem_da_signature === null ) {
+      return false;
+    } else {
+      return true;
+    }
+  },  // Ends scope.has_interview_content()
+
   waitUntilContinued: async function ( scope, { url = '', id = '' }) {
     /** Given a url, attempt to continue until the page url changes or an error
         is found. Return the error object no matter what. */
@@ -2057,12 +2296,14 @@ module.exports = {
       let actual_url = await prop_obj.jsonValue();
 
       let linkPage = await scope.browser.newPage();
+      // This should not check for a server reload timeout error - it could be going anywhere.
       let response = await linkPage.goto( actual_url, { waitUntil: 'domcontentloaded' });
 
       let msg = `The "${ linkText }" link is broken.`;
       if ( !response.ok() ) { await scope.addToReport( scope, { type: `error`, value: msg }); }
       expect( response.ok(), msg ).to.be.true;
 
+      // This may cause problems if the linked page is the interview page
       linkPage.close()
 
       await scope.afterStep(scope);
@@ -2111,7 +2352,7 @@ module.exports = {
       if ( !link ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
       expect( link, msg ).to.exist;
 
-      await link.evaluate( elem => { return elem.click(); });
+      await scope.tapElement( scope, link );
 
       await scope.afterStep(scope, {waitForShowIf: true});
     },  // Ends tap_term()
@@ -2139,6 +2380,7 @@ module.exports = {
       expect( elem, msg ).to.exist;
 
       scope.toDownload = filename;
+      // Should this be using `scope.tapElement()`?
       await elem.evaluate( elem => { return elem.click(); });
       await scope.detectDownloadComplete( scope );
     },  // Ends download()
@@ -2231,8 +2473,26 @@ module.exports = {
 
       // Go to the sign in page
       let login_url = `${ session_vars.get_da_server_url() }/user/sign-in`;
-      await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout })
-      await scope.page.waitForSelector( `.dabody` );
+      
+      // TODO: implement and use scope.handle_possible_timeout()
+      try {
+        // puppeteer will ensure proper timeout.
+        await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout });
+        await scope.page.waitForSelector( `.dabody` );
+
+      } catch ( error ) {
+
+        let err_msg = `Error occurred when ALKiln tried to go to "${ login_url }".`
+        if ( error.name === `TimeoutError` ) {
+          let non_reload_report_msg = `It took too long to load "${ login_url }"`;
+          await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+        } else {
+          // Throw any non-timeout error
+          scope.addToReport( scope, { type: `error`, value: err_msg });
+          throw error;
+        }  // ends if error is timeout error
+
+      }  // ends try/catch
 
       let email = process.env[ email_secret_name ];
       let password = process.env[ password_secret_name ];
@@ -2248,13 +2508,15 @@ module.exports = {
       if ( password === undefined ) {
         await scope.addToReport( scope, { type: `error`, value: password_msg })
       }
+
       expect( email, email_msg ).to.not.equal( undefined );
       expect( password, password_msg ).to.not.equal( undefined );
 
       // Try to log in
       await scope.page.type( `#email`, email );
       await scope.page.type( `#password`, password );
-      await scope.page.$eval( `button[name="submit"]`, (elem) => { return elem.click(); });
+      let elem = await scope.page.$( `button[name="submit"]` );
+      await scope.tapElement( scope, elem );
 
       // Wait to see the success message.
       // This is a bit of an assumption and needs success and failure testing
@@ -2279,7 +2541,6 @@ module.exports = {
 
       // For every field on the page
       for ( let field of fields ) {
-
         // fields[n].tag is HTML tag of node, as a string
 
         // If the field is a button, save it in a list of buttons that we'll deal with later
@@ -2406,7 +2667,7 @@ module.exports = {
           if ( type === `file` ) {
             await scope.addToReport( scope, {
               type: `warning`,
-              value: `Sorry, ALKiln's random-input cannot yet handle "${ type }" fields.`
+              value: `Sorry, ALKiln's random input tests cannot yet handle "${ type }" fields.`
             });
           } else {
             let answer = await scope.get_random_input_for[ type ]( scope );