@suffolklitlab/alkiln 4.7.1 → 4.7.2

package/CHANGELOG.md

@@ -40,6 +40,12 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [4.7.2] - 2022-08-09
+### Changed
+- Inform the developer when a test failure may have been caused by a server reload. See https://github.com/SuffolkLITLab/ALKiln/issues/392.
+- Retry all failing tests once.
+
+
 ## [4.7.1] - 2022-08-07
 ### Changed
 - Ensure unique folder names for random input screenshots in case developer uses multiple random steps in one Scenario.

package/lib/docassemble/docassemble_api_interface.js

@@ -120,6 +120,24 @@ da_i.throw_an_error_if_server_is_not_responding = async function ( options ) {
   }
 };  // Ends da_i.throw_an_error_if_server_is_not_responding()
 
+// node -e 'require("./lib/docassemble/docassemble_api_interface.js").is_server_responding()'
+da_i.is_server_responding = async function ( dev_id_options ) {
+  /** Use _da_REST.get_dev_id() to test if server is up.
+  *    See https://docassemble.org/docs/api.html#user. */
+
+  // Default timeouts
+  let { timeout } = dev_id_options ||  { timeout: DEFAULT_MAX_REQUEST_MS };
+  try {
+    let response = await _da_REST.get_dev_id( timeout );
+    if ( response && response.data && response.data.id ) {
+      return true;
+    } else {
+      return false;
+    }
+  } catch ( error ) {
+    return false;
+  }
+};  // Ends da_i.is_server_responding();
 
 // node -e 'require("./lib/docassemble/docassemble_api_interface.js").get_dev_id()'
 da_i.get_dev_id = async function ( dev_id_options ) {

package/lib/scope.js

@@ -14,6 +14,8 @@ const time = require('./utils/time');
 const log = require('./utils/log');
 const { waitForTimeout } = require('./utils/time');
 const files = require('./utils/files' );
+const server = require('./docassemble/docassemble_api_interface');
+const wrapPromiseWithTimeout = require('./cucumber_8_shim');
 
 
 let ordinal_to_integer = {
@@ -57,6 +59,7 @@ let misc_artifacts_dir = `_alkiln_test-misc_artifacts`;
 
 module.exports = {
   trigger_not_needed_flag: `ALKiln: no trigger variable needed`,
+
   addToReport: async function( scope, { type, value }) {
     /* Add an item to a specific list in a scenario's report.
     * Create report if necessary.
@@ -362,36 +365,49 @@ module.exports = {
 
     expect( elem, msg ).to.exist;
 
+    // Get the text on the button or link for a potential error message later
+    let tapperText = await (await elem.getProperty('textContent')).jsonValue();
+
     let start_url = await scope.page.url();  // so we can later check for navigation
 
-    let clickWait;
-    // Can't use elem[ scope.activate ](). It works locally, but errors on GitHub
-    // with "Node is not visible or not an HTMLElement". We're not using a custom Chromium version
-    // Unfortunately, this won't catch an invisible element. Hopefully we'd catch it before now...?
-    // "Tap" on mobile is more complex to implement ourselves.
-    // See https://stackoverflow.com/a/56547605/14144258
-    // and other convos around it. We haven't made it public that the device
-    // can customized. Until we do that, we'll just use "click" all the time.
-    if (!tabToWaitFor) {
-      // Click with no navigation will end immediately
-      clickWait = Promise.all([
-        elem.evaluate( (el) => { return el.click(); }),
-        scope.page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
-      ]);
-    } else {
-      // Same thing as above, but the promise doesn't wait for navigation.
-      clickWait = elem.evaluate( (el) => {return el.click(); });
+    // We'll wait for the click itself at the very least
+    let click_promises = [ elem.evaluate( (el) => { return el.click(); }) ];
+    
+    // If we're possibly navigating, wait for navigation as well
+    if ( !tabToWaitFor ) {
+      click_promises.push( scope.page.waitForNavigation({ waitUntil: 'domcontentloaded' }) );
     }
-    // Error loads page, so no need to detect to keep things short
-    let winner = await Promise.race([
-      clickWait,
-      // This is meant to detect user-visible alerts/errors,
-      // not breaking errors in the testing code.
-      scope.page.waitForSelector('.alert-danger', { visible: true }),
-      scope.page.waitForSelector('.da-has-error', { visible: true }),
-    ])
-
-    // TODO: Not sure how to detect navigation landing at same page.
+    let clickWait = Promise.all( click_promises );
+
+    let winner = null;
+
+    // TODO: implement and use scope.handle_possible_timeout()
+    try {
+      // Error loads page, so no need to detect to keep things short
+      // Also, puppeteer will ensure proper timeout.
+      winner = await Promise.race([
+        clickWait,
+        // This is meant to detect user-visible alerts/errors,
+        // not breaking errors in the testing code.
+        scope.page.waitForSelector('.alert-danger', { visible: true }),
+        scope.page.waitForSelector('.da-has-error', { visible: true }),
+      ]);
+    } catch ( error ) {
+
+      if ( error.name === `TimeoutError` ) {
+        let non_reload_report_msg = `ALKiln tapped "${ tapperText }", but the interview took to long to finish.`;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+      } else {
+        // Throw any non-timeout error
+        let err_msg = `Error occurred when ALKiln tried to tap "${ tapperText }".`
+        scope.addToReport( scope, { type: `error`, value: err_msg });
+        throw error;
+      }  // ends if error is timeout error
+
+    }  // ends try/catch
+
+    // TODO: Not sure how to detect navigation landing at same page, such as
+    // with a user input validation error. Does that count as navigating?
     // Maybe url change + trigger var change? Maybe just trigger var?
     let end_url = await scope.page.url();
     let navigated = start_url !== end_url;
@@ -414,6 +430,156 @@ module.exports = {
     return winner;
   },  // Ends scope.tapElement()
 
+  // Handle server reload errors
+  server_statuses: [],
+  server_check_interval_ms: 1 * 1000,
+  stop_tracking_server_status: false,
+  track_server_status: async function( scope ) {
+    /** Updates server_statuses list by checking if server is responding until,
+    *    eventually, it is told to stop. */
+    let server_check_start_time = Date.now();
+
+    while ( !scope.stop_tracking_server_status ) {
+      // This code assumes a reasonable response time would be n seconds if the
+      // server is up and that a server reload/restart won't take less than n seconds.
+      // Also, it's non-blocking, so it doesn't slow down tests
+      let is_up = await server.is_server_responding({ timeout: scope.server_check_interval_ms });
+      // Add status to the beginning of the list to make truncating easier
+      scope.server_statuses.unshift( is_up );
+
+      // Ensure we wait longer if a full second hasn't yet passed
+      let ms_passed = Date.now() - server_check_start_time;
+      // Use Math.max to ensure no negative numbers
+      let ms_remaining = Math.max( 0, scope.server_check_interval_ms - ms_passed );
+      await time.waitForTimeout( ms_remaining );
+      // Prepare for next check
+      server_check_start_time = Date.now()
+
+      // To make it easier to check past server statuses, remove defunct server statuses
+      let max_length = Math.round( scope.timeout/scope.server_check_interval_ms );
+      // Trying to extract more items than the actual length is ok
+      scope.server_statuses = scope.server_statuses.slice( 0, max_length );
+    }  // ends while !scope.stop_tracking_server_status
+    
+  },  // Ends scope.track_server_status()
+
+  did_server_reload: async function( scope ) {
+    return scope.server_statuses.includes( false );
+  },  // Ends scope.did_server_reload()
+
+  // handle_possible_timeout: async function( scope, {
+  //   promise: ,
+  //   non_timeout_msg: ,
+  //   normal_timeout_msg: ,
+  //   actual_error: ,
+  // }) {},  // Ends scope.handle_possible_timeout()
+
+  handle_page_timeout_error: async function( scope, { non_reload_report_msg, error } ) {
+    /* For a page timeout error, handles server reload vs. other timeout errors
+    
+    @param options.non_reload_report_msg { str } Non-reload message to add to
+        the report as an error.
+    @param options.error { object | str } Error to throw. It can be the actual
+        error object or a string that will show up as a message in the error.
+    */
+    // If the server was reloading during that time, wait for the server to
+    // respond again so the next test has a better chance of passing. Then
+    // throw an error. The command-line script will re-try the Scenario after.
+    let server_reloaded_during_interval = await scope.did_server_reload( scope );
+    if ( server_reloaded_during_interval ) {
+      // Wait for the server to reload just to make sure the next test
+      // doesn't also fail because of this same server reload.
+      await scope.get_server_reload_promise( scope );
+    } else {
+      // Otherwise just throw the error
+      await scope.addToReport( scope, {
+        type: `error`,
+        value: non_reload_report_msg });
+      throw error;
+    }
+  },  // Ends scope.handle_page_timeout_error()
+
+  get_server_reload_promise: async function( scope ) {
+    /** If the promise already exists in state, return it because it can
+    *    be used anywhere with `await`. Otherwise, make the promise, store
+    *    it in state, and return that.
+    */
+    if ( scope.server_reload_promise !== null ) {
+      // Return the existing promise of the func that will check on reloading the server
+      return scope.server_reload_promise
+
+    } else {
+      // Create and return the promise of the func that will check on reloading the server
+      scope.server_reload_promise = scope.wait_for_server_response( scope );
+      return scope.server_reload_promise;
+    }
+  },  // Ends scope.get_server_reload_promise()
+
+  wait_for_server_response: async function( scope ) {
+    /** Wait for the server to respond using a multiple of scope.timeout
+    *    (or a custom timeout?) so the tests don't fail just because the
+    *    server isn't responding. Should this take an argument of an amount
+    *    of time to check or just assume we're using scope.timeout?
+    *
+    * This should only be called if the server failed to respond at some point.
+    * 
+    * This currently assumes we're either using scope.timeout as basis for
+    * the amount of time in which to detect a finished reload or that
+    * there's a test input that defines a different timeout.
+    */
+    
+    let ms_till_server_timeout;
+    if ( session_vars.get_server_reload_timeout() !== null ) {
+      // Should we require the expected value to be seconds instead of ms?
+      ms_till_server_timeout = parseInt( session_vars.get_server_reload_timeout() );
+    } else {
+      // Default server timeout will always be based on scope.timeout even if it changes
+      ms_till_server_timeout = 5 * scope.timeout;
+    }
+
+    let throw_after_server_reloads = async function () {
+
+      // Continue waiting until server has responded or until it's taken too long
+      let time_accumulator = 0;
+      let max = ms_till_server_timeout/scope.server_check_interval_ms
+
+      while ( scope.server_statuses[0] !== true && time_accumulator < max ) {
+        await time.waitForTimeout( scope.server_check_interval_ms );
+        time_accumulator += 1;
+      }
+
+      // If the server hasn't responded yet even now, let the developer know
+      if ( time_accumulator >= max ) {
+        let timeout_message = `The server took longer than `
+          + `${ Math.round( ms_till_server_timeout/1000 ) } seconds to respond. `
+          + `If the server was just reloading and needed more time, try setting the `
+          + `"SERVER_RELOAD_TIMEOUT_SECONDS" GitHub secret to a longer amount of time.`
+        await scope.addToReport( scope, { type: `error`, value: timeout_message });
+      }
+
+      // Once server has responded, throw an error to end the test. After the first
+      // failure, the whole Scenario should retry because of command line flag. Hopefully
+      // it'll get through the next time. We considered trying to redo the input on the
+      // page instead of re-doing the whole test, but that wouldn't work - we don't
+      // know how many Steps to back to do that and we can't redo Steps anyway.
+      await scope.throw_server_was_reloading_error( scope )
+    };
+
+    await throw_after_server_reloads();
+
+  },  // Ends scope.wait_for_server_response()
+
+  throw_server_was_reloading_error: async function( scope ) {//, { server_timeout_id, timeout_message } ) {
+    /* Throw the server reload error with appropriate side effects. */
+    let reload_failure_msg = `The test was unable to continue because the `
+      + `server was reloading. ALKiln will try this Scenario a total of 2 times. An `
+      + `error will still be printed for this attempt.`;
+    await scope.addToReport( scope, { type: `error`, value: reload_failure_msg });
+    // await scope.addToReport( scope, { type: `error`, value: timeout_message });
+    // clearTimeout( server_timeout_id );
+    throw( reload_failure_msg );
+  },  // Ends scope.throw_server_was_reloading_error()
+
   detectDownloadComplete: async function detectDownloadComplete(scope, endTime) {
     /* Resolve if a download flag has changed. Timeout after a bit less than
     *    the global permitted timeout
@@ -586,18 +752,36 @@ module.exports = {
     /* Try to load the page. Should we also pass a `timeout` arg or just
     *    use scope.timeout? */
 
-    // If there is no browser open, start a new one
-    if (!scope.browser) {
-      scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() });
-    }
-    if ( !scope.page ) { scope.page = await scope.browser.newPage(); }
-
     if ( !file_name.includes('.yml') ) { file_name = `${ file_name }.yml` }
     let base_url = await get_base_interview_url();
     let interview_url = `${ base_url }${ file_name }`;
     if ( session_vars.get_debug() ) { console.log( interview_url ); }
 
-    await scope.page.goto(interview_url, { waitUntil: 'domcontentloaded', timeout: scope.timeout })
+    if ( !scope.page ) { scope.page = await scope.browser.newPage(); }
+
+    // TODO: implement and use scope.handle_possible_timeout()
+    try {
+
+      await scope.page.goto(interview_url, { waitUntil: 'domcontentloaded', timeout: scope.timeout })
+
+    } catch ( error ) {
+
+      if ( error.name === `TimeoutError` ) {
+
+        let non_reload_report_msg = `Timeout error occurred when ALKiln tried to go to the interview url "${ interview_url }".`;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+
+      } else {
+
+        // Throw any non-timeout error
+        scope.addToReport( scope, {
+          type: `error`,
+          value: `Error occurred when ALKiln tried to go to the interview url "${ interview_url }".`
+        });
+        throw( error );
+      }  // ends if error is timeout error
+
+    }  // ends try/catch
 
     let load_result = await scope.getLoadData( scope );
     // Errors, or reaching timeout, causes an exception and is shown to the developer at the end
@@ -1557,7 +1741,9 @@ module.exports = {
   },  // ends scope.enter_answer
 
   uploadFiles: async function ( scope, { handle, answer }) {
-    /* Try to upload one or more files to a file-type input field. */
+    /* Try to upload one or more files to a file-type input field.
+    *    Does not submit, so no danger of timeout with server reload.
+    */
     
     // Tests could be in one of two places. Figure out which directory they're in
     // Makes an absolute path from the directory in which `npm run` was used plus our internal folders
@@ -2110,12 +2296,14 @@ module.exports = {
       let actual_url = await prop_obj.jsonValue();
 
       let linkPage = await scope.browser.newPage();
+      // This should not check for a server reload timeout error - it could be going anywhere.
       let response = await linkPage.goto( actual_url, { waitUntil: 'domcontentloaded' });
 
       let msg = `The "${ linkText }" link is broken.`;
       if ( !response.ok() ) { await scope.addToReport( scope, { type: `error`, value: msg }); }
       expect( response.ok(), msg ).to.be.true;
 
+      // This may cause problems if the linked page is the interview page
       linkPage.close()
 
       await scope.afterStep(scope);
@@ -2164,7 +2352,7 @@ module.exports = {
       if ( !link ) { await scope.addToReport(scope, { type: `error`, value: msg }); }
       expect( link, msg ).to.exist;
 
-      await link.evaluate( elem => { return elem.click(); });
+      await scope.tapElement( scope, link );
 
       await scope.afterStep(scope, {waitForShowIf: true});
     },  // Ends tap_term()
@@ -2192,6 +2380,7 @@ module.exports = {
       expect( elem, msg ).to.exist;
 
       scope.toDownload = filename;
+      // Should this be using `scope.tapElement()`?
       await elem.evaluate( elem => { return elem.click(); });
       await scope.detectDownloadComplete( scope );
     },  // Ends download()
@@ -2284,8 +2473,26 @@ module.exports = {
 
       // Go to the sign in page
       let login_url = `${ session_vars.get_da_server_url() }/user/sign-in`;
-      await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout })
-      await scope.page.waitForSelector( `.dabody` );
+      
+      // TODO: implement and use scope.handle_possible_timeout()
+      try {
+        // puppeteer will ensure proper timeout.
+        await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout });
+        await scope.page.waitForSelector( `.dabody` );
+
+      } catch ( error ) {
+
+        let err_msg = `Error occurred when ALKiln tried to go to "${ login_url }".`
+        if ( error.name === `TimeoutError` ) {
+          let non_reload_report_msg = `It took too long to load "${ login_url }"`;
+          await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+        } else {
+          // Throw any non-timeout error
+          scope.addToReport( scope, { type: `error`, value: err_msg });
+          throw error;
+        }  // ends if error is timeout error
+
+      }  // ends try/catch
 
       let email = process.env[ email_secret_name ];
       let password = process.env[ password_secret_name ];
@@ -2301,13 +2508,15 @@ module.exports = {
       if ( password === undefined ) {
         await scope.addToReport( scope, { type: `error`, value: password_msg })
       }
+
       expect( email, email_msg ).to.not.equal( undefined );
       expect( password, password_msg ).to.not.equal( undefined );
 
       // Try to log in
       await scope.page.type( `#email`, email );
       await scope.page.type( `#password`, password );
-      await scope.page.$eval( `button[name="submit"]`, (elem) => { return elem.click(); });
+      let elem = await scope.page.$( `button[name="submit"]` );
+      await scope.tapElement( scope, elem );
 
       // Wait to see the success message.
       // This is a bit of an assumption and needs success and failure testing

package/lib/steps.js

@@ -24,6 +24,13 @@ const files = require('./utils/files' );
 
 /* Of Note:
 - We're using `*=` for selectors because sometimes da text has funny characters in it that are hard to anticipate
+- Can't use elem[ scope.activate ](). It works locally, but errors on GitHub
+  with "Node is not visible or not an HTMLElement". We're not using a custom Chromium version
+  Unfortunately, this won't catch an invisible element. Hopefully we'd catch it before click
+  attempts are made. "Tap" on mobile is more complex to implement ourselves.
+  See https://stackoverflow.com/a/56547605/14144258
+  and other convos around it. We haven't made it public that the device
+  can customized. Until we do that, we'll just use "click" all the time.
 */
 
 
@@ -61,7 +68,8 @@ regex thoughts: https://stackoverflow.com/questions/171480/regex-grabbing-values
 
 // Hoping that most load and download times will be under 30 seconds.
 // TODO: Check what average load and download times are in our current interviews.
-setDefaultTimeout( 30 * 1000 );
+let default_timeout = 30 * 1000
+setDefaultTimeout( default_timeout );
 
 let click_with = {
   mobile: 'tap',
@@ -69,10 +77,12 @@ let click_with = {
 };
 
 BeforeAll(async () => {
-  scope.timeout = 30 * 1000;
   scope.showif_timeout = 600;
   scope.report = new Map();
 
+  // Start tracking whether server is up and running
+  scope.track_server_status( scope );
+
   // Store path names for files and docs created for the tests
   scope.paths = {};
   // Test setup should have created this filepath. Run setup if there's a problem with this.
@@ -92,9 +102,21 @@ BeforeAll(async () => {
 });
 
 Before(async (scenario) => {
+  // Create browser, which can't be created in BeforeAll() where .driver doesn't exist for some reason
+  if (!scope.browser) {
+    scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() });
+  }
+  // Clean up all previously existing pages
+  for (const page of await scope.browser.pages()) { 
+    await page.close();
+  }
+  // Make a new page
+  scope.page = await scope.browser.newPage()
+
   scope.scenario_id = uuidv4();
   scope.expected_status = null;
   scope.expected_in_report = null;
+  scope.server_reload_promise = null;
 
   await scope.addReportHeading(scope, {scenario});
   // Make folder for this Scenario in the all-tests artifacts folder
@@ -119,7 +141,7 @@ Before(async (scenario) => {
   scope.passed_all_for_a11y = true;
 
   // Reset default timeout
-  scope.timeout = 30 * 1000;
+  scope.timeout = default_timeout;
 });
 
 // Add a check for an error page before each step? After each step?
@@ -132,47 +154,9 @@ Before(async (scenario) => {
 //#####################################
 
 Given(/I start the interview at "([^"]+)"(?: in lang "([^"]+)")?/, {timeout: -1}, async (file_name, language) => {  // √
-  // NO CUCUMBER TIMEOUT. Handling our own timeouts to deal with initial scenario load.
-  // It involves wrapping everything in custom timeouts. See
-  // https://github.com/SuffolkLITLab/ALKiln/issues/389#issuecomment-984844240
-  let max_load_attempts = 3;
-  let num_attempts = 0;
-  // First test long load time discussion:
-  // https://github.com/SuffolkLITLab/ALKiln/issues/389#issuecomment-984844240
-  while ( num_attempts < max_load_attempts ) {
-
-    num_attempts++;
-    let timeout_message = `Failed to load "${ file_name }" after ${ num_attempts } tries. `
-      + `Each try gave the page ${ scope.timeout/1000 } seconds to load.`;
-    
-    let loaded = new Promise(async ( resolve, reject ) => {
-      // Exit if timeout is exceeded. Reject only if final attempt failed
-      let page_timeout = setTimeout(function() {
-        if ( num_attempts === max_load_attempts ) { reject( `${ timeout_message } Internal timeout.` ); }
-        else { resolve( false ); }  // Give it another shot
-      }, scope.timeout);
-      // self-invoke an anonymous async function so we don't have to
-      // abstract the details of resolving and rejecting.
-      (async function() {
-        try {
-          await scope.load( scope, file_name );
-          resolve( true );
-        } catch ( err ) {
-          // Only fail on final attempt. Actual error should be logged by cucumber output.
-          if ( num_attempts === max_load_attempts ) {
-            // This may not be the most useful error for us framework developers, but
-            // we'll have the rejection error I hope
-            await scope.addToReport(scope, { type: `error`, value: timeout_message });
-            reject( err );
-          } else { resolve( false ); }  // Give it another shot
-        } finally {
-          clearTimeout( page_timeout );  // prevent temporary hang at end of tests
-        }
-      })();
-    });  // ends loaded promise
-    await loaded;
-    if ( loaded === true ) { break; }
-  }  // ends while attempting to load page
+
+  // Load the right interview page
+  await scope.load( scope, file_name );
   
   // The page timeout is set in here too in case someone set a custom timeout
   // before a page was loaded.
@@ -276,13 +260,13 @@ Given(/I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^
   * I sign on with the email "USER1_EMAIL" and the password "USER1_PASSWORD" SECRETs
   * I log in with "USER1_EMAIL" and "USER1_PASSWORD"
   */
-  return wrapPromiseWithTimeout(
-    scope.steps.sign_in( scope, {
-      email_secret_name: email,
-      password_secret_name: password
-    }),
-    scope.timeout
-  );
+  // Exiting the custom timeout happens in scope.tapElement() if needed
+  // Couldn't test the timeout for tapping the button because there's not
+  // enough of a pause between initial navigation and pressing button.
+  await scope.steps.sign_in( scope, {
+    email_secret_name: email,
+    password_secret_name: password
+  });
 });
 
 // I am using a mobile/pc
@@ -304,6 +288,7 @@ When(/I am using an? (.*)/, async ( device ) => {
 
 Given(/the max(?:imum)? sec(?:ond)?s for each step (?:in this scenario )?is (\d+)/i, async ( seconds ) => {
   /* At any time, set a custom timeout for the scenario. */
+  // TODO: Add 1 second to this to allow other steps to not fail, like "I wait n seconds"
   scope.timeout = parseInt( seconds ) * 1000;
   
   // Cucumber timeouts are made custom for each step that needs it
@@ -338,18 +323,26 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
   let supported_table = await scope.normalizeTable( scope, { raw_var_data });
 
   let { question_has_id, id, id_matches } = await scope.examinePageID( scope, target_id );
-  // Until we reach the final page or hit an error
+  // Until we reach the target page, hit an error, or time out (take too long)
   while ( !id_matches ) {
     let custom_timeout = new Promise(function( resolve, reject ) {
-      // Set up the timeout
-      let page_timeout = setTimeout(function() {
-        let timeout_message = `The target id was "${ target_id }", but it took too long` +
-        ` to try to reach it (over ${ scope.timeout/1000/60 } min). The test got stuck on "${ id }".`
-        reject( timeout_message );
-      }, scope.timeout);
+
+      // Set up the timeout that avoids infinite loops
+      let on_page_timeout = async function() {
+
+        let non_reload_report_msg = `The target id was "${ target_id }", but it took `
+        + `too long to try to reach it (over ${ scope.timeout/1000/60 } min). The `
+        + `test got stuck on "${ id }".`
+        let error = non_reload_report_msg;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+      };  // Ends on_page_timeout()
+      let page_timeout_id = setTimeout(on_page_timeout, scope.timeout);
 
       // self-invoke an anonymous async function so we don't have to
       // abstract the details of resolving and rejecting.
+      // If we need to call recursively, signature function might look like:
+      // func( scope, { resolve, reject, supported_table, page_timeout_id })
+      // Maybe list of timeout and/or interval ids in state that can all be cleared at once?
       (async function() {
         try {
           let { question_has_id, id, id_matches } = await scope.examinePageID( scope );
@@ -365,9 +358,12 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
           if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
           resolve();
         } catch ( err ) {
+          let err_msg = `Error occurred when tried to answer fields on question "${ id }".`
+          scope.addToReport( scope, { type: `error`, value: err_msg });
+          // Throw error instead?
           reject( err );
         } finally {
-          clearTimeout( page_timeout );  // prevent temporary hang at end of tests
+          clearTimeout( page_timeout_id );  // prevent temporary hang at end of tests
         }
       })();
     });  // ends custom_timeout for setFields
@@ -736,31 +732,25 @@ Then(/the "([^"]+)" link opens a working page/, { timeout: -1 }, async ( linkTex
 When('I tap to continue', { timeout: -1 }, async () => {
   // Any selectors for this seem somewhat precarious
   // No data will cause an automatic continue when `ensure_navigation` is true
-  return wrapPromiseWithTimeout(
-    scope.setFields( scope, { var_data: [], ensure_navigation: true }),
-    scope.timeout
-  );
+
+  // Exiting the custom timeout happens in scope.tapElement() where we can
+  // check if the server is reloading
+  await scope.setFields( scope, { var_data: [], ensure_navigation: true });
 });
 
 When(/I tap the "([^"]+)" tab/i, { timeout: -1 }, async ( tabId ) => {
-  return wrapPromiseWithTimeout(
-    scope.tapTab( scope, tabId ),
-    scope.timeout
-  )
+  // Exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.tapTab( scope, tabId );
 });
 
 When(/I tap the "([^"]+)" element$/i, { timeout: -1 }, async ( elemSelector ) => {
-  return wrapPromiseWithTimeout(
-    scope.tapElementBySelector( scope, elemSelector, 2000 ),
-    scope.timeout
-  )
+  // Exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.tapElementBySelector( scope, elemSelector, 2000 );
 });
 
 When(/I tap the "([^"]+)" element and wait ([0-9]+) second(?:s)?/i, { timeout: -1 }, async ( elemSelector, waitTime ) => {
-  return wrapPromiseWithTimeout(
-    scope.tapElementBySelector( scope, elemSelector, waitTime * 1000 ),
-    scope.timeout
-  )
+  // Exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.tapElementBySelector( scope, elemSelector, waitTime * 1000 )
 });
 
 When(/I check the page for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
@@ -951,20 +941,18 @@ When(/I set the var(?:iable)? "([^"]+)" to "([^"]+)"/, { timeout: -1 }, async (
   *
   * Variations:
   * 1. I set the var "benefits['SSI']" to "false"
-  * 1. I set the var "rent_interval" to "weekly"
+  * 1. I set the variable "rent_interval" to "weekly"
   */
-  return wrapPromiseWithTimeout(
-    scope.steps.set_regular_var( scope, var_name, answer ),
-    scope.timeout
-  );
+  // Exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.steps.set_regular_var( scope, var_name, answer )
+
 });
 
 When(/I set the var(?:iable)? "([^"]+)" to ?(?:the)? ?(?:GitHub)? secret "([^"]+)"/i, { timeout: -1 }, async ( var_name, set_to_env_name ) => {
   /* Set a non-story table variable to a secrete value. Same as the above, but reads the value from `process.env` */
-  return wrapPromiseWithTimeout(
-    scope.steps.set_secret_var( scope, var_name, set_to_env_name ),
-    scope.timeout
-  );
+  // This could theoretically tap an element, so
+  // exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.steps.set_secret_var( scope, var_name, set_to_env_name )
 });
 
 
@@ -983,10 +971,8 @@ Then(/I upload "([^"]+)" to "([^"]+)"/, { timeout: -1 }, async ( filenames, var_
 
 When('I tap the defined text link {string}', { timeout: -1 }, async ( phrase ) => {
   /** Tap a link that doesn't navigate. Depends on the language of the text. */
-  return wrapPromiseWithTimeout(
-    scope.steps.tap_term( scope, phrase ),
-    scope.timeout
-  );
+  // exiting the custom timeout happens in scope.tapElement() if needed
+  await scope.steps.tap_term( scope, phrase );
 });
 
 Then('I sign', { timeout: -1 }, async () => {
@@ -1003,6 +989,9 @@ When(/I download "([^"]+)"$/, { timeout: -1 }, async ( filename ) => {
   *    WARNING: Must not contain any characters that are totally unique to that
   *    page load. Just stick to the name of the file.
   */
+  // TODO:
+  // This should be using `scope.tapElement()`? When someone clicks on a link to
+  // download a document, it sends a request to the server, which could be reloading
   return wrapPromiseWithTimeout(
     scope.steps.download( scope, filename ),
     scope.timeout
@@ -1095,6 +1084,7 @@ Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected
 //#####################################
 
 After(async function(scenario) {
+
   // The accessibility failures don't happen til the end, so the rest of the test can continue
   if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
     // The fact that the scenario failed has already been noted earlier in the report
@@ -1119,6 +1109,7 @@ After(async function(scenario) {
           value: `For security, ALKiln will not create a screenshot for this error. It's possible a secret is being used on this screen.`
         });
       } else {
+
         // Save/download a picture of the screen that's showing during the unexpected status
         // Save one copy in the outer-most artifact folder
         let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` });
@@ -1168,32 +1159,29 @@ After(async function(scenario) {
 
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
-    // Make sure the new interview really starts fresh. Sometimes insufficient.
-    // Also, often is a failure point.
-    // Consider, instead, catching the reload error for a bad restart and just
-    // re-trying at that point.
+    // Catch a possible reload error.
+
+    // TODO: implement and use scope.handle_possible_timeout()
     try {
+      // puppeteer will ensure proper timeout.
       await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});  
-    } catch ( err ) {
-      if ( session_vars.DEBUG ) {
-        console.log( `Trying to reset for the next test a second time because the first time failed.` );
-      }
 
-      try {
-        await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});  
-      
-      } catch ( err ) {
-        await scope.addToReport(scope, {
-          type: `warning`,
-          value: `This test ${ original_scenario_status }, but also there `
-            + `was an error while trying to reset for the next test. You can try `
-            + `re-running the test or checking your server.`
-        });
-      }
-    }
-    
-    await scope.page.close();
-    scope.page = null;
+    } catch ( error ) {
+
+      if ( error.name === `TimeoutError` ) {
+        // TODO: Should a more specific message about signing out get into the report no matter what?
+        // If so, what should the message be in order to not get confused with these other messages?
+        let non_reload_report_msg = `It took too long to sign out in preparation for the next test.`;
+        await scope.handle_page_timeout_error( scope, { non_reload_report_msg, error });
+      } else {
+        // Throw any non-timeout error
+        let err_msg = `Error occurred when ALKiln tried to sign out in preparation for the next test.`
+        scope.addToReport( scope, { type: `error`, value: err_msg });
+        throw error;
+      }  // ends if error is timeout error
+
+    }  // ends try/catch
+
   }  // ends if scope.page
 
   // Add report text to Scenario folder after everything has been added to the report
@@ -1205,6 +1193,9 @@ After(async function(scenario) {
 });
 
 AfterAll(async function() {
+  // Stop collecting server response statuses
+  scope.stop_tracking_server_status = true;
+  
   let report = await scope.getPrintableReport( scope );
   // Log the report. Can't get this.attach() or this.log() to work in `After()` or here.
   // May need newest version of cucumberjs

package/lib/utils/session_vars.js

@@ -25,6 +25,7 @@ module.exports = session_vars;
 session_vars.get_debug = function () { return process.env.DEBUG || null; };
 session_vars.get_dev_api_key = function () { return process.env.DOCASSEMBLE_DEVELOPER_API_KEY || null; };
 session_vars.get_repo_url = function () { return process.env.REPO_URL || null; };
+session_vars.get_server_reload_timeout = function () { return process.env.SERVER_RELOAD_TIMEOUT_SECONDS || null; };
 session_vars.is_dev_env = function () { return process.env.DEV || null; }
 
 // More complex logic

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "4.7.1",
+  "version": "4.7.2",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
     "test": "npm run unit && npm run langs_setup && npm run cucumber -- $@",
     "cucumber_base": "run_cucumber(){ ./node_modules/.bin/cucumber-js \"$@\"; status=$?; ./lib/utils/clean_reports.js; return $status; }; run_cucumber",
-    "cucumber": "run_base(){ npm run cucumber_base -- \"$@\" tests/features/*.feature; }; run_base",
+    "cucumber": "run_base(){ npm run cucumber_base -- \"$@\" tests/features/*.feature --retry 1; }; run_base",
     "cucumber_external": "run_cucumber(){ tmp_path=`pwd`; ( new_path=\"$1\"; shift; cd \"$new_path\"; npm run --prefix \"${tmp_path}\" cucumber_base -- \"$@\" \"$new_path\"/docassemble/*/data/sources/*.feature;) }; run_cucumber",
     "langs": "npm run langs_setup && ./node_modules/.bin/cucumber-js --require ./lib/index.js tests/features/*_lang*.feature",
     "langs_setup": "./lib/utils/langs.js 'tests/features/*.feature'",

package/tests/features/reload.feature

@@ -0,0 +1,155 @@
+@reload
+Feature: Errors caused by server reload
+
+# These can only be run locally one at a time once you uncomment them:
+# 1. Make sure you have two versions of the repo locally and open a terminal window for each of them
+# 2. With one repo, start one specific test
+# 3. Be ready to run the setup step for the 2nd repo
+# 4. Prepatory explanation: With some noted exceptions, when the 1st test is _starting_ its long wait time (which they should each have), start the setup step in the other terminal.
+# 4a. Start the test in the 1st repo. Ex: npm run cucumber -- "--tags" "@reload1"
+# 4b. Wait until the long wait time is just starting (count the number of steps to match the progress dots)
+# 4c. Start the setup in the 2nd repo: npm run setup
+# 5. Make sure the test waits longer than the scope.timeout
+# 6. Make sure the reload error is triggered for the first attempt of the test
+# 7. Make sure the reload error is not present for the second attempt
+# 8. For the second repo: npm run takedown
+
+# # Tests that need to be written and run:
+# # Theoretically implemented
+# - [ ] 1. sign in to server with `.sign_in` on two levels:
+#   - [x] (throw_after_server_reloads() solved this I think) when navigating to server
+#   - [ ] when submitting to log in (impossible to time? they both happen at once)
+# - [x] 2. (waits full time for sign-out page, but I think it's working otherwise) I tap the ... tab (should not cause timeout at all, so should be fine)
+# - [x] 3. tap elements
+#   - [x] I tap the ... element$
+#   - [ ] (same as above?) I tap the ... element and wait
+# - [x] 4. Set the var of a continue button
+#   - [x] I set the var ... (using a button press to continue)
+#   - [x] (same as above?) I set the var ... GitHub)? secret (for a button press)
+#   - [ ] (same as above? We don't have an interview ready for this yet. Seen notes in 4c) I set the var for a button in a story table
+# - [x] 5. Signing out at end of step
+# - [x] 6. Tap to continue
+#   - [x] I tap to continue
+#   - [x] In the middle of a story table (basically same as "tap to continue")
+# - [x] 7. Fails both tests
+# - [x] 8. The server does actually finish reloading during the first attempt, so there's only one error printed in the report.
+# 
+# # Not yet implemented
+# - [ ] I download
+#
+# Skipping for now because not a documented step yet
+# - [ ] I tap the defined text link
+
+## For this scenario, start the server setup before you start running the test and wait till 4th attempt to pull in the code.
+## No screenshots are taken because these are secrets
+#@reload1a
+#Scenario: The server reloads while I navigate to the server sign in page
+#  Given the max seconds for each step in this scenario is 5
+#  Given I sign in with the email "USER1_EMAIL" and the password "USER1_PASSWORD"
+#
+#@reload1b
+##Scenario: The server reloads while I'm submitting my signin (impossible)
+#
+## Should error after finishing test, but only because can't finish the test by going to the sign out page during server #reload. Sign out page is a necessary evil at the moment to make sure the next test starts fresh.
+#@reload2
+#Scenario: The server reloads while I tap a tab
+#  Given I start the interview at "test_taps"
+#  And I wait 20 seconds
+#  And I tap the "Tests-second_template-tab" tab
+#  Then I see the phrase "villify"
+#
+#@reload3a
+#Scenario: The server reloads while I tap an element selector
+#  Given I start the interview at "all_tests"
+#  And I wait 18 seconds
+#  Given the max seconds for each step in this scenario is 5
+#  Then I tap the "button.btn.btn-da.btn-primary[type='submit']" element
+#
+##@reload3b
+##  Given I start the interview at "all_tests"
+##  And I wait 18 seconds
+##  Given the max seconds for each step in this scenario is 5
+##  Then I tap the "button.btn.btn-da.btn-primary[type='submit']" element and wait 1 seconds
+#
+## Start setup on the 12th dot
+#@reload4a
+#Scenario: The server reloads while I set the var of a continue button
+#  Given I start the interview at "all_tests"
+#  And I tap to continue
+#  And I tap to continue
+#  When I set the var "checkboxes_yesno" to "True"
+#  And I set the var "checkboxes_other['checkbox_other_opt_1']" to "true"
+#  And I set the var "dropdown_test" to "dropdown_opt_2"
+#  And I set the var "radio_yesno" to "False"
+#  And I set the var "radio_other" to "radio_other_opt_3"
+#  And I set the var "text_input" to "Regular text input field value"
+#  And I set the var "textarea" to "Multiline text\narea value"
+#  And I wait 25 seconds
+#  Given the max seconds for each step in this scenario is 5
+#  Then I set the var "direct_standard_fields" to "True"
+#
+## You must add the env var `STANDARD_FIELDS`. Otherwise, it's just like @reload4a
+#@reload4b
+#Scenario: The server reloads while I set the var of a continue button
+#  Given I start the interview at "all_tests"
+#  And I tap to continue
+#  And I tap to continue
+#  When I set the var "checkboxes_yesno" to "True"
+#  And I set the var "checkboxes_other['checkbox_other_opt_1']" to "true"
+#  And I set the var "dropdown_test" to "dropdown_opt_2"
+#  And I set the var "radio_yesno" to "False"
+#  And I set the var "radio_other" to "radio_other_opt_3"
+#  And I set the var "text_input" to "Regular text input field value"
+#  And I set the var "textarea" to "Multiline text\narea value"
+#  And I wait 25 seconds
+#  Given the max seconds for each step in this scenario is 5
+#  Then I set the var "direct_standard_fields" to the github secret "STANDARD_FIELDS"
+#
+### We don't have an interview set up for this yet. We need one where the _first_ page
+### has a continue button field that sets a variable.
+##@reload4c
+##Scenario: The server reloads while I run a story table
+##  Given I start the interview at "all_tests"
+##  And I wait 18 seconds
+##  Given the max seconds for each step in this scenario is 3
+##  And I get to "id of second page" with this data:
+##    | var | value | trigger |
+##    | some_continue_button_var_on_first_page | true |  |
+#
+#@reload5
+#Scenario: The server reloads while signing out
+#  Given I start the interview at "all_tests"
+#  And I wait 16 seconds
+#  Given the max seconds for each step in this scenario is 1
+#
+#@reload6a
+#Scenario: The server reloads while I tap to continue
+#  Given I start the interview at "all_tests"
+#  And I wait 18 seconds
+#  Given the max seconds for each step in this scenario is 3
+#  And I tap to continue
+#
+#@reload6b
+#Scenario: The server reloads while I run a story table
+#  Given I start the interview at "all_tests"
+#  And I wait 18 seconds
+#  Given the max seconds for each step in this scenario is 3
+#  And I get to "direct standard fields" with this data:
+#    | var | value | trigger |
+#    | double_quote_dict['double_quote_key']["dq_two"] | true |  |
+#    | single_quote_dict["single_quote_key"]['sq_two'] | true |  |
+#
+## This test might not be necessary. The test run should end in failure and the message should appear in both reports.
+## Run `npm run setup` both times. This will complicate takedown - it creates two projects, so at the end you have to takedown #the current project, which removes the project name in the config, then you have to put the name of the first project into the #config and then run takedown again.
+#@reload7
+#Scenario: The server is reloading in both attempts
+#  Given I start the interview at "all_tests"
+#  And I wait 22 seconds
+#  Given the max seconds for each step in this scenario is 1
+#
+#@reload8
+#Scenario: The server finishes reloading before the first timeout
+#  Given I start the interview at "all_tests"
+#  And I wait 18 seconds
+#  Given the max seconds for each step in this scenario is 30
+#  And I tap to continue