@suffolklitlab/alkiln 5.16.1 → 5.17.0-fix-stop-on-error-1

package/.github/workflows/da_image_v_sha_7d13853.yml

@@ -31,9 +31,6 @@ jobs:
           && github.event.pull_request.head.repo.fork
         )
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [24.x]
       
     steps:
       # Place the root directory in this branch to access relative paths to
@@ -65,4 +62,4 @@ jobs:
           DOCASSEMBLE_DEVELOPER_API_KEY: "${{ steps.github_server.outputs.DOCASSEMBLE_DEVELOPER_API_KEY }}"
           INSTALL_METHOD: "server"
           ALKILN_TAG_EXPRESSION: "@e3"  # One very simple test
-          # ALKILN_VERSION: 
+          ALKILN_VERSION: "stop_on_error"

package/.github/workflows/github_server.yml

@@ -36,9 +36,6 @@ jobs:
           && github.event.pull_request.head.repo.fork
         )
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [24.x]
 
     #### Developer note: You can probably leave out these specific
     #### environment variables. You may not even need the `env`
@@ -160,7 +157,7 @@ jobs:
           #### Developer note: You can probably leave the rest out
           #### To learn more, see https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#optional-inputs
           ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
-          # ALKILN_VERSION: 
+          ALKILN_VERSION: "stop_on_error"
 
       #### Developer note: Example of making an issue when tests fail
       #### that includes the text of the failure output file

package/.github/workflows/playground.yml

@@ -27,9 +27,6 @@ jobs:
           && github.event.pull_request.head.repo.fork
         )
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [24.x]
 
     env:
       #### Developer note: We recommend turning these into environment
@@ -83,4 +80,4 @@ jobs:
           # want to check up on this.
           ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
           #### Developer note: You can probably leave this out
-          # ALKILN_VERSION: 
+          ALKILN_VERSION: "stop_on_error"

package/CHANGELOG.md

@@ -43,7 +43,35 @@ Format:
 - 
 -->
 
-<!-- ## [Unreleased] -->
+## [Unreleased]
+
+### Changed
+
+- Extended error message for reaching a docassemble system error page (like syntax errors, etc.) It now should contain the relevant contents of the page.
+
+### Fixed
+
+- Fixed linear Steps not failing on error screens. See #656.
+
+### Internal
+
+- Fixed confusing error message when tests expected to fail actually pass.
+
+## [5.17.0] - 2026-09-10
+
+### Changed
+
+- GitHub action release: Restored our GitHub action's default for ALKiln version to the latest version 5 again. Released first on GitHub actions. NPM release will come in time, but npm has no impact on GitHub action releases.
+
+### Fixed
+
+- Fixed GitHub+You action outdated docassemble cli version causing Scenario timeouts.
+- Part 1 of detecting custom AL error screen: Story Table now catches `al_custom_error_action` error. Previously, Story Tables that ran into this screen would get stuck on the AL error page infinitely. Next steps: Linear Steps may need additional development. Currently linear Steps succeed even when they run into AL's error page. That bears additional discussion. See [issue #656](https://github.com/SuffolkLITLab/ALKiln/issues/656) and [PR #1116](https://github.com/SuffolkLITLab/ALKiln/pull/1116).
+
+### Internal
+
+- Simplify workflow files by removing `node` env setup (leaving it to the actions)
+- Updated both of our actions' dependencies (the checkout, setup-node, setup-python, upload-artifacts, download-artifacts actions). Closes [#1095](https://github.com/suffolkLITLab/aLKiln/issues/1095). Once again, action related.
 
 ## [5.16.1] - 2026-06-01
 

package/action.yml

@@ -14,7 +14,7 @@ inputs:
   ALKILN_VERSION:
     description: 'Version of ALKiln to get from npm.'
     required: false
-    default: '5.15.4'
+    default: '^5.0.0'
   MAX_SECONDS_FOR_SETUP:
     description: "Amount of time to give the Project to upload your package's GitHub code."
     required: false
@@ -32,7 +32,7 @@ inputs:
   DOCASSEMBLECLI_VERSION:
     description: "The version of docassemblecli to install."
     required: false
-    default: "0.0.17"
+    default: "0.0.25"
   ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO:
     description: "The maximum number of tests ALKiln will generate for every Scenario that can generate tests."
     required: false
@@ -59,7 +59,7 @@ runs:
   using: "composite"
   steps:
     # Place the root directory in this branch
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
 
     # Set environment variables
     - name: "💡 ALK0023 INFO: Set environment variables"
@@ -111,7 +111,7 @@ runs:
 
     # Install ALKiln during GitHub workflow
     - name: "💡 ALK0025 INFO: Set up node"
-      uses: actions/setup-node@v5
+      uses: actions/setup-node@v6
       with:
         node-version: "24.x"
         package-manager-cache: false
@@ -155,7 +155,7 @@ runs:
 
     # Install package on playground
     # Don't rely on the environment's version of python
-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v6
       if: ${{ inputs.INSTALL_METHOD == 'playground' }}
       with:
         python-version: '3.10'
@@ -213,7 +213,7 @@ runs:
     # Upload artifacts that subscribers can download on the Actions summary page
     - name: "💡 ALK0033 INFO: Upload artifacts folder"
       if: ${{ always() }}
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: ${{ env.ARTIFACT_FOLDER }}
         path: ./alkiln-*
@@ -221,7 +221,7 @@ runs:
     # Download artifacts folder internally to create action output
     - name: "💡 ALK0183 INFO: Get uploaded artifacts folder"
       if: ${{ always() }}
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v8
       with:
         name: ${{ env.ARTIFACT_FOLDER }}
 

package/action_for_github_server/action.yml

@@ -49,7 +49,7 @@ runs:
   using: "composite"
   steps:
       # Place the root directory in this branch
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - shell: bash
         run: |
@@ -253,7 +253,7 @@ runs:
           done
 
       # Don't rely on the environment's version of python
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: '3.10'
       # No output
@@ -285,7 +285,7 @@ runs:
           echo "$(docker output docassemble_container)" >> "${{ env.DEBUG_LOGS_PATH }}"
       - name: "💡 ALK0020 INFO: Upload artifacts folder"
         if: ${{ always() && env.OUTPUT_DOCKER == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: ${{ env.GITHUB_SERVER_ARTIFACT_NAME }}
           path: ${{ env.DEBUG_LOGS_PATH }}

package/lib/scope.js

@@ -255,9 +255,81 @@ module.exports = {
   },  // Ends afterStep
 
   tapElementAndNavigate: async function(scope, { elem, waitForTimeout=null }) {
-    /** Tap a given element, detect navigation, and do appropriate related things. */
+    /** 
+     * Tap a given element, detect navigation, and do appropriate related things.
+     * 
+     * For other ideas for detecting navigation, see
+     * https://github.com/SuffolkLITLab/ALKiln/issues/229.
+     * 
+     * TODO: Duplicates much of `waitUntilContinued`. Need to de-duplicate.
+     *   Alternatively, continue other work on navigation detection with server
+     *   responses.
+     * 
+     * @param {int} obj.waitForTimeout - How long to wait for a timeout
+     * */
+    // await scope.throwAnySystemError( scope );  // error on current pg
+
+    // let prev_url = url;
+    // let curr_url = await scope.page.url();
+
+    // let timed_out = false;  // Lazy timeout?
+    // let cont_timeout = setTimeout(function(){ timed_out = true }, scope.timeout);
+
+    // // Attempt to solve weird race condition issues where page has not properly continued
+    // while ( curr_url === prev_url && !page_result.element && !timed_out ) {
+    //   await time.waitForTimeout( 50 );
+    //   curr_url = await scope.page.url();
+    //   page_result = await scope.throwAnySystemError( scope );  // error on new pg
+    // }
 
-    let start_url = await scope.page.url();  // so we can later check for navigation
+    // // TODO: We sometimes log multiple error messages for an error because we're
+    // //   logging at multiple levels. We should try to refactor to log just once
+    // //   per error
+    // let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. You can check the picture of this page unless a secret variable was used.`
+    // if ( timed_out ) { reports.addToReport(scope, { type: `error`, code: `ALK0146`, value: timeout_err_message }); }
+    // expect( timed_out, timeout_err_message ).to.be.false;
+
+    // clearTimeout( cont_timeout );
+
+    // return page_result;
+
+
+    // Invalid user answer will prevent navigation
+    // Shoot. How does this work for .setFields()...?
+    // Think. Checked for correct id. Possibly not. If did continue:
+    //   Fill fields/press var of continue button or
+    //   Press continue anyway (tap to continue or no fields to set)
+    // If on page with user error... id should have recognized it as valid
+    //   destination or we shouldn't get stuck on the page. Which would make
+    //   this work either way.
+    // That's fine. But. I wanted to change behavior so that a Story Table
+    //   could set fields on the final page. If that were the case and _if_
+    //   the field were a continue button, that would break our rule that
+    //   Story Tables can contain extra fields without worry.
+    //   Maybe that can be a flag - on final id page, so don't try to press
+    //   continue button fields, even when it sets a variable in the table.
+    //   Or any buttons? Hmm. I'll have to make an issue.
+    // TODO: Make an issue for the alternative Story Table behavior.
+    // But also. What if... well, under some circumstances the author will
+    //   want to navigate to a page and get invalid values. But brain. Brain
+    //   brain brain. But if the author navigates after expecting the page
+    //   to have invalid values, then this should fail too.
+    // TODO: Write all this up in here, but better.
+    // But what if the author is navigating a different way, like with an
+    //   action button. So, invalid answers, but then wants to navigate with
+    //   an action button. That might indeed be a real problem. Right? Brain?
+    //   You should be allowed to navigate away from any page.
+    // Ok, we really do need to split up system errors from user errors.
+    // Dangit.
+
+
+    // If on system error page, can't navigate.
+    // TODO: Discuss how to allow AL to test its custom error page buttons.
+    //   Maybe avoiding a system error here would be sufficient.
+    await scope.throwAnySystemError( scope );
+
+    let start_url = await scope.page.url();  // for navigation check later
+    let curr_url = start_url;
 
     await scope.tapElementAndStayOnPage(scope, {
       elem,
@@ -267,29 +339,49 @@ module.exports = {
       ]
     });
 
-    // For other ideas for detecting navigation, see
-    // https://github.com/SuffolkLITLab/ALKiln/issues/229
-    let end_url = await scope.page.url();
-    let navigated = start_url !== end_url;
+    // Attempt to solve weird race condition issues where page has not properly
+    // continued. May exceed actual timeout
+    let timed_out = false;
+    let cont_timeout = setTimeout(
+      function(){ timed_out = true; }, ( scope.timeout - 10 )
+    );
+    while ( curr_url === start_url && !timed_out ) {
+      try {
+        await scope.throwAnySystemError( scope );  // quick exit on system error
+      } catch ( sys_err_msg ) {
+        clearTimeout( cont_timeout );
+        throw sys_err_msg;
+      }
+      await time.waitForTimeout( 50 );
+      curr_url = await scope.page.url();
+    }
+    clearTimeout( cont_timeout );
+
+    await scope.throwAnySystemError( scope );  // system error on new page
 
-    // Might be able to handle this in `scope.afterStep`
-    if ( navigated ) {
+    // TODO: We sometimes log multiple error messages for an error because we're
+    //   logging at multiple levels. We should try to refactor to log just once
+    //   per error
+    let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. You can check the picture of this page unless a secret variable was used.`
+    if ( timed_out ) { reports.addToReport(scope, { type: `error`, code: `ALK0283`, value: timeout_err_message }); }
+    expect( timed_out, timeout_err_message ).to.be.false;
 
-      // TODO: Can we remove this?
-      // Save the id that comes next
-      let { question_has_id, id, id_matches } = await scope.examinePageID( scope, 'none to match' );
+    // Might be able to handle this in `scope.afterStep`
+    if ( start_url !== curr_url ) {
 
+      scope.navigated = true;
       // If navigation, wait for 200ms to let things finish loading and moving
       await scope.afterStep(scope, {
         waitForTimeout: ( waitForTimeout || 200 ),
         navigated: true
       });
+
     } else {
-      // Discuss: Fail here or later since the test didn't manage to navigate?
-      await scope.afterStep(scope, {
-        waitForTimeout: ( waitForTimeout || 0 ),
-        navigated: false
-      });
+
+      // TODO: #656, this with one other change may be all we need
+      let non_nav_msg = `Was unable to navigate away from page with the id "${ start_url }".`
+      reports.addToReport(scope, { type: `error`, code: `ALK0284`, value: non_nav_msg });
+      throw new Error( non_nav_msg );
     }
   },  // Ends scope.tapElementAndNavigate()
 
@@ -347,11 +439,14 @@ module.exports = {
         // This is meant to detect user-visible alerts/errors,
         // not breaking errors in the testing code.
         // No need for aborters since puppeteer has its own timeout
+        // TODO: Discuss why we need to detect invalid answers on a page where
+        //   we don't plan to navigate.
         scope.page.waitForSelector('.alert-danger', { visible: true }),
         scope.page.waitForSelector('.da-has-error', { visible: true }),
       ]);
     } catch ( error ) {
       if ( error.name === `TimeoutError` ) {
+        // TODO: Discuss - this is a navigation timeout even though the tap shouldn't navigate
         let non_reload_report_msg = `After ALKiln tapped "${ tapperText }" it took too long to load the next page.`;
         await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
           code: `ALK0103`,
@@ -608,6 +703,7 @@ module.exports = {
 
   checkForA11y: async function (scope ) {
     /** Return if there were any violations and the file containing aXe output with more details of the checks */
+    // Discuss: What problem was the page id solving? When was there a mismatch?
     let { id } = await scope.examinePageID( scope, 'none to match' );
     if ( scope.page_id !== id ) {
       reports.addToReport(scope, { type: `page_id info`, code: `ALK0108`, value: `${ id }`, });
@@ -643,7 +739,7 @@ module.exports = {
 
   tapSelectorToNavigate: async function ( scope, selector, waitForTimeout ) {
     let elem = await scope.page.$(`${ selector }`);
-    let error_msg = `Couldn't find "${ selector }" on the page, is there a typo?`;
+    let error_msg = `Couldn't find "${ selector }" on the page. Is there a typo?`;
     if ( !elem ) {
       reports.addToReport(scope, { type: `error`, code: `ALK0111`, value: error_msg });
     }
@@ -2304,12 +2400,16 @@ module.exports = {
 
     // Press a continue button with a story table variable name if it exists
     let did_navigate = false;
-    let error = {};
     for ( let field of fields ) {
       // All other fields should have been set
       if ( field.tag === 'button' && field.type === 'submit' ) {
         let page_url = await scope.page.url();
 
+        // TODO: Attempt avoiding `awaiting` and instead allow
+        //   waitUntilContinued to detect navigation.
+        //   Except `row_used` should be set before an error page. Maybe
+        //   `row_used` should always be set before all this, since we know the
+        //   field exists on the page (we know it's a submit button).
         let row_used = await scope.setVariable( scope, { field, var_data, });
         if ( row_used ) {
 
@@ -2334,10 +2434,7 @@ module.exports = {
           // Make sure no system or user error messages appear
           // Discuss moving this out of here. Having to pass in `id` just for
           // these messages smells bad. Con: Already a lot of code out there.
-          error = await scope.waitUntilContinued( scope, { url: page_url, id });
-          // if ( error.was_found ) {
-          //   await scope.throwPageError( scope, { id: id });
-          // }
+          await scope.waitUntilContinued( scope, { url: page_url });
 
           did_navigate = true;
           log.stdout({}, `\x1b[36m${ 'v' }\x1b[0m`);  // pressed button successfully
@@ -2370,6 +2467,8 @@ module.exports = {
     if ( !did_navigate && ensure_navigation  ) {
       // All fields on this page should be taken care of
       let page_url = await scope.page.url();
+      // TODO: Attempt avoiding `awaiting` and instead allow
+      //   waitUntilContinued to detect navigation.
       await scope.continue( scope );
       // Have to catch errors right now or may get stuck in an infinite loop?
       // TODO: Don't catch errors - maybe the developer expected/wanted an error
@@ -2381,10 +2480,7 @@ module.exports = {
       // these messages smells bad. Con: Already a lot of code out there.
 
       // TODO: How do we let 'I set the var __ to __' also afterwards check for an invalidation message?
-      error = await scope.waitUntilContinued( scope, { url: page_url, id });
-      // if ( error.was_found ) {
-      //   await scope.throwPageError( scope, { id: id });  
-      // }
+      await scope.waitUntilContinued( scope, { url: page_url });
 
       did_navigate = true;
       log.stdout({}, `\x1b[36m${ '>' }\x1b[0m`);  // continued successfully
@@ -2400,7 +2496,7 @@ module.exports = {
     // TODO: Since we now continue only once no matter what, we can put
     // the error in the caller of the function, though we may want
     // to pass the symbol to print as well.
-    return error;
+    return { ok: true };
     // TODO: Maybe we should return something else or something in addition about used vars
   },  // Ends scope.setFields()
 
@@ -2636,80 +2732,483 @@ module.exports = {
     }
   },  // Ends scope.has_interview_content()
 
-  waitUntilContinued: async function ( scope, { url = '', id = '' }) {
-    /** Given a url, attempt to continue until the page url changes or an error
-        is found. Return the error object no matter what. */
-    let prev_url = url;
-    let curr_url = await scope.page.url();
-    let error = await scope.checkForError( scope );
+  waitUntilContinued: async function ( scope, { url = '' }) {
+    /** Given a url, attempt to detect navigation until:
+     * 
+     *    - a page shows a system error or invalid user answers
+     *    - the page url changes
+     *    - the navigation times out
+     * 
+     * Note: intended to run after an attempt to navigate.
+     * 
+     * TODO: Duplicates much of `tapElementAndNavigate`. Need to de-duplicate.
+     *   Alternatively, continue other work on navigation detection with server
+     *   responses.
+     * 
+     * If the navigation fails, throw an error.
+     * */
+
+    // Discuss: Delay system err check? May allow AL to test custom error page.
+    await scope.throwAnySystemError( scope );  // system error on current page
+    await scope.throw_on_invalid_answers( scope );  // invalid user answers
 
-    let timedOut = false;  // Lazy timeout?
-    let cont_timeout = setTimeout(function(){ timedOut = true }, scope.timeout);
+    let start_url = url;  // for navigation check later
+    let curr_url = start_url;
 
     // Attempt to solve weird race condition issues where page has not properly continued
-    while ( curr_url === prev_url && !error.was_found && !timedOut ) {
+    // May exceed actual timeout
+    let timed_out = false;
+    let cont_timeout = setTimeout(
+      function(){ timed_out = true; }, ( scope.timeout - 10 )
+    );
+    while ( curr_url === start_url && !timed_out ) {
+      try {
+        await scope.throwAnySystemError( scope );  // system error on new page
+      } catch ( sys_err_msg ) {
+        clearTimeout( cont_timeout );
+        throw sys_err_msg;
+      }
       await time.waitForTimeout( 50 );
       curr_url = await scope.page.url();
-      error = await scope.checkForError( scope );
     }
+    clearTimeout( cont_timeout );
 
-    let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. Check the picture of this page. If a secret variable was used, there will be no picture.`
-    if ( timedOut ) { reports.addToReport(scope, { type: `error`, code: `ALK0146`, value: timeout_err_message }); }
-    expect( timedOut, timeout_err_message ).to.be.false;
+    await scope.throwAnySystemError( scope );  // system error on new page
+
+    // TODO: We sometimes log multiple error messages for an error because we're
+    //   logging at multiple levels. We should try to refactor to log just once
+    //   per error
+    let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. You can check the picture of this page unless a secret variable was used.`
+    if ( timed_out ) { reports.addToReport(scope, { type: `error`, code: `ALK0146`, value: timeout_err_message }); }
+    expect( timed_out, timeout_err_message ).to.be.false;
+
+    // Note: Invalid answers on the new page would be ok, though they probably
+    //   won't exist yet anyway.
+
+    if ( start_url !== curr_url ) {
+      scope.navigated = true;
+    }
+
+    return true;
 
-    clearTimeout( cont_timeout );
 
-    return error;
+    // let prev_url = url;
+    // let curr_url = await scope.page.url();
+    // let error_data = await scope.getPageError( scope );  // error on current pg
+
+    // let timed_out = false;
+    // let cont_timeout = setTimeout(
+    //   function(){ timed_out = true }, ( scope.timeout - 10 )
+    // );
+
+    // // Attempt to solve weird race condition issues where page has not properly continued
+    // while ( curr_url === prev_url && !error_data.element && !timed_out ) {
+    //   await time.waitForTimeout( 50 );
+    //   curr_url = await scope.page.url();
+    //   error_data = await scope.getPageError( scope );  // error on new pg
+    // }
+
+    // clearTimeout( cont_timeout );
+
+    // // TODO: Log only one error log per... error.
+    // let timeout_err_message = `The test tried to continue to the next page, but it took longer than ${ (scope.timeout/1000)/60 } min. You can check the picture of this page unless a secret variable was used.`
+    // if ( timed_out ) { reports.addToReport(scope, { type: `error`, code: `ALK0146`, value: timeout_err_message }); }
+    // expect( timed_out, timeout_err_message ).to.be.false;
+
+    // return error_data;
   },  // Ends scope.waitUntilContinued()
 
-  checkForError: async function ( scope ) {
-    /* Returns whether any system or user error elements were found and, if so,
-    * returns those elements. The error elements are probably exclusive, but
-    * we're not sure. */
-
-    // All results are lists to match `$x`
-    let possible_errors = {
-      complex_user_error: await scope.page.$$(`.alert-danger`),
-      // The below does not work if there's not a space between ':' and 'none'
-      simple_user_error: await scope.page.$$(`xpath/.//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]`),
-      system_error: await scope.page.$$(`xpath/.//meta[@content="docassemble: Error"]` ),
-    };
+  throwAnySystemError: async function ( scope ) {
+    /** 
+     * TODO: Stack trace won't show user answers. Could they reveal other
+     *   sensitive information?
+     * TODO: Add a state variable if any secret vars have been used thus far
+     *   and, if so, disallow pics, JSON, HTML, and page contents collecting.
+     *   In addition, allow author to disable all the above with a Step.
+     * 
+     * @param {Object} scope - Tests state object
+     * */
+    let sys_err_msg = `Was unable to get a specific system error message, sorry.`;
 
-    let error_handlers = {};
-    let was_found = false;
-    for ( let error_key in possible_errors ) {
-      if ( possible_errors[ error_key ][0] ) {
-        // If there's a page visible (locally), give us as much time to examine it as timeout allows.
-        // Because of previous actions, this may still be a bit too long.
-        // Also, if you're debugging and you close the window early, the process will still
-        // take this long to exit after the tests finish. Not sure how to account for that.
-        if ( session_vars.get_debug() ) {
-          await time.waitForTimeout( ( scope.timeout - (1 * 1000)) );
-        }
-        was_found = true;
-        error_handlers[ error_key ] = possible_errors[ error_key ][0];
-      }
-    }
+    let da_sys_err = await scope.page.$(`xpath/.//meta[@content=": Error"]|.//*[contains(@id, "damain")]//h1[@content="Error"]`);
+    // `al_custom_error_action` in base64. This acts like a continue screen, but actually replaces system error pages.
+    let al_custom_err = await scope.page.$(`.question-custom-error-action .daattributions #trigger[data-variable="YWxfY3VzdG9tX2Vycm9yX2FjdGlvbg"]`);
+
+    // if ( day_sys_err ) {
+    //   sys_err_msg = await scope.get_da_system_err( scope );
+    // } else if ( al_custom_err ) {
+    //   sys_err_msg = await scope.get_al_system_err( scope );
+    // }
 
-    return { was_found, error_handlers };
-  },  // Ends scope.checkForError()
+    error_indicator = da_sys_err || al_custom_err;
 
-  throwPageError: async function ( scope, { error_handlers = {}, id = '' }) {
-    /* Throw a cucumber error with the error that appeared on the page. */
-    if ( error_handlers.system_error !== undefined ) {
-      let sys_err_message = `The test tried to continue to the next page after ${ id }, but there was a system error. Check the picture of this page.`
-      reports.addToReport(scope, { type: `error`, code: `ALK0147`, value: sys_err_message });
-      expect( was_system_error, sys_err_message ).to.be.false;
+    if ( error_indicator ) {
+      let sys_err_msg = await scope.get_system_error_page_sample(
+        scope, { error_indicator }
+      );
 
-    } else {
-      // If it wasn't a system error, it was a user error
+      // TODO: Discuss adding some logs to just a Scenario's report as opposed
+      //   to full report.
+      reports.addToReport( scope, { type: `error`, code: `ALK0147`, value: sys_err_msg });
+      throw new Error( sys_err_msg );
+    }
+  },
+
+  get_invalid_answer_indicators: async function ( scope ) {
+    /**
+     * Return any elements associated with invalid user answers. Avoid throwing.
+     *   Invalid answers may be the author's goal.
+     * 
+     * Note: May later want the actual text of the error messages.
+     * 
+     * @param {Object} scope - Tests state object
+     * */
+    // TODO: While alerts come at the top of the page for validation errors,
+    //   an author can also use them as flavor text on the page. Ensure that
+    //   the latter doesn't trigger failure.
+    let invalid_indicator1 = await scope.page.$$(`.alert-danger`);
+    // ⚠️ 'display:' and 'none' MUST be separated by a space
+    let invalid_indicator2 = await scope.page.$$(`xpath//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]`);
+    invalidity_indicators = [ ...invalid_indicator1, ...invalid_indicator2 ];
+
+    return invalidity_indicators;
+  },
+
+  throw_on_invalid_answers: async function ( scope ) {
+    /**
+     * Throw an error if the user has given invalid answers.
+     * 
+     * @param {Object} scope - Tests state object
+     * */
+    let invalidity_indicators = await scope.get_invalid_answer_indicators( scope );
+    if ( invalidity_indicators.length > 0 ) {
+      let { id } = await scope.examinePageID( scope, 'none to match' );
       let user_err_message = `The test tried to continue to the next page after ${ id }, but the user got an error. Check the picture of this page.`
       reports.addToReport(scope, { type: `error`, code: `ALK0148`, value: user_err_message });
-      expect( true, user_err_message ).to.be.false;
+      expect( invalidity_indicators, user_err_message ).isEmpty([]);
+    }
+  },
+
+  // getPageError: async function ( scope ) {
+  //   /** 
+  //    * Returns an object with a system/user error elem if they exist, As well
+  //    *   as other data.
+  //    * 
+  //    * TODO: Discuss how we can split system vs. user errors more effectively.
+  //    * 
+  //    * @param {Object} scope - Tests state object
+  //    * */
+
+  //   // let possible_errors = {
+  //   //   // TODO: Ensure that intentional callouts don't trigger failure since they
+  //   //   // too use .alert-danger I believe (bootstrap component)
+  //   //   user_alert_error: await scope.page.$$(`.alert-danger`),
+  //   //   // ⚠️ The below does not work if there's not a space between ':' and 'none'
+  //   //   user_field_error: await scope.page.$$(`xpath/.//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]`),
+  //   //   // ⚠️ This xpath assumes this content stays in the metadata of the page
+  //   //   system_error: await scope.page.$$(`xpath/.//meta[@content=": Error"]|.//*[contains(@id, "damain")]//h1[@content="Error"]` ),
+  //   //   // Hardcoding the `al_custom_error_action` screen (that's what the base64 here is). It hides actual errors and looks like a continue screen, but isn't.
+  //   //   al_custom_system_error: await scope.page.$$(`.question-custom-error-action .daattributions #trigger[data-variable="YWxfY3VzdG9tX2Vycm9yX2FjdGlvbg"]`),
+  //   // };
+
+  //   let result = { ok: true, type: ``, element: null, msg: `` };
+
+  //   let error_patterns = [
+  //     // TODO: While alerts come at the top of the page for validation errors,
+  //     //   an author can also use them as flavor text on the page. Ensure that
+  //     //   the latter doesn't trigger failure.
+  //     { type: `user_alert`, selector: `.alert-danger` },
+  //     // ⚠️ 'display:' and 'none' MUST have a space between them
+  //     { type: `user_field`, selector: `xpath//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]` },
+  //     { type: `system_da`, selector: `xpath/.//meta[@content=": Error"]|.//*[contains(@id, "damain")]//h1[@content="Error"]` },
+  //     // `al_custom_error_action` in base64. This acts like a continue screen, but actually replaces system error pages.
+  //     { type: `system_al`, selector: `.question-custom-error-action .daattributions #trigger[data-variable="YWxfY3VzdG9tX2Vycm9yX2FjdGlvbg"]` },
+  //   ];
+  //   for ( let pattern of error_patterns ) {
+  //     let element = await scope.page.$( pattern.selector );
+  //     if ( element ) {
+  //       result.type = type: pattern.type;
+  //       result.element = element;
+  //     }
+  //   }
+
+  //   if ( result.type.includes(`system`)) {
+  //     result.ok = false;
+  //     result.msg = await scope.get_system_error_page_sample(
+  //       scope, { error_element: result.element }
+  //     );
+  //   } else if ( result.type.includes(`user`) ) {
+  //     // Invalid user answers may be intentional and ok.
+  //     result.msg = `User gave invalid answer.`
+  //   }
+
+  //   return result;
+
+
+
+
+
+
+  //   // let error = { type: `user_alert`, element: await scope.page.$$(`.alert-danger`) }
+  //   // if ( !error.element ) {
+  //   //   error = { type: `user_field`, element: await scope.page.$$(`xpath/.//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]`) }
+  //   // }
+  //   // if ( !error.element ) {
+  //   //   error = { type: `system_da`, element: await scope.page.$$(`xpath/.//meta[@content=": Error"]|.//*[contains(@id, "damain")]//h1[@content="Error"]` ) }
+  //   // }
+  //   // if ( !error.element ) {
+  //   //   error = { type: `system_al`, await scope.page.$$(`.question-custom-error-action .daattributions #trigger[data-variable="YWxfY3VzdG9tX2Vycm9yX2FjdGlvbg"]`) }
+  //   // }
+
+  //   // let error_elems_by_type = {};
+  //   // let was_found = false;
+  //   // let type = null;
+  //   // for ( let error_key in possible_errors ) {
+  //   //   error_elems_by_type[ error_key ] = possible_errors[ error_key ][0];
+  //   //   if ( possible_errors[ error_key ][0] ) {
+  //   //     // If there's a page visible (locally), give us as much time to examine it as timeout allows.
+  //   //     // Because of previous actions, this may still be a bit too long.
+  //   //     // Also, if you're debugging and you close the window early, the process will still
+  //   //     // take this long to exit after the tests finish. Not sure how to account for that.
+  //   //     if ( session_vars.get_debug() ) {
+  //   //       await time.waitForTimeout( ( scope.timeout - (1 * 1000)) );
+  //   //     }
+  //   //     if ( error_key.includes(`user`) ) { type = `user`; }
+  //   //     if ( error_key.includes(`system`) ) { type = `system`; }
+  //   //     was_found = true;
+  //   //   }
+  //   // }
+
+  //   // return { was_found, type, error_elems_by_type };
+  // },  // Ends scope.getPageError()
+
+  get_system_error_page_sample: async function ( scope ) {
+  // get_da_system_err: async function ( scope ) {
+    /**
+     * Returns message for a docassemble system error page that includes a
+     *   sample of the page contents
+     * 
+     * @param {Object} scope - Tests state object
+     * 
+     * @returns {string}
+     * */
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    let page_err_text = `Unable to get the text on this page. Check the picture of this page if possible.`
+    try {
+      
+      page_err_text = await scope.page.evaluate(() => {
+        return (
+            ( document.querySelector(`#YWxfZm9ybWF0dGVkX2Vycm9yX21lc3NhZ2U`)
+            || document.querySelector(`.dabody`)
+          ).textContent
+          .split(`\n`)
+          .slice( 0, 8 )
+          .join(`\n`)
+          .trim()
+          + `\n...`
+        );
+      });
+
+    } catch ( pupp_err ) {
+      log.debug({ code: `ALK0281`, context: `internal`, },
+        `Skipped getting system error screen text with puppeteerjs:`, pupp_err,
+      );
+    }
+
+    return `Ran into an error page after page id "${ id }".
+
+${ page_err_text }`;
+
+  },
+
+  get_al_system_err: async function ( scope ) {
+    /**
+     * Returns message for a docassemble system error page that includes a
+     *   sample of the page contents
+     * 
+     * @param {Object} scope - Tests state object
+     * 
+     * @returns {string}
+     * */
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    let page_err_text = `Unable to get the text on this page. Check the picture of this page if possible.`
+    try {
+      
+      page_err_text = await scope.page.evaluate(() => {
+        return (
+          // b64 for `al_formatted_error_message`, which seems most relevant
+          document.querySelector(`#YWxfZm9ybWF0dGVkX2Vycm9yX21lc3NhZ2U`)
+          .textContent
+          .split(`\n`)
+          .slice( 0, 10 )
+          .join(`\n`)
+        );
+      });
+
+    } catch ( pupp_err ) {
+      log.debug({ code: `ALK0`, context: `internal`, },
+        `Skipped getting system error text on Assembly Line custom error screen with puppeteerjs:`, pupp_err,
+      );
     }
 
-    return scope;
-  },  // Ends scope.throwPageError()
+    return `Ran into an error page after page id "${ id }".
+
+${ page_err_text }`;
+
+  },
+
+  // warnOrThrowOnInvalidPage: async function ( scope, { page_result={} }) {
+  //   /**
+  //    * Given a page with invalid user input or a system error page, warn about
+  //    *   it or throw an error.
+  //    * 
+  //    * @param {Object} scope - Tests state object
+  //    * @param {Object} obj - Named arguments
+  //    * @param (Object) obj.page_result - Data from examining a docassemble page
+  //    * */
+  //   let error = page_result?.error;
+  //   if ( page_result && !page_result.ok ) {
+  //     if ( error.type.includes(`system`)) {
+  //       // Throw on a system error or an unhandled error of our own
+  //       reports.addToReport( scope, { type: `error`, code: `ALK0147`, value: error.msg });
+  //       throw new Error( error.msg );
+  //     } else {
+  //       reports.addToReport( scope, { type: `warn`, code: `ALK0282`, value: error.msg });
+  //     }
+  //   }
+  // },
+
+  // checkForError: async function ( scope ) {
+  //   /* Returns whether any system or user error elements were found and, if so,
+  //   * returns those elements. The error elements are probably exclusive, but
+  //   * we're not sure. */
+
+  //   // All results are lists to match `$x`
+  //   let possible_errors = {
+  //     // TODO: Ensure that intentional callouts don't trigger failure since they
+  //     // too use .alert-danger I believe (bootstrap component)
+  //     user_alert_error: await scope.page.$$(`.alert-danger`),
+  //     // ⚠️ The below does not work if there's not a space between ':' and 'none'
+  //     user_field_error: await scope.page.$$(`xpath/.//*[contains(@class, "da-has-error")][not(contains(@style,"display: none"))]`),
+  //     // ⚠️ This xpath assumes this content stays in the metadata of the page
+  //     system_error: await scope.page.$$(`xpath/.//meta[@content=": Error"]|.//*[contains(@id, "damain")]//h1[@content="Error"]` ),
+  //     // Hardcoding the `al_custom_error_action` screen (that's what the base64 here is). It hides actual errors and looks like a continue screen, but isn't.
+  //     al_custom_system_error: await scope.page.$$(`.question-custom-error-action .daattributions #trigger[data-variable="YWxfY3VzdG9tX2Vycm9yX2FjdGlvbg"]`),
+  //   };
+
+  //   let error_elems_by_type = {};
+  //   let was_found = false;
+  //   let type = null;
+  //   for ( let error_key in possible_errors ) {
+  //     error_elems_by_type[ error_key ] = possible_errors[ error_key ][0];
+  //     if ( possible_errors[ error_key ][0] ) {
+  //       // If there's a page visible (locally), give us as much time to examine it as timeout allows.
+  //       // Because of previous actions, this may still be a bit too long.
+  //       // Also, if you're debugging and you close the window early, the process will still
+  //       // take this long to exit after the tests finish. Not sure how to account for that.
+  //       if ( session_vars.get_debug() ) {
+  //         await time.waitForTimeout( ( scope.timeout - (1 * 1000)) );
+  //       }
+  //       if ( error_key.includes(`user`) ) { type = `user`; }
+  //       if ( error_key.includes(`system`) ) { type = `system`; }
+  //       was_found = true;
+  //     }
+  //   }
+
+  //   return { was_found, type, error_elems_by_type };
+  // },  // Ends scope.checkForError()
+
+  // processPageError: async function ( scope, { error_elems_by_type = {}, id = `None` }) {
+  //   /**
+  //    * Throw a cucumber error with the error that appeared on the page.
+  //    * 
+  //    * Discuss: Should we show the text on the page?
+  //    * 
+  //    * @param {Object} scope - Tests state object
+  //    * @param {Object} obj - Named arguments
+  //    * @param {Object} obj.error_elems_by_type - A map of DOM Node | undefined
+  //    *     by type of error
+  //    * @param {string} obj.id - The current page id
+  //    * */
+  //   let error_elems_by_type = error_data.error_elems_by_type
+  //   // TODO: #656, wasn't checking for this error specifically. May be almost all we need for that.
+  //   if ( error_elems_by_type.system_error !== undefined || error_elems_by_type.al_custom_system_error !== undefined ) {
+  //     return await scope.throwAnySystemError( scope, { error_elems_by_type, id });
+  //   }
+
+  //   // If it wasn't a system error, it was a user error
+  //   // TODO: Explore whether this still allows an author to check for correctly
+  //   // triggering user input invalidation
+  //   let user_err_message = `The test tried to continue to the next page after ${ id }, but the user got an error. Check the picture of this page.`
+  //   reports.addToReport(scope, { type: `error`, code: `ALK0148`, value: user_err_message });
+  //   expect(
+  //     ( error_elems_by_type.user_alert_error == undefined && error_elems_by_type.user_field_error == undefined ),
+  //       user_err_message
+  //   ).to.be.true;
+
+  //   return scope;
+  // },  // Ends scope.processPageError()
+
+//   getPageError: async function( scope, { error_elems_by_type = {}, id = `None` }) {
+//     /**
+//      * Throw an error if the author's code caused a system error page to appear.
+//      * 
+//      * @param {Object} scope - Tests state object
+//      * @param {Object} obj - Named arguments
+//      * @param {Object} obj.error_elems_by_type - A map of DOMNode | undefined
+//      *     by type of error
+//      * @param {string} obj.id - The current page id
+//      * */
+//     // Get the relevant text on the page
+//     let err_elem = error_elems_by_type.system_error || error_elems_by_type.al_custom_system_error;
+//     let page_err_text = '';
+//     if ( err_elem ) {
+//       page_err_text = await get_page_error_text( scope, { error_elem = err_elem });
+//     }
+
+//     // Add to the report
+//     let sys_err = new Error(`The test tried to continue to the next page after ${ id }, but there was a system error.
+
+// ${ page_err_text }`);
+//     reports.addToReport(scope, { type: `error`, code: `ALK0147`, value: sys_err });
+
+//     // expect(
+//     //   ( error_elems_by_type.system_error == undefined && error_elems_by_type.al_custom_system_error == undefined ),
+//     //     sys_err
+//     // ).to.be.true;
+
+//     return sys_err;
+//   },
+
+  // get_page_error_text: async function( scope, { error_elem }) {
+  //   /**
+  //    * Given an element on an error page, return the relevant content.
+  //    * 
+  //    * @param {object} scope - Tests state object
+  //    * @param {object} obj - Named arguments
+  //    * @param {Object} obj.error_elem - Element that indicated a system err
+  //    * */
+  //   let page_err_text = `Unable to get the text on this page. Check the picture of this page if possible.`
+  //   try {
+      
+  //     page_err_text = await page.evaluate(( err_elem ) => {
+        
+  //       let text_container = (
+  //         err_elem.closest(`.dabody`)
+  //         || err_elem.closest(`#dabody`)
+  //       ).querySelector(`* > .danavbarcontainer ~ *`);
+  //       return text_container.textContent;
+
+  //     }, err_elems[0]);
+
+  //   } catch ( pupp_err ) {
+  //     log.debug({ code: `ALK0281`, context: `internal`, },
+  //       `Skipped getting system error screen text with puppeteerjs:`, pupp_err,
+  //     );
+  //   }
+
+  //   return page_err_text;
+  // },
 
   take_a_screenshot: async ( scope,{ path }) => {
     /* Takes a jpeg screenshot. Avoids destroying signatures. */
@@ -2930,7 +3429,7 @@ module.exports = {
       /** Given a fetch response headers' Content-Disposition str, return the
        *  filename of the response's file.
        *
-       * @return { string | null } - Name of fetched file
+       * @returns { string | null } - Name of fetched file
        * */
       let filename = default_name;
       if ( disposition ) {

package/lib/steps.js

@@ -400,7 +400,7 @@ When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/i, {timeout: -1}, async (
       // If we need to call recursively, signature function might look like:
       // func( scope, { resolve, reject, supported_table, page_timeout_id })
       // Maybe list of timeout and/or interval ids in state that can all be cleared at once?
-      (async function() {
+      (async function setFieldsFromStoryTable() {
         try {
           let id = await scope.get_page_id( scope );
 
@@ -416,25 +416,28 @@ When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/i, {timeout: -1}, async (
             `Rows, including special rows:`, ensured_var_data
           );
 
-          // THIS FUNCTION ACTUALLY MATTERS
-          // Some errors happen in there to be closer to the relevant code. We'd have to try/catch anyway.
-          let error = await scope.setFields(scope, { var_data: ensured_var_data, id, ensure_navigation: true });
+          let result = await scope.setFields( scope, {
+            var_data: ensured_var_data, id, ensure_navigation: true
+          });
+
+          resolve( result );
+
+        // } catch ( err ) {
+        //   // // Discuss: This error message seems unnecessary cruft
+        //   // let err_msg = `ALKiln ran into an error when it tried to answer fields on the page with the question id "${ id }". The text above might have more information.`
+        //   // reports.addToReport( scope, { type: `error`, code: `ALK0062`, value: err_msg });
+
+        //   // Catch a system error or an unhandled error
+        //   // A system page error trace here may be more useful than in a deeper function
+        //   reject( err );
 
-          // Discuss: use Log class to throw this error?
-          if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
-          resolve();
-        } catch ( err ) {
-          let err_msg = `ALKiln ran into an error when it tried to answer fields on the page with the question id "${ id }". The text above might have more information.`
-          reports.addToReport( scope, { type: `error`, code: `ALK0062`, value: err_msg });
-          // Throw error instead?
-          reject( err );
         } finally {
           clearTimeout( page_timeout_id );  // prevent temporary hang at end of tests
         }
       })();
-    });  // ends custom_timeout for setFields
+    });  // ends custom_timeout for filling in fields
 
-    await custom_timeout;
+    let page_result = await custom_timeout;
 
     // Prep for next loop
     id = await scope.get_page_id( scope );
@@ -641,7 +644,7 @@ Then('I will be told an answer is invalid', async () => {  // √
 
   let { was_found, error_handlers } = await scope.checkForError( scope );
 
-  let no_error_msg = `No error message was found on the page`;
+  let no_error_msg = `The user should have been told their answer was invalid, but that text is missing on the page`;
   if ( !was_found ) { reports.addToReport(scope, { type: `error`, code: `ALK0076`, value: no_error_msg }); }
   expect( was_found, no_error_msg ).to.be.true;
 
@@ -653,6 +656,11 @@ Then('I will be told an answer is invalid', async () => {  // √
   await scope.afterStep(scope);
 });
 
+// ==============
+// Welp, we have a diversion (da DOM changes)
+// Resume here when ready
+// ==============
+
 Then('I arrive at the next page', async () => {  // √
   /* Tests for detection of url change from button or link tap.
   *    Can other things trigger navigation? Re: other inputs things
@@ -820,6 +828,8 @@ Then(/the "([^"]+)" link opens a working page/, { timeout: -1 }, async ( linkTex
   );
 });
 
+// #656, problem: tap to continue doesn't go through the full Story Table cycle
+// and thus doesn't get any page errors detected.
 When(/I tap to continue/i, { timeout: -1 }, async () => {
   // Any selectors for this seem somewhat precarious
   // No data will cause an automatic continue when `ensure_navigation` is true
@@ -849,7 +859,7 @@ When(/I tap the "([^"]+)" element and wait (?:for )?([0-9]+) second(?:s)?/i, { t
   // Also, this is always supposed to navigate, and that handles its own waiting.
 
   // `page` timeout will deal with custom timeout
-  await scope.tapSelectorToNavigate( scope, elemSelector, waitTime * 1000 )
+  await scope.tapSelectorToNavigate( scope, elemSelector, waitTime * 1000 );
 });
 
 When(/I check the page for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
@@ -972,6 +982,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
             buttons_exist = await scope.continue_exists( scope );
             if ( buttons_exist ) {
               await scope.continue( scope );
+              // TODO: Use warnOrThrowInvalidPage to error on error page
             }  // ends if continue exists
           }  // ends if buttons_exist
           if (!( await scope.is_on_server_still(scope))) {
@@ -1458,18 +1469,20 @@ After(async function(scenario) {
       // test), it failed. If is was supposed to pass and failed, it also
       // failed. E.g. a11y failures let the Scenario pass initially, but
       // should cause a failure here.
-      changeable_test_status = `FAILED`;
-      // Flag this failure to trigger an error later, so cucumber shows
-      // it as failed too.
-      expected_status_was_met = false;
-      // Keep it general. Users may see this message, e.g. for a11y tests.
+
+      // Keep this msg general. Users may see this message, e.g. for a11y tests.
       let msg = `Test ${ changeable_test_status }. `
-          + `Expected status: ${scope.expected_status}. `
+          + `Expected status: ${ scope.expected_status }. `
           + `See the report for details.`;
       let code = `ALK0099`;
       expected_status_err_msg = log.debug({ code: code, level: `error`, },
         msg
       );
+
+      // Flag this failure to trigger an error later, so cucumber shows
+      // it as failed too.
+      changeable_test_status = `FAILED`;
+      expected_status_was_met = false;
     }
   }
   scope.expected_status = null;  // reset for next Scenario

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.16.1",
+  "version": "5.17.0-fix-stop-on-error-1",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {