@suffolklitlab/alkiln 5.16.1-always-html-5 → 5.16.1-always-html-6

package/CHANGELOG.md

@@ -47,19 +47,20 @@ Format:
 
 ### Changed
 
-- On pages with sensitive answers, store the HTML of the page. The HTML excludes field values, so those sensitive answers will not be in the saved file. Still avoid taking a pic of the screen, which would reveal sensitive answers. NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML.
-
+- On pages with sensitive answers, store the HTML of the page. The HTML excludes field values, so those sensitive answers will not be in the saved file. Still avoid taking a pic of the screen, which would reveal sensitive answers. ‼️ NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML. See #1099
 - GitHub action release: Restored our GitHub action's default for ALKiln version to the latest version 5 again. Released first on GitHub actions. NPM release will come in time, but npm has no impact on GitHub action releases.
 
 ### Fixed
 
-- Fixed GitHub+You action outdated docassemble cli version causing Scenario timeouts.
+- Updated docassemble cli version.
+- Better detect navigation when email and password fail to sign in to a docassemble server account.
 
 ### Internal
 
-- Break out the function in `Before()` to attempt better error tracing. Goal: try to repeat this for other functions in that file.
-
+- Break out the function in `Before()` and `After()` to attempt better error tracing. Goal: try to repeat this for other functions in that file.
 - Updated both of our actions' dependencies (the checkout, setup-node, setup-python, upload-artifacts, download-artifacts actions). Closes [#1095](https://github.com/suffolkLITLab/aLKiln/issues/1095). Once again, action related.
+- When our report phrases are missing, list all missing phrases at one time in our custom message. The cucumber message will stay the same.
+- Silence errors from screenshots & HTML downloads when trying to provide more information about failing tests. Those records are nice to have, but not absolutely necessary, and if a server is busy puppeteer will rightly have lost execution context, preventing interactions with the page and we don't need a noisy error about it. Log any problems.
 
 ## [5.16.1] - 2026-06-01
 

package/lib/scope.js

@@ -3090,7 +3090,6 @@ module.exports = {
       /** Allow the developer to log a user into their server using GitHub
       * secrets to authenticate.
       */
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-1.jpg`});
       // Don't take a picture of a failed login in case one of the inputs is correct
       scope.disable_error_screenshot = true;
       
@@ -3102,7 +3101,6 @@ module.exports = {
 
       // Go to the sign in page
       let login_url = `${ session_vars.get_da_server_url() }/user/sign-in`;
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-2.jpg`});
       
       // TODO: implement and use scope.handle_possible_timeout()
       try {
@@ -3110,13 +3108,10 @@ module.exports = {
         await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout });
         await scope.page.waitForSelector( `.dabody` );
 
-        await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-3.jpg`});
-
       } catch ( error ) {
 
         let err_msg = `Error occurred when ALKiln tried to go to "${ login_url }".`
         if ( error.name === `TimeoutError` ) {
-          // console.log(`🖊️ 110`);
           let non_reload_report_msg = `It took too long to load "${ login_url }"`;
           await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
             code: `ALK0159`,
@@ -3124,8 +3119,6 @@ module.exports = {
           }, error });
         } else {
           // Throw any non-timeout error
-          // console.log(`🖊️ 111`);
-          await scope.take_a_screenshot(scope, {path:`./_alkiln_temp/Temp-4.jpg`});
           reports.addToReport( scope, { type: `error`, code: `ALK0160`, value: err_msg });
           throw error;
         }  // ends if error is timeout error
@@ -3166,8 +3159,6 @@ module.exports = {
           scope.scenarios.get( scope.scenario_id ).api_keys.push( api_key );
         }
       }
-      // console.log(`🖊️ 112`);
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-4.jpg`});
 
       expect( email, email_msg ).to.not.equal( undefined );
       expect( password, password_msg ).to.not.equal( undefined );
@@ -3177,14 +3168,9 @@ module.exports = {
       await scope.page.type( `#password`, password );
       let elem = await scope.page.$( `button[type="submit"]` );
       await scope.guard_against_missing_tap_element(scope, { elem });
-      // console.log(`🖊️ 113`);
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-5.jpg`});
 
       // Submit and see what happens
       let winner = await scope.steps.race_sign_in_navigation( scope, { elem, login_url });
-      // console.log(`🖊️ 113.5 ${ winner }`);
-      // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-6.jpg`});
-      // console.log(`🖊️ 113.6 ${ winner }`);
       // Add the result to the report and possibly throw errors
       if ( winner[0] === `success` ) {
         reports.addToReport( scope, {
@@ -3197,7 +3183,6 @@ module.exports = {
           type: `error`, code: `ALK0209`,
           value: `Failed to sign into ${ login_url }. Make sure you followed the instructions at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#sign-in.`
         });
-        // console.log(`🍉 ${ error_msg }`);
         throw new Error( error_msg );
       } else if ( winner[0] === `error` ) {
         let error_msg = reports.addToReport( scope, {
@@ -3206,94 +3191,63 @@ module.exports = {
         });
         throw new Error( error_msg );
       }
-      // console.log(`🖊️ 114`);
-      // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-7.jpg`});
-      // if ( error_msg ) {  // undefined sometimes now
-      //   return [ error_msg ];
-      // } else {
-      //   return [];
-      // }
     },  // Ends scope.steps.sign_in()
 
     race_sign_in_navigation: async function ( scope, { elem, login_url='unknown' }) {
       /** Wait for sign in navigation success or failure, or system error. */
 
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-0.jpg`});
-
       // After everything, clean up incomplete promises
       const controller = new AbortController;
-      // console.log(`⏳ 200`);
 
       // Redirect
-      let redirect_promise = scope.page.waitForResponse(async ( response ) => {
-        // console.log(`⏳ 300`);
+      let redirect_promise = scope.page.waitForResponse(function ( response ) {
         return response.status() === 302;
       }, { signal: controller.signal })
-      .catch(async ( catch_info ) => {
-        // console.log(`⏳ 201`);
-        // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-1.jpg`});
+      .catch(( catch_info ) => {
         log.debug({ code: `ALK0204`, context: `nav`, },
           `302 sign-in wait catch error:`, catch_info.name, catch_info,
         );
       });
 
       // Invalid credentials, no navigation, no sign in
-      let wrong_sign_in_promise = scope.page.waitForResponse(async ( response ) => {
-        // console.log(`⏳ 301`);
+      let wrong_sign_in_promise = scope.page.waitForResponse(function ( response ) {
         return response.status() >= 200 && response.status() < 300 && response.url().includes(`/sign-in`);
       }, { signal: controller.signal })
-      .catch(async ( catch_info ) => {
-        // errors?
-        // console.log(`⏳ 202`);
-        // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-2.jpg`});
+      .catch(( catch_info ) => {
         log.debug({ code: `ALK0205`, context: `nav`, },
           `200s sign-in wait catch error:`, catch_info.name, catch_info,
         );
       });
 
       // System error
-      let error_promise = scope.page.waitForResponse(async ( response ) => {
-        // console.log(`⏳ 302`);
+      let error_promise = scope.page.waitForResponse(function ( response ) {
         return response.status() >= 500 && response.status() < 600;
       }, { signal: controller.signal })
-      .catch(async ( catch_info ) => {
-        // console.log(`⏳ 203`);
-        await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-3.jpg`});
+      .catch(( catch_info ) => {
         log.debug({ code: `ALK0206`, context: `nav`, },
           `500s sign-in wait catch error:`, catch_info.name, catch_info,
         );
       });
 
-      // Duh, the other catches were triggered because those promises were aborted
-
       let click_promise = elem.click();  // MUST complete
       let nav_promise = scope.nav_race(scope, {}).result;
-      let pic_promise = true; // scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-10.jpg`});
-      // console.log(`⏳ 204`);
       const winner = await Promise.race([
-        Promise.all([ `success`, click_promise, pic_promise, redirect_promise ]),
-        Promise.all([ `failure`, click_promise, pic_promise, wrong_sign_in_promise ]),
-        Promise.all([ `error`, click_promise, pic_promise, error_promise ]),
+        Promise.all([ `success`, click_promise, redirect_promise ]),
+        Promise.all([ `failure`, click_promise, wrong_sign_in_promise ]),
+        Promise.all([ `error`, click_promise, error_promise ]),
       ]).catch(async ( error ) => {
-        // console.log(`⏳ 205`);
-        // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-4.jpg`});
         let error_msg = reports.addToReport( scope, {
           type: `error`, code: `ALK0207`,
           value: `Unknown error waiting for results during sign in at ${ login_url }.`
         });
-        // console.log(`⏳ 205.1`, error);
         throw error;
       });
 
-      // console.log(`⏳ 206`);
-      // // await waitForTimeout(500);
-      // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-5.jpg`});
-
       // Clean up unresolved promises
       controller.abort();
-      // console.log(`⏳ 207`);
+      // Wait to finish navigating
+      // TODO: Test the winner 'error' under this circumstance
       await nav_promise;
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp3-1.jpg`});
 
       log.debug({ code: `ALK0208`, context: `nav` },
         `Sign-in winner:`, winner

package/lib/steps.js

@@ -116,7 +116,6 @@ BeforeAll(async function() {
 
 Before( beforeScenario );
 async function beforeScenario( scenario ) {
-  // console.log('👁️ 111 start beforeScenario');
   // Start the running "progress bar" for the Scenario
   log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `);
 
@@ -126,20 +125,15 @@ async function beforeScenario( scenario ) {
       // Will only run in the Playground outside of a sandbox. TODO: There's a
       // better way to do this, though it's more complicated. See comments in
       // https://github.com/SuffolkLITLab/ALKiln/issues/661
-      // console.log('👁️ 112 no browser, await new local browser');
       scope.browser = await scope.driver.launch({ args: ['--no-sandbox'] });
     } else {
-      // console.log('👁️ 113 no browser, await new remote browser');
       scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() });
     }
   }
-  // console.log('👁️ 114 we have ensured browser exists, awaiting pages');
   // Clean up all previously existing pages
   for (const page of await scope.browser.pages()) {
-    // console.log('👁️ 115 awaiting closing page');
     await page.close();
   }
-  // console.log('👁️ 116 awaiting opening a new page');
   // Make a new page
   scope.page = await scope.browser.newPage()
 
@@ -153,10 +147,8 @@ async function beforeScenario( scenario ) {
   scope.server_reload_promise = null;
 
   reports.addReportHeading(scope, {scenario});
-  // console.log('👁️ 117 awaiting getting safe scenario name');
   // Make folder for this Scenario in the all-tests artifacts folder
   scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario});
-  // console.log('👁️ 118 got safe scenario name');
   // Add a date for uniqueness in case dev has accidentally copied a Scenario description
   let date = Date.now();
   scope.paths.scenario = `${ scope.paths.artifacts }/${ scope.base_filename }-${ date }`;
@@ -297,10 +289,8 @@ Given(/I go to "([^"]+)"/i, {timeout: -1}, async ( url ) => {
   return result;
 });
 
-foo = null
 Given(
   /I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^"]+)",?(?: and)?(?: the password)? "([^"]+)"(?: SECRETs)?(?:,?(?: and)?(?: the API key)? "([^"]+)")?/i,
-  { timeout: -1 },
   async ( email, password, api_key ) => {
   /** Uses the names of environment variables (most often GitHub SECRETs) to
   *     log into an account on the user's server. Must be secure.
@@ -315,21 +305,11 @@ Given(
   // `page` timeout will deal with custom timeout
   // Couldn't test the timeout for tapping the button because there's not
   // enough of a pause between initial navigation and pressing button.
-  foo = scope.page
-  // console.log(`⭐️ ${ JSON.stringify(scope.page, null, 2) }`)
   await scope.steps.sign_in( scope, {
     email_secret_name: email,
     password_secret_name: password,
     api_key_secret_name: api_key
   });
-
-
-  // console.log(`🖊️ 115`);
-  // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-8.jpg`});
-
-  // if ( !result.ok && result.errors.length > 0 ) {
-  //   throw result.errors[ result.errors.length - 1 ];
-  // }
 });
 
 // I am using a mobile/pc
@@ -1343,8 +1323,6 @@ Then(/I fail to delete (\d) detected interview(?:s)? and get no error __internal
 
 AfterStep(async function({ result }) {
   // TODO: Abstract this
-  // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-13.jpg`});
-  // console.log(`🛑 100`);
   if ( result.status === `PASSED` ) { log.stdout({ records_only: true }, `.`); }
   else if ( result.status === `FAILED` ) { log.stdout({ records_only: true }, `F`); }
   else if ( result.status === `UNDEFINED` ) { log.stdout({ records_only: true }, `U`); }
@@ -1352,13 +1330,10 @@ AfterStep(async function({ result }) {
   else if ( result.status === `PENDING` ) { log.stdout({ records_only: true }, `P`); }
 
   reports.outdent();
-  // console.log(`🛑 101`);
 });
 
 After(afterScenario);
 async function afterScenario(scenario) {
-  // console.log(`👀 1 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`);
-  await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-30.jpg`});
   // Log errors
   if ( scenario.result.message ) {
     log.debug({ code: `ALK0091`, context: `scenario`, },
@@ -1377,8 +1352,6 @@ async function afterScenario(scenario) {
       value: `Accessibility standards tests failed. See information above.`
     });
   }
-  // console.log(`👀 2`);
-  await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-9.jpg`});
 
   if ( scope.failed_pdf_compares.length > 0) {
     let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n―――\n${ new_msg }`)
@@ -1400,46 +1373,54 @@ async function afterScenario(scenario) {
     );
 
     if ( scope.page ) {
-      // console.log(`👀 3`);
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-10.jpg`});
 
       if ( !!scope.disable_error_screenshot ) {
         reports.addToReport(scope, {
           type: `row info`,
           code: `ALK0095`,
           // Discuss: Might people use secret variables to prevent any
-          // information about a screen from getting out? For example, they have
-          // proprietary info in the HTML itself.
-          value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML, which omits the values in the fields.`
+          // information about a screen from getting out? For example, they may
+          // have proprietary info in the HTML itself.
+          value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML as a file. The HTML omits the values in the fields.`
+        });
+      }
+
+      // Could have an error if page can't load or something similar
+      try {
+        // Save/download a picture of the screen that's showing during the unexpected status
+        // Save one copy in the outer-most artifact folder
+        let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` });
+        let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`;
+        await scope.take_a_screenshot( scope, {
+          path: path_outer,
+          disable_pic: scope.disable_error_screenshot,
+          disable_html: false,
+        });
+
+        // Save another copy in the artifact's Scenario folder
+        let screenshot_name = `error_on`;
+        let { id } = await scope.examinePageID( scope, 'none to match' );
+        let short_id = `${ id }`.substring(0, 20);
+        screenshot_name += `-${ short_id }`;
+        let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`;
+        await scope.take_a_screenshot( scope, {
+          path: path_scenario,
+          disable_pic: scope.disable_error_screenshot,
+          disable_html: false,
         });
+
+      } catch ( page_error ) {
+        // Fail silently. Our inability to take a pic shouldn't cause confusion
+        // about why a test failed
+        if ( page_error.message.lower().includes(`execution context`) ) {
+          reports.addToReport(scope, {
+            type: `row warning`,
+            code: `ALK0281`,
+            value: `ALKiln is unable to get any record of this page whatsoever, even the page HTML. Your server may be busy.`
+          });
+        }
+        log.debug({ code: `ALK0282`, level: `note`, }, page_error );
       }
-      // console.log(`👀 4 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`);
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-11.jpg`});
-
-      // Save/download a picture of the screen that's showing during the unexpected status
-      // Save one copy in the outer-most artifact folder
-      let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` });
-      let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`;
-      await scope.take_a_screenshot( scope, {
-        path: path_outer,
-        disable_pic: scope.disable_error_screenshot,
-        disable_html: false,
-      });
-      // console.log(`👀 5 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`);
-      await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-12.jpg`});
-
-      // Save another copy in the artifact's Scenario folder
-      let screenshot_name = `error_on`;
-      let { id } = await scope.examinePageID( scope, 'none to match' );
-      let short_id = `${ id }`.substring(0, 20);
-      screenshot_name += `-${ short_id }`;
-      let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`;
-      await scope.take_a_screenshot( scope, {
-        path: path_scenario,
-        disable_pic: scope.disable_error_screenshot,
-        disable_html: false,
-      });
-      // console.log(`👀 6`);
 
     }  // ends if scope.page exists
 
@@ -1447,59 +1428,48 @@ async function afterScenario(scenario) {
     if (changeable_test_status === `FAILED`) {
       reports.addToReport(scope, { type: `outcome failure info`, code: `ALK0096`, value: `**-- Scenario Failed --**` });
     }
-    // console.log(`👀 7`);
 
   }  // ends if not passed
-  // console.log(`👀 8`);
 
   // Add report text to Scenario folder after everything has been added to the report
   let scenario_report_obj = scope.report.get( scope.scenario_id );
-  // console.log(`👀 8.01`);
   let report = reports.getPrintableScenario( scenario_report_obj );
   // Save the report as in the Scenario folder
-  // console.log(`👀 8.02`);
   fs.writeFileSync( scope.paths.scenario_report, report );
-  // console.log(`👀 8.03`);
   // TODO: Save any cucumber failure message for the Scenario here (instead of waiting till run_cucumber.js)
 
   // ---------------- Check internal test results ----------------
 
   if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
-    // console.log(`👀 8.04`);
     let data = await scope.reportIncludesAllExpected(
       scope,
       { expected: scope.expected_in_report }
     );
     let report_includes_all_expected_strings = data.all_included;
-    // console.log(`👀 8.05`);
     if ( !report_includes_all_expected_strings ) {
       changeable_test_status = `FAILED`;
       let msg = `These strings are missing from the report:
         ${ JSON.stringify(data.missing, null, 2) }
         Instead the report had this text:
         ${ report }`;
+      // These failure messages won't match exactly, which is a shame, but at
+      // least our devs will have more data _somewhere_ about their failure.
       expect( report, msg ).to.contain( data.missing[0] );
     }
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_in_report = null;
 
   }
-  // console.log(`👀 8.06`);
 
   if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) {
-    // console.log(`👀 8.07`);
     let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude(
       scope,
       { not_expected: scope.expected_not_in_report }
     );
-    // console.log(`👀 8.08`);
     if ( !all_prohibited_strings_were_absent ) { changeable_test_status = `FAILED`;}
-    // console.log(`👀 8.09`);
     // Reset report values no matter what so they don't mess up future scenarios
     scope.expected_not_in_report = null;
-    // console.log(`👀 8.091`);
   }
-  // console.log(`👀 9`);
 
   // ---------------- Ends internal results check ----------------
 
@@ -1537,7 +1507,6 @@ async function afterScenario(scenario) {
       );
     }
   }
-  // console.log(`👀 10`);
   scope.expected_status = null;  // reset for next Scenario
 
   let need_to_force_failed_status = false;
@@ -1578,15 +1547,12 @@ async function afterScenario(scenario) {
   let signout_error_msg = ``;
   // If there is a page open, then sign out and close it
   if ( scope.page ) {
-    // console.log(`👀 11`);
     // Catch a possible reload error.
     try {
       // puppeteer will ensure proper timeout.
-      // console.log(`👀 12`);
       await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`});
 
     } catch ( error ) {
-      // console.log(`👀 13`);
       signout_succeeded = false;
 
       if ( error.name === `TimeoutError` ) {
@@ -1613,7 +1579,6 @@ async function afterScenario(scenario) {
     }  // ends try/catch
 
   }  // ends if scope.page
-  // console.log(`👀 14`);
 
   // Now that we're no longer on the interview page, one way or
   // another, try to delete the interviews created during the test
@@ -1670,7 +1635,6 @@ async function afterScenario(scenario) {
       internal_errors
     );
   }
-  // console.log(`👀 15`);
 
   // Find the race condition.
   log.debug({ code: `ALK0100`, context: `scenario`, },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.16.1-always-html-5",
+  "version": "5.16.1-always-html-6",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {