@suffolklitlab/alkiln 5.12.0 → 5.13.0-logs2

package/lib/steps.js

@@ -5,6 +5,7 @@ const {
   Given,
   After,
   Before,
+  AfterStep,
   BeforeAll,
   AfterAll,
   setDefaultTimeout //,
@@ -18,10 +19,11 @@ const { v4: uuidv4 } = require('uuid');
 
 // Ours
 const scope = require('./scope');
-const log = require('./utils/log');
+const Log = require(`./utils/log`);
 const session_vars = require('./utils/session_vars');
 const files = require('./utils/files' );
 const reports = require(`./utils/reports`);
+const da_api = require('./docassemble/docassemble_api_interface');
 
 /* Of Note:
 - We're using `*=` for selectors because sometimes da text has funny characters in it that are hard to anticipate
@@ -81,8 +83,13 @@ let click_with = {
   pc: 'click',
 };
 
-BeforeAll(async () => {
+let log = null;
+
+BeforeAll(async function() {
   scope.showif_timeout = 600;
+  // TODO: Do these really need to be maps? As sets, they may make the output
+  // more predictable to the user, but they add complexity.
+  scope.scenarios = new Map();
   scope.report = new Map();
 
   // Start tracking whether server is up and running
@@ -91,29 +98,29 @@ BeforeAll(async () => {
   // Store path names for files and docs created for the tests
   scope.paths = {};
 
-  // TODO: figure out how to test file and folder creation stuff,
-  // especially when setup is excluded from the flow.
-  
-  // Make the folder to store all the files for this run of the tests
-  scope.paths.artifacts = files.make_artifacts_folder();
-  // Save its name in a file. `session_vars` can get it back later.
-  // The value we are saving is used by at least run_cucumber.js.
-  session_vars.save_artifacts_path_name( scope.paths.artifacts );
+  // // Make the folder to store all the files for this run of the tests
+  // scope.paths.artifacts = files.make_artifacts_folder();
+  // // Save its name in a file. `session_vars` can get it back later.
+  // // The value we are saving is used by at least run_cucumber.js.
+
+  // A little indirect, but the artifacts path should always have already been
+  // created by now, either by `run_cucumber.js` or the relevant test
+  scope.paths.artifacts = session_vars.get_artifacts_path_name();
+  log = new Log({ path: scope.paths.artifacts, context: `steps` });
+  scope.set_log({ log_obj: log });
 
-  scope.paths.report = `${ scope.paths.artifacts }/report.txt`;
+  scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_filename }`;
+  scope.paths.report_log = `${ scope.paths.artifacts }/${ log.report_log_filename }`;
+  scope.paths.report = `${ scope.paths.artifacts }/${ log.report_filename }`;
   scope.paths.failures = `${ scope.paths.artifacts }/${ log.unexpected_filename }`;
-  scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_file }`;
-  fs.writeFileSync( scope.paths.debug_log, "" );
 
   reports.create( scope );
 });
 
 Before(async (scenario) => {
 
-  // Useful for those with a lot of tests
-  // Is unable to print the first Scenario name for some reason
-  console.log('');
-  process.stdout.write(`Scenario: ${ scenario.pickle.name }: `);
+  // Start the running "progress bar" for the Scenario
+  log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `);
 
   // Create browser, which can't be created in BeforeAll() where .driver doesn't exist for some reason
   if (!scope.browser) {
@@ -127,13 +134,16 @@ Before(async (scenario) => {
     }
   }
   // Clean up all previously existing pages
-  for (const page of await scope.browser.pages()) { 
+  for (const page of await scope.browser.pages()) {
     await page.close();
   }
   // Make a new page
   scope.page = await scope.browser.newPage()
 
   scope.scenario_id = uuidv4();
+  scope.scenarios.set( scope.scenario_id, {} );
+  scope.scenarios.get( scope.scenario_id ).interviews = {};
+  scope.scenarios.get( scope.scenario_id ).api_keys = [];
   scope.expected_status = null;
   scope.expected_in_report = null;
   scope.expected_not_in_report = null;
@@ -179,7 +189,7 @@ Given(/I start the interview at "([^"]+)"/, {timeout: -1}, async (file_name) =>
 
   // Load the right interview page
   await scope.load( scope, file_name );
-  
+
   // The page timeout is set in here too in case someone set a custom timeout
   // just before a page was loaded for the Scenario. Move this to just after page creation?
   // The function itself won't really timeout here, so we don't need to wrap it in
@@ -274,21 +284,27 @@ Given(/I go to "([^"]+)"/i, {timeout: -1}, async ( url ) => {
   return result;
 });
 
-Given(/I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^"]+)" and(?: the password)? "([^"]+)"(?: SECRETs)?/i, {timeout: -1}, async ( email, password ) => {
+Given(
+  /I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^"]+)",?(?: and)?(?: the password)? "([^"]+)"(?: SECRETs)?(?:,?(?: and)?(?: the API key)? "([^"]+)")?/i,
+  { timeout: -1 },
+  async ( email, password, api_key ) => {
   /** Uses the names of environment variables (most often GitHub SECRETs) to
-  *     log into an account on the user's server. Must be SECRETs to stay secure.
+  *     log into an account on the user's server. Must be secure.
   * 
   * Variations:
-  * I sign into the server with the email "USER1_EMAIL" and the password "USER1_PASSWORD" SECRETs
-  * I sign on with the email "USER1_EMAIL" and the password "USER1_PASSWORD" SECRETs
-  * I log in with "USER1_EMAIL" and "USER1_PASSWORD"
+  * I sign into the server with the email "USER2_EMAIL" and the password "USER2_PASSWORD"
+  * I sign on with the email "USER2_EMAIL" and the password "USER2_PASSWORD"
+  * I log in with "USER2_EMAIL" and "USER2_PASSWORD"
+  * I sign in with the email "USER2_EMAIL", the password "USER2_PASSWORD", and the API key "USER2_API_KEY"
+  * I sign in with "USER2_EMAIL", "USER2_PASSWORD", "USER2_API_KEY"
   */
   // `page` timeout will deal with custom timeout
   // Couldn't test the timeout for tapping the button because there's not
   // enough of a pause between initial navigation and pressing button.
   await scope.steps.sign_in( scope, {
     email_secret_name: email,
-    password_secret_name: password
+    password_secret_name: password,
+    api_key_secret_name: api_key
   });
 });
 
@@ -313,7 +329,7 @@ Given(/the max(?:imum)? sec(?:ond)?s for each step (?:in this scenario )?is (\d+
   /* At any time, set a custom timeout for the scenario. */
   // TODO: Add 1 second to this to allow other steps to not fail, like "I wait n seconds"
   scope.timeout = parseInt( seconds ) * 1000;
-  
+
   // Cucumber timeouts are made custom for each step that needs it
 
   // If there is no page yet, we've ensured custom timeout will still be set
@@ -383,11 +399,9 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
             from_story_table: true,
           });
 
-          log.debug({
-            type: `info`,
-            code: `ALK0061`,
-            pre: `Rows, including special rows:, ${JSON.stringify( ensured_var_data )}`,
-          });
+          log.debug({ code: `ALK0061`, context: `table`, },
+            `Rows, including special rows:`, ensured_var_data
+          );
 
           // THIS FUNCTION ACTUALLY MATTERS
           // Some errors happen in there to be closer to the relevant code. We'd have to try/catch anyway.
@@ -637,7 +651,7 @@ Then('I arrive at the next page', async () => {  // √
 // Link observations have the 'Escape' button link in mind
 Then('I should see the link {string}', async ( linkText ) => {  // √
   /* Loosely match link visible text. Should probably deprecate this. */
-  let [link] = await scope.page.$x(`//a[contains(text(), "${linkText}")]`);
+  let [link] = await scope.page.$$(`xpath/.//a[contains(text(), "${linkText}")]`);
 
   let msg = `Cannot find a link with the text "${ linkText }".`;
   if ( !link ) { reports.addToReport(scope, { type: `error`, code: `ALK0079`, value: msg }); }
@@ -653,16 +667,16 @@ Then('I should see the link to {string}', async ( url ) => {  // √
   let msg = `Cannot find a link to "${ url }".`;
   if ( !link ) { reports.addToReport(scope, { type: `error`, code: `ALK0080`, value: msg }); }
   expect(link, msg ).to.exist;
-  
+
   await scope.afterStep(scope);
 });
 
 Then(/the "([^"]+)" link leads to "([^"]+)"/, async (linkText, expected_url) => {  // √
   /* Find link by its visible text and where it leads. */
-  let [link] = await scope.page.$x(`//a[contains(text(), "${linkText}")]`);
+  let [link] = await scope.page.$$(`xpath/.//a[contains(text(), "${linkText}")]`);
   // TODO: Possibly add a check here for the link text, though note this
   // Step is not currently supported.
-  
+
   let prop_obj = await link.getProperty('href');
   let actual_url = await prop_obj.jsonValue();
 
@@ -674,7 +688,7 @@ Then(/the "([^"]+)" link leads to "([^"]+)"/, async (linkText, expected_url) =>
 });
 
 Then(/the "([^"]+)" link opens in (a new window|the same window)/, async (linkText, which_window) => {  // √
-  let [link] = await scope.page.$x(`//a[contains(text(), "${linkText}")]`);
+  let [link] = await scope.page.$$(`xpath/.//a[contains(text(), "${linkText}")]`);
 
   let prop_obj = await link.getProperty('target');
   let target = await prop_obj.jsonValue();
@@ -756,7 +770,7 @@ Then(/I expect the baseline PDF "([^"]+)" and the new PDF "([^"]+)" to be the sa
   * Variations:
   * Then I expect the baseline PDF "simple-doc-Baseline.pdf" and the new PDF "simple-doc.pdf" to be the same
   */
-  
+
   return await wrapPromiseWithTimeout(
     scope.steps.compare_pdfs(scope, {existing_pdf_path, new_pdf_path}),
     scope.timeout
@@ -880,7 +894,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
 
       // Set up timeout to stop infinite loops
       let page_timeout_id = setTimeout(async function() {
-        
+
         // Make sure to only throw once
         if ( has_thrown ) { return; }
         else { has_thrown = true; }
@@ -959,7 +973,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
           }  // ends if !has_content
 
           if ( buttons_exist ) {
-            process.stdout.write(`\x1b[36m${ 'v' }\x1b[0m`);  // pressed button
+            log.stdout({}, `\x1b[36m${ 'v' }\x1b[0m`);  // pressed button
           }
 
           resolve();
@@ -1178,6 +1192,71 @@ Given(/the final Scenario status should be "([^"]+)"/i, async ( expected_status
   scope.expected_status = expected_status.toUpperCase();
 });
 
+Then(/of two interviews, I delete one __internal__/i, async () => {
+  /** Test that an interview does get successfully deleted. This tests whether
+   *    the API code is working correctly.
+   */
+  // Make sure we have two interviews in `scope.scenario.interviews` (two session ids)
+  let interviews = scope.scenarios.get( scope.scenario_id ).interviews;
+  let interview_keys = Object.keys( interviews );
+  expect(
+    interview_keys.length,
+    `Of 2 interview sessions, ALKiln only saved ${ interview_keys.length }`
+  ).to.equal( 2 );
+
+  // Check that both interviews exist on the server. 1 √ 2 √
+  let api_keys = scope.scenarios.get( scope.scenario_id ).api_keys;
+  let visible_interviews = await da_api.get_interviews({ interviews, api_keys });
+  let visible_keys = Object.keys( visible_interviews );
+  expect(
+    visible_keys.length,
+    `2 interviews should be on the server. Instead: "${ visible_keys.join(', ') }"`
+  ).to.equal( 2 );
+
+  // Delete one. 1 x 2 √
+  let to_delete = interviews[ interview_keys[ 0 ]];
+  await da_api.quietly_try_to_delete_interviews({
+    interviews: { [interview_keys[ 0 ]]: to_delete },
+    api_keys
+  });
+
+  // Check that only one session exists on the server. This tests that
+  // 1. An interview has been deleted and
+  // 2. We are fetching interviews correctly because we are getting one
+  // back (e.g. not passing null into the api function)
+  // 1 x 2 √
+  let after_deletion = await da_api.get_interviews({ interviews, api_keys });
+  let keys_after_deletion = Object.keys( after_deletion );
+  expect(
+    keys_after_deletion.length,
+    `1 interviews should be on the server: ${ keys_after_deletion.join(', ') }`
+  ).to.equal( 1 )
+  // In After(), the other session should get deleted
+});
+
+Then(/I fail to delete (\d) detected interview(?:s)? and get no error __internal__/i, async ( num ) => {
+  /** Test that there is no error when ALKiln is unable to delete an interview. */
+  num = parseInt( num );
+  // Make sure we started an interview
+  let interviews = scope.scenarios.get( scope.scenario_id ).interviews;
+  expect(
+    Object.keys( interviews ).length,
+    `The test should have saved ${ num } interview session(s)`
+  ).to.equal( num );
+
+  // There should be no error about an unauthorized user
+  let api_keys = scope.scenarios.get( scope.scenario_id ).api_keys;
+  let visible_interviews = await da_api.get_interviews({ interviews, api_keys });
+  let visible_keys = Object.keys( visible_interviews );
+  expect(
+    visible_keys.length,
+    `User should detect 0 interviews on the server. Instead: "${ visible_keys.join(', ') }"`
+  ).to.equal( 0 );
+
+  // Deleting with an anonymous user should fail silently
+  await da_api.quietly_try_to_delete_interviews({ interviews, api_keys });
+});
+
 
 //#####################################
 //#####################################
@@ -1185,11 +1264,22 @@ Given(/the final Scenario status should be "([^"]+)"/i, async ( expected_status
 //#####################################
 //#####################################
 
+AfterStep(async function({ result }) {
+  // TODO: Abstract this
+  if ( result.status === `PASSED` ) { log.stdout({ records_only: true }, `.`); }
+  else if ( result.status === `FAILED` ) { log.stdout({ records_only: true }, `F`); }
+  else if ( result.status === `UNDEFINED` ) { log.stdout({ records_only: true }, `U`); }
+  else if ( result.status === `SKIPPED` ) { log.stdout({ records_only: true }, `-`); }
+  else if ( result.status === `PENDING` ) { log.stdout({ records_only: true }, `P`); }
+});
+
 After(async function(scenario) {
 
   // Log errors
   if ( scenario.result.message ) {
-    log.debug({ type: `info`, code: `ALK0091`, pre: `cucumber error: ${ scenario.result.message }` }); //, verbose: true });
+    log.debug({ code: `ALK0091`, context: `scenario`, },
+      scenario.result.message
+    );
   }
 
   let changeable_test_status = scenario.result.status;
@@ -1218,11 +1308,9 @@ After(async function(scenario) {
   reports.saveFinalScenarioData( scope, { status: changeable_test_status });
 
   if (changeable_test_status !== `PASSED`) {
-    log.debug({
-      type: `error`,
-      code: `ALK0094`,
-      pre: `Non-passed status occurred while running test: ${changeable_test_status}`,
-    });
+    log.debug({ code: `ALK0094`, level: `warn`, },
+      `Non-passed status occurred while running test: ${ changeable_test_status }`
+    );
 
     if ( scope.page ) {
 
@@ -1314,12 +1402,15 @@ After(async function(scenario) {
       // it as failed too.
       expected_status_was_met = false;
       // Keep it general. Users may see this message, e.g. for a11y tests.
-      let msg = `Test failed. Expected status: ${scope.expected_status}. `
-          + `ALKiln's assessed status: ${ changeable_test_status }. `
+      let msg = `Test ${ changeable_test_status }. `
+          + `Expected status: ${scope.expected_status}. `
           + `See the report for details.`;
       let code = `ALK0099`;
-      log.debug({ type: `error`, code: code, pre: msg, })
-      expected_status_err_msg = `🤕 ${ code } ERROR: ${ msg }`
+      expected_status_err_msg = log.debug({ code: code, level: `error`, },
+        msg
+      );
+      // Temp for debugging new log output
+      expect( expected_status_err_msg ).to.equal(`🤕 ${ code } ERROR: ${ msg }`);
     }
   }
   scope.expected_status = null;  // reset for next Scenario
@@ -1339,17 +1430,25 @@ After(async function(scenario) {
     // Make sure that tests which ALKiln has calculated as failed
     // cause a cucumber error later. Just changing the status to `FAILED`
     // causes unhelpful cucumber errors. TODO: Add to internal docs.
-    log.debug({
-      type: `error`,
-      code: force_failure_code,
-      pre: force_failure_msg
-    });
+    // TODO: Should we move this to where the error is thrown?
+    log.debug({ code: force_failure_code, level: `error`, },
+      force_failure_msg
+    );
     // TODO: Look into setDefinitionFunctionWrapper to catch errors
     // of failing tests that should indeed be failing:
     // https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
     need_to_force_failed_status = true;
   }
 
+  // Not sure if this is necessary
+  if ( changeable_test_status === 'PASSED' ) {
+    log.stdout({ records_only: true }, `.` );
+  } else if ( changeable_test_status === `SKIPPED` ) {
+    log.stdout({ records_only: true }, `-` );
+  } else {
+    log.stdout({ records_only: true }, changeable_test_status[0].toUpperCase() );
+  }
+
   let signout_succeeded = true;
   let signout_error_msg = ``;
   // If there is a page open, then sign out and close it
@@ -1372,7 +1471,7 @@ After(async function(scenario) {
             code: code,
             message: non_reload_report_msg,
           }, error });
-        } catch {
+        } catch ( error ) {
           signout_error_msg = `🤕 ${ code } ERROR: ${ non_reload_report_msg }`;
         }
       } else {
@@ -1387,21 +1486,66 @@ After(async function(scenario) {
 
   }  // ends if scope.page
 
-  let errors = []
-  if ( !expected_status_was_met ) { errors.push( expected_status_err_msg ); }
-  if ( !signout_succeeded ) { errors.push( signout_error_msg ); }
-  if ( !expected_status_was_met || !signout_succeeded ) {
-    throw new Error( `${errors.join( errors, '\n')}` );
+  // Now that we're no longer on the interview page, one way or
+  // another, try to delete the interviews created during the test
+  // Authors running this manually get to choose what to do with
+  // the interviews they're creating unless we hear otherwise.
+  if ( session_vars.get_origin() !== `playground` ) {
+    try {
+      let interviews = scope.scenarios.get( scope.scenario_id ).interviews;
+      let api_keys = scope.scenarios.get( scope.scenario_id ).api_keys;
+      await da_api.quietly_try_to_delete_interviews({ interviews, api_keys });
+      let visible_interviews = await da_api.get_interviews({ interviews, api_keys });
+      let visible_keys = Object.keys( visible_interviews );
+      // Deleting interviews is nice, but not non-critical. Don't bother the
+      // user with it. Don't want to cover up a more meaningful failure (like
+      // going to the wrong file). If it becomes a problem we can track it down later.
+      if ( visible_keys.length > 0 ) {
+        // Can an API key ever see an interview that it can't delete?
+        log.debug({ code: `ALK0197`, context: `internal`, level: `warn`, }
+          `The API key can see ${ visible_keys.length } interviews that were not deleted:`,
+          visible_keys
+        );
+      }
+    } catch ( error ) {
+      // Deleting interviews is nice, but not non-critical. Don't bother the
+      // user with it. Don't want to cover up a more meaningful failure (like
+      // going to the wrong file). Example of OK failure: the user owns the
+      // interview is different than the API key that's trying to delete the
+      // interview. If it becomes a problem we can track it down later.
+      // Consider add "internal" to docs: "If you see this, make an
+      // issue in ALKiln with your debug/verbose log"
+      log.debug({ code: `ALK0198`, context: `internal`, level: `warn`, error },
+        `Cleaning up interviews incomplete, but unimportant.`
+      );
+    }
   }
 
+  // Collect all the relevant special errors
+  let internal_errors = []
+  if ( !expected_status_was_met ) { internal_errors.push( expected_status_err_msg ); }
+  if ( !signout_succeeded ) { internal_errors.push( signout_error_msg ); }
+
   // Theoretically, we should never get here:
   // If everything else worked, if we need to force an internal test failure
   if ( need_to_force_failed_status ) {
-    throw new Error( `🤕 ${force_failure_code} unreachable ERROR: ${force_failure_msg}` );
+    if ( internal_errors.length > 0 ) {
+      // Handle all the accumulated errors
+      throw new Error( `${internal_errors.join( internal_errors, '\n')}` );
+    }
+    throw new Error( `🤕 ${ force_failure_code } unreachable ERROR: ${ force_failure_msg }` );
+
+  } else if ( internal_errors.length > 0 ) {
+    log.debug({ code: `ALK0203`, context: `internal`, level: `warn`, },
+      `There was no need to force failed status and yet there are error messages.`,
+      internal_errors
+    );
   }
 
   // Find the race condition.
-  log.debug({ type: `cucumber`, code: `ALK0100`, verbose: true, pre: `Scenario AfterStep() message`, data: scenario.result.message });
+  log.debug({ code: `ALK0100`, context: `scenario`, },
+    `Scenario After() message:`, scenario.result.message
+  );
 
 });
 

package/lib/utils/files.js

@@ -22,18 +22,20 @@ files.readable_date = function() {
 };  // Ends files.readable_date()
 
 files.make_artifacts_folder = function (folder_name) {
-  /** Makes a folder in the root of the project, either with the given name
-  *   or with a default name.
-  * 
-  * Works for local development as well, unlike action.yml or the workflow file.
-  */
+  /** Makes a folder in the root of the project if it doesn't
+   *  already exist, either with the given name or with a default name.
+   *
+   * Works for local development as well, without action.yml or workflow files.
+   * */
   if (folder_name) {
     folder_name = safe_filename( folder_name );
   } else {
     folder_name = `alkiln-${ files.readable_date() }`
   }
 
-  fs.mkdirSync( folder_name );
+  // `recursive` avoids errors if folder already exists
+  // https://nodejs.org/docs/latest-v18.x/api/fs.html#fspromisesmkdirpath-options
+  fs.mkdirSync( folder_name, { recursive: true });
   return folder_name;
 };  // Ends files.make_artifacts_folder()