@suffolklitlab/alkiln 5.3.0 → 5.4.1-feat-github-server-15

package/lib/scope.js

@@ -9,6 +9,7 @@ const safe_filename = require("sanitize-filename");
 const faker = require('@faker-js/faker/locale/en').faker;
 
 // Ours
+const pdf_reader = require('./utils/pdf_reader');
 const session_vars = require('./utils/session_vars');
 const get_base_interview_url = require(`./docassemble/get_base_interview_url`);
 const time = require('./utils/time');
@@ -45,6 +46,9 @@ let printable_var_value = function(row) {
   /* `row` might be a row, a source row, or an original row. */
   let original = get_original_row(row);
   if ( is_secret_var(original) ) {
+    if ( session_vars.get_dangerous_debug() ) {
+      return `alkiln_dangerous ${ row.value }`;
+    }
     return original.flags.secret_name;
   }
   return row.value;
@@ -1679,90 +1683,77 @@ module.exports = {
     return valid_date;
   },  // Ends scope.today_to_date()
 
-  uploadFiles: async function ( scope, { handle, answer }) {
-    /* Try to upload one or more files to a file-type input field.
-    *    Does not submit, so no danger of timeout with server reload.
+  findFiles: async function( scope, {to_find_names}) {
+    /** Returns a list of paths to the sources folder associated
+    *    with the given list of file names.
     * 
-    * Returns whether the row was used or not.
+    * @param to_find_names { [str] } List of file names
+    * 
+    * @returns { [str] } sources folder paths
     */
-
     // First make sure the list of files was defined. Don't remove spaces yet -
     // allow a typo where someone forgot to include a space after a comma.
-    let uploads_names = answer.split(`,`);
-    if ( uploads_names.length === 0 ) {
-      reports.addToReport( scope, {
-        type: `error`,
-        value: `Your list of files to upload is empty.`
-      });
-      return false;  // The row wasn't used
-    }
-
-    let dirs = [];
-    if ( session_vars.get_origin() === `playground` ) {
-      let files_playground = fg.sync(
-        [ `/usr/share/docassemble/files/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-      let files_s3 = fg.sync(
-        [ `/tmp/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-
-      dirs = files_playground.concat( files_s3 );
-    } else {
-      dirs = fg.sync(
-        [ `**/docassemble/**/data/sources` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-    }
-
-    if ( session_vars.get_debug() ) {
-      console.log( `All discovered dirs to upload from:`, dirs );
-    }
-
-    // Make sure sources directory exists and has files in it
-    if ( dirs.length === 0 ) {
+    // TODO: wish we had the var name here that folks are trying to set
+    if ( to_find_names.length === 0 ) {
       reports.addToReport( scope, {
         type: `warning`,
-        value: `The test files need to be in the "sources" directory, `
-          + `but ALKiln couldn't find the directory or it was empty.`
+        value: `Your Step is missing the names of which file(s) to use.`
       });
-      return false;  // The row wasn't used
+      return [];
     }
 
-    // Get the full paths of all the files
-    let dir = null;
+    let sources_dirs = session_vars.get_sources_paths();
+    // Get the full paths of all the existing files
     let paths = [];
     let missing_files = [];  // For any user warnings
-    for ( let name of uploads_names ) {
-     
+    for ( let name of to_find_names ) {
+
       let trimmed_name = name.trim();
       let matching_paths = [];
-
-      for ( let one_dir of dirs ) {
-        // accumulate all matching paths
-        let new_paths = fg.sync([ `${ one_dir }/${ trimmed_name }` ]);
+      // Look for the file in all "sources" folders
+      for ( let one_dir of sources_dirs ) {
+        let new_paths = fg.sync(
+          [ `${ one_dir }/${ trimmed_name }` ],
+          { suppressErrors: true }
+        );
         matching_paths.push( ...new_paths );
-      }  // ends for each dir
+      }
 
       if ( matching_paths.length === 0 ) {
         missing_files.push( trimmed_name );
       } else {
         paths.push( ...matching_paths );
       }
-
     }  // ends for each file
 
     if ( missing_files.length > 0 ) {
       reports.addToReport( scope, {
         type: `warning`,
-        value: `ALKiln could not find "${ missing_files.join('", "') }" in any of these folders: "${ dirs.join('", "') }".`
+        value: `ALKiln could not find "${ missing_files.join('", "') }" in the folder(s) at "${ sources_dirs.join('", "') }".`
       });
     }
 
-    // If all files missing, return that the row wasn't used
-    if ( missing_files.length === uploads_names.length ) {
-      return false;  // The row wasn't used
+    // If all files missing, return an empty array
+    if ( missing_files.length === to_find_names.length ) {
+      return [];
+    }
+    return paths;
+  },  // Ends scope.findFiles()
+
+  uploadFiles: async function ( scope, { handle, answer }) {
+    /* Try to upload one or more files to a file-type input field.
+    *    Does not submit, so no danger of timeout with server reload.
+    * 
+    * Returns whether the row was used or not.
+    */
+
+    // First make sure the list of files was defined. Don't remove spaces yet -
+    // allow a typo where someone forgot to include a space after a comma.
+    let to_find_names = answer.split(`,`);
+    let paths = await scope.findFiles(scope, {to_find_names});
+    if (paths.length == 0) {
+      // The row wasn't used
+      return false;
     }
 
     // Prepare to wait for upload to complete
@@ -1988,7 +1979,7 @@ module.exports = {
       }
       if ( num_unused_rows > 0 ) {
         let { id } = await scope.examinePageID( scope, 'none to match' );
-        let msg = `Missing variable or variables on page "${id}"`;
+        let msg = `Error on page "${id}". See screenshot and see text above for more details.`;
         reports.addToReport(scope, { type: `error`, value: msg });
         expect( num_unused_rows, msg ).to.equal( 0 );
       }
@@ -2449,6 +2440,10 @@ module.exports = {
       /** Sets a non-story table variable to a "secret" value (i.e. an env var) */
       // Prevent pictures of a screen with a secret.
       scope.disable_error_screenshot = true;
+      if ( session_vars.get_dangerous_debug() ) {
+        scope.disable_error_screenshot = false;
+      }
+
       let var_data = await scope.normalizeTable( scope, {
               var_data: [{
                 var: var_name,
@@ -2503,6 +2498,32 @@ module.exports = {
       await scope.detectDownloadComplete( scope );
     },  // Ends scope.steps.download()
 
+    compare_pdfs: async function (scope, {existing_pdf_path, new_pdf_path}) {
+      let existing_paths = await scope.findFiles(scope, {to_find_names: [existing_pdf_path]});
+      if (existing_paths.length == 0) {
+        // The row wasn't used
+        let msg = `Could not find the existing PDF at ${ existing_pdf_path }`;
+        reports.addToReport(scope, { type: `error`, value: msg });
+        scope.failed_pdf_compares.push(msg);
+        // Return early since we couldn't find the PDF
+        return;
+      }
+      let full_existing_path = existing_paths[0];
+      let full_new_pdf_path = `${scope.paths.scenario}/${new_pdf_path}`;
+      let diffs = await pdf_reader.compare_pdfs(full_existing_path, full_new_pdf_path);
+      if (diffs) {
+        let added_str = diffs.filter(part => part.added).reduce((err_str, part) => {
+          return err_str + `- ${ part.value }\n`
+        }, 'The new PDF added: \n');
+        let removed_str = diffs.filter(part => part.removed).reduce((err_str, part) => {
+          return err_str + `- ${ part.value }\n`
+        }, 'The new PDF removed: \n');
+        let msg = `The PDFs were not the same.\n${ added_str }\n${removed_str}\n\n You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`;
+        reports.addToReport(scope, { type: `error`, value: msg });
+        scope.failed_pdf_compares.push(msg);
+      }
+    },
+
     set_al_name: async ( scope, var_base, name_str ) => {
       /** Given a variable name and a space-separated string, fills the
       *    appropriate fields on an Assembly Line name question.
@@ -2592,7 +2613,10 @@ module.exports = {
       */
       // Don't take a picture of a failed login in case one of the inputs is correct
       scope.disable_error_screenshot = true;
-
+      if ( session_vars.get_dangerous_debug() ) {
+        scope.disable_error_screenshot = false;
+      }
+      
       // If there is no browser open, start a new one
       if (!scope.browser) {
         scope.browser = await scope.driver.launch({ headless: !session_vars.DEBUG });
@@ -2625,6 +2649,10 @@ module.exports = {
       let email = process.env[ email_secret_name ];
       let password = process.env[ password_secret_name ];
 
+      if ( session_vars.get_dangerous_debug() ) {
+        log.debug({ type: `unsafe`, data: `login: ${ email }, ${ password }` });
+      }
+
       // Give as many useful messages to the user as possible if the env vars
       // aren't defined. Eventually, just provide a link to the docs because
       // this is complex
@@ -2930,4 +2958,5 @@ module.exports = {
 
     return none_included;
   },
+
 };

package/lib/steps.js

@@ -147,6 +147,7 @@ Before(async (scenario) => {
   scope.page_id = null;
   scope.check_all_for_a11y = false;
   scope.passed_all_for_a11y = true;
+  scope.failed_pdf_compares = [];
 
   // Reset default timeout
   scope.timeout = default_timeout;
@@ -705,6 +706,19 @@ Then(/I get the(?: page's)?(?: JSON)? var(?:iable)?s and val(?:ue)?s/i, { timeou
   );
 });
 
+Then(/I expect the baseline PDF "([^"]+)" and the new PDF "([^"]+)" to be the same/i, async (existing_pdf_path, new_pdf_path) => {
+  /** Compare the text or fields of a baseline PDF and the newly downloaded PDF.
+  * 
+  * Variations:
+  * Then I expect the baseline PDF "simple-doc-Baseline.pdf" and the new PDF "simple-doc.pdf" to be the same
+  */
+  
+  return await wrapPromiseWithTimeout(
+    scope.steps.compare_pdfs(scope, {existing_pdf_path, new_pdf_path}),
+    scope.timeout
+  );
+});
+
 //#####################################
 //#####################################
 // Actions
@@ -1104,12 +1118,21 @@ After(async function(scenario) {
   if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
     // The fact that the scenario failed has already been noted earlier in the report
     changeable_test_status = `FAILED`;
-    scope.addToReport(scope, {
+    reports.addToReport(scope, {
       type: `error`,
       value: `Accessibility standards weren't met. See information above.`
     });
   }
 
+  if ( scope.failed_pdf_compares.length > 0) {
+    let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n---\n${ new_msg }`)
+    changeable_test_status = `FAILED`;
+    reports.addToReport(scope, {
+      type: `error`,
+      value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\n---\n${ msg }\n---\n`
+    });
+  }
+
   // Record status before internal report tests can interfere with values
   reports.setReportScenarioStatus( scope, { status: changeable_test_status });
 

package/lib/utils/log.js

@@ -42,7 +42,7 @@ log.error = function ({ type='', pre='', data='', post='' }) {
 
 log.debug = function ({ type='', pre='', data='', post='' }) {
   /* If debugging is turned on, log a message that includes `ALKiln` and `debug` */
-  if ( session_vars.get_debug() ) {
+  if ( session_vars.get_debug() || session_vars.get_dangerous_debug() ) {
     let start = create_msg_start([ type, 'debug' ], pre );
     // Log them each on a separate line
     console.log( `ALKiln ${ type } debug: ${ pre }` );

package/lib/utils/pdf_reader.js

@@ -0,0 +1,113 @@
+// All functions dealing with getting information from PDFs
+
+const pdfjs_lib = require("pdfjs-dist/legacy/build/pdf.js");
+const diff_lib = require("diff");
+
+let _load_pdf = async function(pdf_path) {
+  // Simple wrapper to get the document
+  return await pdfjs_lib.getDocument({
+    url: pdf_path,
+    // https://github.com/mozilla/pdf.js/issues/4244#issuecomment-1479534301
+    useSystemFonts: true
+  }).promise;
+};
+
+let _get_content = async function(pdf_doc) {
+  let num_pages = pdf_doc.numPages;
+  let contents = "";
+  for (var i = 1; i <= num_pages; i++) {
+    const page_contents = await (await pdf_doc.getPage(i)).getTextContent();
+    contents += "\n" + page_contents.items.map((item) => item.str).join(" ");
+  }
+  return contents;
+};
+
+let pdf_reader = {};
+
+pdf_reader.get_content = async function(pdf_path) {
+  /* Gets the full text content from the PDF.
+   *
+   * @returns str
+   */
+
+  let doc = await _load_pdf(pdf_path);
+  return await _get_content(doc);
+}; // Ends get_content()
+
+pdf_reader.get_fields = async function(pdf_path) {
+  /* Gets all of the fields in the PDF
+   * 
+   * Example:
+   * {
+   *   "users1_name_first": [
+   *      {
+   *         "id": "6R",
+   *         "value": "aeu",
+   *         "defaultValue": "",
+   *         "multiline": false,
+   *         "password": false,
+   *         "charLimit": 0,
+   *         "comb": false,
+   *         "editable": true,
+   *         "hidden": false,
+   *         "name": "users1_name_first",
+   *         "rect": [
+   *             143.52,
+   *             660.72,
+   *             315.11,
+   *             674.88
+   *         ],
+   *         "actions": null,
+   *         "page": 0,
+   *         "strokeColor": null,
+   *         "fillColor": null,
+   *         "rotation": 0,
+   *         "type": "text"
+   *       }
+   *   ],
+   */
+  let doc = await _load_pdf(pdf_path);
+  const field_objects = await doc.getFieldObjects();
+  return field_objects;
+}; // Ends get_fields
+
+pdf_reader.get_field_value = async function(pdf_path, field_name) {
+  let field_objects = await pdf_reader.get_fields(pdf_path);
+  if (field_name in field_objects) {
+    for (let inst of field_objects[field_name]) {
+      if ("value" in inst) {
+        return inst["value"];
+      }
+    }
+  }
+  return null;
+}; // Ends get_field
+
+pdf_reader.compare_pdfs = async function(existing_pdf_path, new_pdf_path) {
+  /**
+   * Returns the change objects between the PDF fields, if there's any diff
+   */
+  let existing_doc = await _load_pdf(existing_pdf_path);
+  let new_doc = await _load_pdf(new_pdf_path);
+  let existing_content = await _get_content(existing_doc);
+  let new_content = await _get_content(new_doc);
+
+  // Assumption: authors won't be changing their PDF templates programmatically;
+  // so if the text of the PDFs are different, then it's actually the fields that are different,
+  // but they aren't editable / are flattened into the PDF. 
+  let change = diff_lib.diffWords(existing_content, new_content);
+  if (change.some(diff => diff.added || diff.removed)) {
+    return change;
+  }
+
+  const existing_fields = await existing_doc.getFieldObjects();
+  const new_fields = await new_doc.getFieldObjects();
+
+  let changeObjs = diff_lib.diffJson(existing_fields, new_fields);
+  if (changeObjs.some(diff => diff.added || diff.removed)) {
+    return changeObjs;
+  }
+  return null;
+}; // End compare_pdfs
+
+module.exports = pdf_reader;

package/lib/utils/session_vars.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const fg = require('fast-glob');
 const path = require('path'); 
 require('dotenv').config();
 
@@ -23,6 +24,11 @@ module.exports = session_vars;
 // These default to null
 // TODO: How to isolate debug elsewhere so this file can use the logger?
 session_vars.get_debug = function () { return process.env.DEBUG || null; };
+session_vars.get_dangerous_debug = function () {
+  /** Note this will cause failure of tests that reassure the user about
+  *   security precautions we took, like not taking a screenshot. */
+  return process.env._ALKILN_DANGEROUS_DEBUG || null;
+};
 session_vars.is_dev_env = function () { return process.env.DEV || null; }
 
 session_vars.get_dev_api_key = function () { return process.env.DOCASSEMBLE_DEVELOPER_API_KEY || null; };
@@ -37,13 +43,6 @@ session_vars.get_origin = function () {
 };
 session_vars.get_user_project_name = function () { return process.env._PROJECT_NAME || null; };
 session_vars.get_user_id = function () { return process.env._USER_ID || null; };
-session_vars.get_tags = function () {
-  if ( process.env._TAGS ) {
-    return process.env._TAGS.split(` `);
-  } else {
-    return [];
-  }
-};
 
 // More complex logic
 session_vars.get_server_reload_timeout = function () {
@@ -210,6 +209,124 @@ session_vars.get_artifacts_path_name = function () {
   }
 };  // Ends get_artifacts_path_name()
 
+
+const install_type_key = `da_install_type`;
+session_vars.save_install_type = function (install_type) {
+  /* Save a key with the name of the current artifacts folder in case it doesn't exist. */
+
+  // Ensure artifacts path file exists
+  if ( !fs.existsSync( runtime_config_path ) ) {
+    // If the file doesn't exist, make one because next we need to read it
+    fs.writeFileSync( runtime_config_path, JSON.stringify( {}, null, 2 ) );
+  }
+
+  // Get the json, changing it, and re-write the file from scratch
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ) );
+  json[ install_type_key ] = install_type;
+  let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
+
+  return file;
+};  // Ends save_install_type()
+
+session_vars.get_install_type = function () {
+  /* Get the name of the current artifacts path. */
+  try {
+    let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+    let install_type = json[ install_type_key ];
+
+    return install_type;
+  } catch ( error ) {
+    return null;
+  }
+};  // Ends get_install_type()
+
+
+const repo_folder_key = `da_repo_folder_name`;
+session_vars.save_repo_folder_name = function (repo_folder_name) {
+  /** Save a key with the name of the main package folder in the repository. */
+
+  // Ensure runtime config file exists
+  if ( !fs.existsSync( runtime_config_path ) ) {
+    // If the file doesn't exist, make one because next we need to read it
+    fs.writeFileSync( runtime_config_path, JSON.stringify( {}, null, 2 ) );
+  }
+
+  // Get the json, changing it, and re-write the file from scratch
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ) );
+  json[ da_repo_folder_name ] = repo_folder_name;
+  let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
+
+  return file;
+};  // Ends session_vars.save_repo_folder_name()
+
+session_vars.get_repo_folder_name = function () {
+  /** Get the name of the main folder in the repository. */
+  try {
+    let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+    let name = json[ da_repo_folder_name ];
+
+    return name;
+  } catch ( error ) {
+    return null;
+  }
+};  // Ends session_vars.get_repo_folder_name()
+
+
+session_vars.set_sources_paths = function ( paths ) {
+  /** Returns whether at least one of the given path exists. Store
+  *   a list of the existing paths. Warn of any missing paths. Can be
+  *   used for getting, e.g .feature files, files to upload, etc.
+  * 
+  * Our code should ensure this gets run before cucumber tests get triggered.
+  * 
+  * @param paths { [str] } List of paths to the 'sources' folder.
+  * 
+  * @returns bool
+  */
+
+  // Check every path, tracking which exist and which are missing
+  let existing_paths = [];
+  let missing_paths = [];
+  for ( let path of paths ) {
+    let dirs = fg.sync(
+      [ path ],
+      { onlyFiles: false, suppressErrors: true }
+    );
+
+    if ( dirs.length > 0 ) {
+      existing_paths.push( ...dirs );
+    } else {
+      missing_paths.push( path );
+    }
+  }
+
+  // Warn about missing paths
+  if ( missing_paths.length > 0 ) {
+    console.warn(
+      `ALKiln WARNING: ALKiln can't find folders at "${ paths.join(", ") }".`
+    );
+  }
+
+  // If no paths were valid
+  if ( existing_paths.length <= 0 ) {
+    session_vars._sources_paths = null;
+    return false;
+  }
+
+  // Otherwise, store valid paths
+  session_vars._sources_paths = existing_paths;
+  return true;
+};
+
+session_vars.get_sources_paths = function () {
+  /** Get the "sources" folder path. Should never be undefined during
+  *   a run of the test. run_cucumber.js should make sure it has
+  *   a default value.
+  * 
+  * @returns str || null */
+  return session_vars._sources_paths;
+};
+
 session_vars.validateEnvironment = function() {
   /** Throw a useful error for the developer if any of the env vars
   *    required by absolutely all tests are missing. Gather all
@@ -274,3 +391,36 @@ session_vars.validateEnvironment = function() {
 };
 
 session_vars.validateEnvironment();
+
+
+const save_runtime_config_var = function (key, value) {
+  /** Save a key and its value in the runtime config file. */
+
+  // Ensure runtime config file exists
+  if ( !fs.existsSync( runtime_config_path ) ) {
+    // If the file doesn't exist, make one because next we need to read it
+    fs.writeFileSync( runtime_config_path, JSON.stringify( {}, null, 2 ) );
+  }
+
+  // Get the json, changing it, and re-write the file from scratch
+  let json = JSON.parse( fs.readFileSync( runtime_config_path ) );
+  json[ key ] = value;
+  let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
+
+  if ( session_vars.get_debug() ) {
+    console.log( `ALKiln debug: Stored "${key}" locally as "${value}".` );
+  }
+  return file;
+};  // Ends save_runtime_config_var()
+
+const get_runtime_config_var = function (key) {
+  /** Get the value of the given key from the runtime config file. */
+  try {
+    let json = JSON.parse( fs.readFileSync( runtime_config_path ));
+    let value = json[ key ];
+
+    return value;
+  } catch ( error ) {
+    return null;
+  }
+};  // Ends get_runtime_config_var()