@suffolklitlab/alkiln 5.3.0 → 5.4.1

package/CHANGELOG.md

@@ -44,6 +44,19 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [5.4.1] - 2023-10-18
+
+### Internal
+- BREAKS ALKilnInThePlayground v1.2.0 and below (as we discussed) as that project is still exploratory. Can now take the location of the "sources" folder or folders as an argument in the command line. The formats are `npm run cucumber @tags -- --sources=./foo` and `alkiln-run @tags --sources=./foo`.
+
+## [5.4.0] - 2023-10-14
+
+### Added
+
+- New step to compare PDFs!
+    - An example: `I expect the baseline PDF "something-in-sources.pdf" and the new PDF "downloaded.pdf" to be the same`
+    - See [#774](https://github.com/SuffolkLITLab/ALKiln/pull/774) for more details.
+
 ## [5.3.0] - 2023-10-14
 
 ### Added

package/lib/run_cucumber.js

@@ -6,64 +6,97 @@ const session_vars = require('./utils/session_vars');
 
 const cucumber_api = require('@cucumber/cucumber/api');
 
-(async function main() {
-  let argv = process.argv;
-
-  const environment = { cwd: process.cwd() };
-  console.log("alkiln-run: environment: %o", environment);
-  const provided_config = {
-    "publishQuiet": true,
-    "require": "./lib/index.js",
-    "format": ["progress", "summary:cucumber-report.txt"],
-    "paths": ["docassemble/*/data/sources/*.feature"],
-    "retry": 1,
-  }
-
-
-  // If they're running it from an interview on their server instead of from GitHub
-  if ( session_vars.get_origin() === `playground` ) {
-    provided_config.paths.push(`/usr/share/docassemble/files/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}/*.feature`);
-    // For S3
-    provided_config.paths.push(`/tmp/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}/*.feature`);
+function build_config( command_args ) {
+  // Turn all the paths to the sources folders into paths to the feature files
+  let sources_paths = session_vars.get_sources_paths();
+  let feature_paths = [];
+  for ( let source of sources_paths ) {
+    feature_paths.push( `${ source }/*.feature` );
   }
 
-  let tags = argv.slice(2);
-  if ( session_vars.get_origin() === `playground` ) {
-    tags = session_vars.get_tags();
+  const config = {
+    publishQuiet: true,
+    require: `./lib/index.js`,
+    format: [`progress`, `summary:cucumber-report.txt`],
+    paths: feature_paths,
+    retry: 1,
   }
 
+  // Normalize tag expressions, if there are any
+  let tags = command_args[`_`];
   if (tags.length > 0) {
     let joined_tags = '';
     // Cucumber expects tags in boolean expressions, like (@a0 or @a1) and @random.
     // If we get a list of only tags, then "or" them all together
-    if (tags.every((val) => val.startsWith("@"))) {
-      joined_tags = tags.join(' or ');
+    if (tags.every((val) => val.startsWith(`@`))) {
+      joined_tags = tags.join(` or `);
     }
     // If we get a list of seemingly no tags, assume they meant to use tags
-    else if (tags.every((val) => !val.startsWith("@"))) {
-      joined_tags = tags.map((val) => "@" + val).join(' or ');
+    else if (tags.every((val) => !val.startsWith(`@`))) {
+      joined_tags = tags.map((val) => `@` + val).join(` or `);
     }
     // If it's a mix of tags and no tags, it's probably a boolean expression!
     else {
-      joined_tags = tags.join(' ');
+      joined_tags = tags.join(` `);
     }
 
-    provided_config.tags = joined_tags;
-  }  // end if there are tags
+    config.tags = joined_tags;
+  }
+
+  return config;
+};  // Ends build_config()
+
+
+(async function main() {
+  /** Create the cucumber config, run the tests, create and
+  *   manipluate the output. */
+  const environment = { cwd: process.cwd() };
+  console.log("alkiln-run: environment: %o", environment);
+
+  // We'll use the user info to construct the possible paths to /sources
+  let id = session_vars.get_user_id();
+  // `project` is the name in the server docassemble "Project" page
+  let project = session_vars.get_user_project_name();
+
+  // process.argv is a list of strings of commands and args.
+  // If using `npm run cucumber`, use `--` before `--sources=./foo`
+  let argv = require(`minimist`)( process.argv.slice(2), {
+    default: {
+      sources: [
+        `./docassemble/*/data/sources`,  // In GitHub folder
+        `/usr/share/docassemble/files/playgroundsources/${id}/${project}/*.feature`, // From playground without S3
+        `/tmp/playgroundsources/${id}/${project}/*.feature` // From playground with S3
+      ],
+    }
+  });
+  
+  // Make sure `sources` is always a list
+  let sources_list = argv.sources;
+  if ( typeof argv.sources === `string` ) {
+    sources_list = [ argv.sources ];
+  }
 
+  let one_dir_exists = session_vars.set_sources_paths( sources_list );
+  if ( !one_dir_exists ) {
+    // If we don't throw, it'll be more confusing when no tests run
+    throw new ReferenceError( `ALKiln ERROR: ALKiln can't find your "sources" folder(s). See above for details.`);
+  }
+  
   const { runConfiguration } = await cucumber_api.loadConfiguration({
-    provided: provided_config
+    provided: build_config( argv ),
   }, environment);
+
   console.log("alkiln-run: runConfiguration: %o", runConfiguration);
+
   var path = require('path');
   var codeDir = path.dirname(require.main.filename);
+  console.log("alkiln-run: working directory: %o", codeDir);
   const support = await cucumber_api.loadSupport({ support: {
-    requireModules: [`${ codeDir }/index.js`],
-    requirePaths: [ `${ codeDir }/index.js`],
+    requireModules: [ `${ codeDir }/index.js` ],
+    requirePaths: [ `${ codeDir }/index.js` ],
     importPaths: [],
   }, sources: {paths: []}}, environment);
 
-  console.log("alkiln-run: working directory: %o", codeDir);
   console.log(`\n\n`);
 
   const { success } = await cucumber_api.runCucumber({...runConfiguration, support});
@@ -86,6 +119,6 @@ const cucumber_api = require('@cucumber/cucumber/api');
 
     // Put it in the correct file in the correct folder
     let artifacts_path = session_vars.get_artifacts_path_name();
-    fs.appendFileSync( `${ artifacts_path}/${ log.debug_log_file }`, data);
+    fs.appendFileSync( `${ artifacts_path }/${ log.debug_log_file }`, data);
   });
 })();

package/lib/scope.js

@@ -9,6 +9,7 @@ const safe_filename = require("sanitize-filename");
 const faker = require('@faker-js/faker/locale/en').faker;
 
 // Ours
+const pdf_reader = require('./utils/pdf_reader');
 const session_vars = require('./utils/session_vars');
 const get_base_interview_url = require(`./docassemble/get_base_interview_url`);
 const time = require('./utils/time');
@@ -1679,90 +1680,77 @@ module.exports = {
     return valid_date;
   },  // Ends scope.today_to_date()
 
-  uploadFiles: async function ( scope, { handle, answer }) {
-    /* Try to upload one or more files to a file-type input field.
-    *    Does not submit, so no danger of timeout with server reload.
+  findFiles: async function( scope, {to_find_names}) {
+    /** Returns a list of paths to the sources folder associated
+    *    with the given list of file names.
     * 
-    * Returns whether the row was used or not.
+    * @param to_find_names { [str] } List of file names
+    * 
+    * @returns { [str] } sources folder paths
     */
-
     // First make sure the list of files was defined. Don't remove spaces yet -
     // allow a typo where someone forgot to include a space after a comma.
-    let uploads_names = answer.split(`,`);
-    if ( uploads_names.length === 0 ) {
-      reports.addToReport( scope, {
-        type: `error`,
-        value: `Your list of files to upload is empty.`
-      });
-      return false;  // The row wasn't used
-    }
-
-    let dirs = [];
-    if ( session_vars.get_origin() === `playground` ) {
-      let files_playground = fg.sync(
-        [ `/usr/share/docassemble/files/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-      let files_s3 = fg.sync(
-        [ `/tmp/playgroundsources/${session_vars.get_user_id()}/${session_vars.get_user_project_name()}` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-
-      dirs = files_playground.concat( files_s3 );
-    } else {
-      dirs = fg.sync(
-        [ `**/docassemble/**/data/sources` ],
-        { onlyFiles: false, suppressErrors: true }
-      );
-    }
-
-    if ( session_vars.get_debug() ) {
-      console.log( `All discovered dirs to upload from:`, dirs );
-    }
-
-    // Make sure sources directory exists and has files in it
-    if ( dirs.length === 0 ) {
+    // TODO: wish we had the var name here that folks are trying to set
+    if ( to_find_names.length === 0 ) {
       reports.addToReport( scope, {
         type: `warning`,
-        value: `The test files need to be in the "sources" directory, `
-          + `but ALKiln couldn't find the directory or it was empty.`
+        value: `Your Step is missing the names of which file(s) to use.`
       });
-      return false;  // The row wasn't used
+      return [];
     }
 
-    // Get the full paths of all the files
-    let dir = null;
+    let sources_dirs = session_vars.get_sources_paths();
+    // Get the full paths of all the existing files
     let paths = [];
     let missing_files = [];  // For any user warnings
-    for ( let name of uploads_names ) {
-     
+    for ( let name of to_find_names ) {
+
       let trimmed_name = name.trim();
       let matching_paths = [];
-
-      for ( let one_dir of dirs ) {
-        // accumulate all matching paths
-        let new_paths = fg.sync([ `${ one_dir }/${ trimmed_name }` ]);
+      // Look for the file in all "sources" folders
+      for ( let one_dir of sources_dirs ) {
+        let new_paths = fg.sync(
+          [ `${ one_dir }/${ trimmed_name }` ],
+          { suppressErrors: true }
+        );
         matching_paths.push( ...new_paths );
-      }  // ends for each dir
+      }
 
       if ( matching_paths.length === 0 ) {
         missing_files.push( trimmed_name );
       } else {
         paths.push( ...matching_paths );
       }
-
     }  // ends for each file
 
     if ( missing_files.length > 0 ) {
       reports.addToReport( scope, {
         type: `warning`,
-        value: `ALKiln could not find "${ missing_files.join('", "') }" in any of these folders: "${ dirs.join('", "') }".`
+        value: `ALKiln could not find "${ missing_files.join('", "') }" in the folder(s) at "${ sources_dirs.join('", "') }".`
       });
     }
 
-    // If all files missing, return that the row wasn't used
-    if ( missing_files.length === uploads_names.length ) {
-      return false;  // The row wasn't used
+    // If all files missing, return an empty array
+    if ( missing_files.length === to_find_names.length ) {
+      return [];
+    }
+    return paths;
+  },  // Ends scope.findFiles()
+
+  uploadFiles: async function ( scope, { handle, answer }) {
+    /* Try to upload one or more files to a file-type input field.
+    *    Does not submit, so no danger of timeout with server reload.
+    * 
+    * Returns whether the row was used or not.
+    */
+
+    // First make sure the list of files was defined. Don't remove spaces yet -
+    // allow a typo where someone forgot to include a space after a comma.
+    let to_find_names = answer.split(`,`);
+    let paths = await scope.findFiles(scope, {to_find_names});
+    if (paths.length == 0) {
+      // The row wasn't used
+      return false;
     }
 
     // Prepare to wait for upload to complete
@@ -1988,7 +1976,7 @@ module.exports = {
       }
       if ( num_unused_rows > 0 ) {
         let { id } = await scope.examinePageID( scope, 'none to match' );
-        let msg = `Missing variable or variables on page "${id}"`;
+        let msg = `Error on page "${id}". See screenshot and see text above for more details.`;
         reports.addToReport(scope, { type: `error`, value: msg });
         expect( num_unused_rows, msg ).to.equal( 0 );
       }
@@ -2503,6 +2491,32 @@ module.exports = {
       await scope.detectDownloadComplete( scope );
     },  // Ends scope.steps.download()
 
+    compare_pdfs: async function (scope, {existing_pdf_path, new_pdf_path}) {
+      let existing_paths = await scope.findFiles(scope, {to_find_names: [existing_pdf_path]});
+      if (existing_paths.length == 0) {
+        // The row wasn't used
+        let msg = `Could not find the existing PDF at ${ existing_pdf_path }`;
+        reports.addToReport(scope, { type: `error`, value: msg });
+        scope.failed_pdf_compares.push(msg);
+        // Return early since we couldn't find the PDF
+        return;
+      }
+      let full_existing_path = existing_paths[0];
+      let full_new_pdf_path = `${scope.paths.scenario}/${new_pdf_path}`;
+      let diffs = await pdf_reader.compare_pdfs(full_existing_path, full_new_pdf_path);
+      if (diffs) {
+        let added_str = diffs.filter(part => part.added).reduce((err_str, part) => {
+          return err_str + `- ${ part.value }\n`
+        }, 'The new PDF added: \n');
+        let removed_str = diffs.filter(part => part.removed).reduce((err_str, part) => {
+          return err_str + `- ${ part.value }\n`
+        }, 'The new PDF removed: \n');
+        let msg = `The PDFs were not the same.\n${ added_str }\n${removed_str}\n\n You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`;
+        reports.addToReport(scope, { type: `error`, value: msg });
+        scope.failed_pdf_compares.push(msg);
+      }
+    },
+
     set_al_name: async ( scope, var_base, name_str ) => {
       /** Given a variable name and a space-separated string, fills the
       *    appropriate fields on an Assembly Line name question.
@@ -2930,4 +2944,5 @@ module.exports = {
 
     return none_included;
   },
+
 };

package/lib/steps.js

@@ -147,6 +147,7 @@ Before(async (scenario) => {
   scope.page_id = null;
   scope.check_all_for_a11y = false;
   scope.passed_all_for_a11y = true;
+  scope.failed_pdf_compares = [];
 
   // Reset default timeout
   scope.timeout = default_timeout;
@@ -705,6 +706,19 @@ Then(/I get the(?: page's)?(?: JSON)? var(?:iable)?s and val(?:ue)?s/i, { timeou
   );
 });
 
+Then(/I expect the baseline PDF "([^"]+)" and the new PDF "([^"]+)" to be the same/i, async (existing_pdf_path, new_pdf_path) => {
+  /** Compare the text or fields of a baseline PDF and the newly downloaded PDF.
+  * 
+  * Variations:
+  * Then I expect the baseline PDF "simple-doc-Baseline.pdf" and the new PDF "simple-doc.pdf" to be the same
+  */
+  
+  return await wrapPromiseWithTimeout(
+    scope.steps.compare_pdfs(scope, {existing_pdf_path, new_pdf_path}),
+    scope.timeout
+  );
+});
+
 //#####################################
 //#####################################
 // Actions
@@ -1104,12 +1118,21 @@ After(async function(scenario) {
   if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
     // The fact that the scenario failed has already been noted earlier in the report
     changeable_test_status = `FAILED`;
-    scope.addToReport(scope, {
+    reports.addToReport(scope, {
       type: `error`,
       value: `Accessibility standards weren't met. See information above.`
     });
   }
 
+  if ( scope.failed_pdf_compares.length > 0) {
+    let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n---\n${ new_msg }`)
+    changeable_test_status = `FAILED`;
+    reports.addToReport(scope, {
+      type: `error`,
+      value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\n---\n${ msg }\n---\n`
+    });
+  }
+
   // Record status before internal report tests can interfere with values
   reports.setReportScenarioStatus( scope, { status: changeable_test_status });
 

package/lib/utils/pdf_reader.js

@@ -0,0 +1,113 @@
+// All functions dealing with getting information from PDFs
+
+const pdfjs_lib = require("pdfjs-dist/legacy/build/pdf.js");
+const diff_lib = require("diff");
+
+let _load_pdf = async function(pdf_path) {
+  // Simple wrapper to get the document
+  return await pdfjs_lib.getDocument({
+    url: pdf_path,
+    // https://github.com/mozilla/pdf.js/issues/4244#issuecomment-1479534301
+    useSystemFonts: true
+  }).promise;
+};
+
+let _get_content = async function(pdf_doc) {
+  let num_pages = pdf_doc.numPages;
+  let contents = "";
+  for (var i = 1; i <= num_pages; i++) {
+    const page_contents = await (await pdf_doc.getPage(i)).getTextContent();
+    contents += "\n" + page_contents.items.map((item) => item.str).join(" ");
+  }
+  return contents;
+};
+
+let pdf_reader = {};
+
+pdf_reader.get_content = async function(pdf_path) {
+  /* Gets the full text content from the PDF.
+   *
+   * @returns str
+   */
+
+  let doc = await _load_pdf(pdf_path);
+  return await _get_content(doc);
+}; // Ends get_content()
+
+pdf_reader.get_fields = async function(pdf_path) {
+  /* Gets all of the fields in the PDF
+   * 
+   * Example:
+   * {
+   *   "users1_name_first": [
+   *      {
+   *         "id": "6R",
+   *         "value": "aeu",
+   *         "defaultValue": "",
+   *         "multiline": false,
+   *         "password": false,
+   *         "charLimit": 0,
+   *         "comb": false,
+   *         "editable": true,
+   *         "hidden": false,
+   *         "name": "users1_name_first",
+   *         "rect": [
+   *             143.52,
+   *             660.72,
+   *             315.11,
+   *             674.88
+   *         ],
+   *         "actions": null,
+   *         "page": 0,
+   *         "strokeColor": null,
+   *         "fillColor": null,
+   *         "rotation": 0,
+   *         "type": "text"
+   *       }
+   *   ],
+   */
+  let doc = await _load_pdf(pdf_path);
+  const field_objects = await doc.getFieldObjects();
+  return field_objects;
+}; // Ends get_fields
+
+pdf_reader.get_field_value = async function(pdf_path, field_name) {
+  let field_objects = await pdf_reader.get_fields(pdf_path);
+  if (field_name in field_objects) {
+    for (let inst of field_objects[field_name]) {
+      if ("value" in inst) {
+        return inst["value"];
+      }
+    }
+  }
+  return null;
+}; // Ends get_field
+
+pdf_reader.compare_pdfs = async function(existing_pdf_path, new_pdf_path) {
+  /**
+   * Returns the change objects between the PDF fields, if there's any diff
+   */
+  let existing_doc = await _load_pdf(existing_pdf_path);
+  let new_doc = await _load_pdf(new_pdf_path);
+  let existing_content = await _get_content(existing_doc);
+  let new_content = await _get_content(new_doc);
+
+  // Assumption: authors won't be changing their PDF templates programmatically;
+  // so if the text of the PDFs are different, then it's actually the fields that are different,
+  // but they aren't editable / are flattened into the PDF. 
+  let change = diff_lib.diffWords(existing_content, new_content);
+  if (change.some(diff => diff.added || diff.removed)) {
+    return change;
+  }
+
+  const existing_fields = await existing_doc.getFieldObjects();
+  const new_fields = await new_doc.getFieldObjects();
+
+  let changeObjs = diff_lib.diffJson(existing_fields, new_fields);
+  if (changeObjs.some(diff => diff.added || diff.removed)) {
+    return changeObjs;
+  }
+  return null;
+}; // End compare_pdfs
+
+module.exports = pdf_reader;

package/lib/utils/session_vars.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const fg = require('fast-glob');
 const path = require('path'); 
 require('dotenv').config();
 
@@ -37,13 +38,6 @@ session_vars.get_origin = function () {
 };
 session_vars.get_user_project_name = function () { return process.env._PROJECT_NAME || null; };
 session_vars.get_user_id = function () { return process.env._USER_ID || null; };
-session_vars.get_tags = function () {
-  if ( process.env._TAGS ) {
-    return process.env._TAGS.split(` `);
-  } else {
-    return [];
-  }
-};
 
 // More complex logic
 session_vars.get_server_reload_timeout = function () {
@@ -210,6 +204,61 @@ session_vars.get_artifacts_path_name = function () {
   }
 };  // Ends get_artifacts_path_name()
 
+session_vars.set_sources_paths = function ( paths ) {
+  /** Returns whether at least one of the given path exists. Store
+  *   a list of the existing paths. Warn of any missing paths. Can be
+  *   used for getting, e.g .feature files, files to upload, etc.
+  * 
+  * Our code should ensure this gets run before cucumber tests get triggered.
+  * 
+  * @param paths { [str] } List of paths to the 'sources' folder.
+  * 
+  * @returns bool
+  */
+
+  // Check every path, tracking which exist and which are missing
+  let existing_paths = [];
+  let missing_paths = [];
+  for ( let path of paths ) {
+    let dirs = fg.sync(
+      [ path ],
+      { onlyFiles: false, suppressErrors: true }
+    );
+
+    if ( dirs.length > 0 ) {
+      existing_paths.push( ...dirs );
+    } else {
+      missing_paths.push( path );
+    }
+  }
+
+  // Warn about missing paths
+  if ( missing_paths.length > 0 ) {
+    console.warn(
+      `ALKiln WARNING: ALKiln can't find folders at "${ paths.join(", ") }".`
+    );
+  }
+
+  // If no paths were valid
+  if ( existing_paths.length <= 0 ) {
+    session_vars._sources_paths = null;
+    return false;
+  }
+
+  // Otherwise, store valid paths
+  session_vars._sources_paths = existing_paths;
+  return true;
+};
+
+session_vars.get_sources_paths = function () {
+  /** Get the "sources" folder path. Should never be undefined during
+  *   a run of the test. run_cucumber.js should make sure it has
+  *   a default value.
+  * 
+  * @returns str || null */
+  return session_vars._sources_paths;
+};
+
 session_vars.validateEnvironment = function() {
   /** Throw a useful error for the developer if any of the env vars
   *    required by absolutely all tests are missing. Gather all