@suffolklitlab/alkiln 5.13.2 → 5.13.3-feat-random

package/package.json

@@ -1,18 +1,23 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.13.2",
+  "version": "5.13.3-feat-random",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
-    "test": "npm run unit && npm run codes && npm run cucumber",
-    "cucumber": "./lib/run_cucumber.js",
-    "local": "npm run setup && npm run cucumber; npm run takedown",
-    "pass": "npm run cucumber 'not @error and not @temp_error'",
-    "fail": "npm run cucumber '@error'",
+    "test": "npm run build && npm run unit && npm run codes && npm run cucumber",
+    "local": "npm run build && npm run setup && npm run cucumber; npm run takedown",
+    "pass": "npm run build && ./lib/run_cucumber.js 'not @error and not @temp_error'",
+    "fail": "npm run build && npm run cucumber '@error'",
+    "cucumber": "npm run build && ./lib/run_cucumber.js",
     "setup": "./lib/docassemble/setup.js",
     "takedown": "./lib/docassemble/takedown.js",
     "server_install": "./lib/docassemble/server_install.js",
-    "unit": "mocha tests/unit_tests/*.test.*",
+    "build": "npm run artifacts && npm run files",
+    "files": "npm run validate && npm run generate",
+    "validate": "./lib/setup/validate.js",
+    "generate": "./lib/setup/generate.js",
+    "unit": "npm run artifacts && mocha --recursive 'tests/unit_tests/**/*.test.*'",
+    "artifacts": "./lib/setup/artifacts.js",
     "codes": "tests/log_codes/check_codes.sh ."
   },
   "repository": {
@@ -57,6 +62,9 @@
     "alkiln-server-install": "./lib/docassemble/server_install.js",
     "alkiln-setup": "./lib/docassemble/setup.js",
     "alkiln-takedown": "./lib/docassemble/takedown.js",
-    "alkiln-run": "./lib/run_cucumber.js"
+    "alkiln-run": "./lib/run_cucumber.js",
+    "alkiln-artifacts": "./lib/setup/artifacts.js",
+    "alkiln-validate": "./lib/setup/validate.js",
+    "alkiln-generate": "./lib/setup/generate.js"
   }
 }

package/tests/log_codes/check_codes.sh

@@ -11,12 +11,13 @@ exit_code=0
 
 echo " -------------------------------------------------------
 | Usage:                                                |
-| bash $(basename $0) [dir] [c file] [c folder] [-l arg]
+| bash $(basename $0) [dir] [c file] [c folder]         |
 | dir:      The directory in which to search for logs   |
 | c file:   Name of the log instance counter file       |
 | c folder: Path to the folder of the log counter file  |
-| -l arg:   \"1\" prints extra logs                       |
  -------------------------------------------------------"
+# | bash $(basename $0) [dir] [c file] [c folder] [-l arg]
+# | -l arg:   \"1\" prints extra logs                       |
 
 # Get flags and their values
 loudness="0"
@@ -55,7 +56,8 @@ fi
 
 # removed and are no longer used. Syntax used works with GitHub cli - [[:digit:]]
 # https://stackoverflow.com/a/6901221
-instances=$(find "$where_to_look" -type f ! -name "check_codes.sh" ! -name "CONTRIBUTING.md" ! -name "$expected_instances_file" ! -name "debug_log.txt" ! -name "cucumber-report.txt" ! -path "*/node_modules/*" ! -path "*/ALKilnTests/*" ! -path "*/alkiln-*/*" ! -path "*/_alkiln*/*" -print0 | xargs -0 -P 4 grep -0 -ro 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' | grep -v -- '--' )
+# Exclude files and paths
+instances=$(find "$where_to_look" -type f ! -name "CONTRIBUTING.md" ! -name "$expected_instances_file" ! -name "debug_log.txt" ! -name "cucumber-report.txt" ! -name "_alkiln*" ! -path "*/tests/*" ! -path "*/node_modules/*" ! -path "*/ALKilnTests/*" ! -path "*/alkiln-*/*" ! -path "*/_alkiln*/*" ! -path "*/docs/*" ! -path "*/\.*" -print0 | xargs -0 -P 4 grep -0 -ro 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' | grep -v -- '--' )
 
 if [[ "$loudness" != "0" ]]; then
   total_instances=$(echo "$instances" | wc -l)
@@ -100,7 +102,7 @@ fi
 indx=0
 too_many=()
 missing=()
-while [ "$indx" -lt "$highest_code" ]; do
+while [ "$indx" -le "$highest_code" ]; do
   # Turn the index into a log code by prepending ALK and adding leading zeros
   log_code=$(printf "ALK%04d" "$indx")
   # Count ";" - a stand-in for the number of
@@ -125,24 +127,27 @@ while [ "$indx" -lt "$highest_code" ]; do
   # To add to the list of strings to print later
   short_msg="$log_code $num_paths/$num_expected"
   long_msg="$log_code act/exp "
-  long_msg+="$num_paths/$num_expected:"
+  long_msg+="$num_paths/$num_expected"
+  if [ "$num_paths" -gt "$num_expected" ]; then
+    long_msg+=":"
+  fi
   long_msg+=$(echo "${codes_unique_paths[$indx]}" | sed 's/;/\n  - /g')
 
   # missing
   if [ "$num_paths" -lt "$num_expected" ]; then
-    if [[ "$loudness" != "0" ]]; then
+    # if [[ "$loudness" != "0" ]]; then
       missing+=("$long_msg")
-    else
-      missing+=("$short_msg")
-    fi
+    # else
+      # missing+=("$short_msg")
+    # fi
 
   # too_many
   elif [ "$num_paths" -gt "$num_expected" ]; then
-    if [[ "$loudness" != "0" ]]; then
+    # if [[ "$loudness" != "0" ]]; then
       too_many+=("$long_msg")
-    else
-      too_many+=("$short_msg")
-    fi
+    # else
+      # too_many+=("$short_msg")
+    # fi
   fi
 
   let indx++
@@ -190,4 +195,4 @@ else
 fi
 
 
-exit $exit_code
+exit $exit_code

package/tests/log_codes/log_code_expected_instances.txt

@@ -4,7 +4,11 @@ Why we need duplicate codes sometimes: GitHub actions sometimes don't have ways
 
 The script that tests the log codes will ignore this file (or any file with this name), and any file with these names: check_codes.sh, debug_log.txt, cucumber-report.txt. It will also ignore files in folders with these names:, node_modules, ALKilnTests, alkiln-*, _alkiln*. That list might get updated, so for an up-to-date list, always check the file check_codes.sh where we use `--exclude=` and such.
 
-ALK0000:8
+Reasons for deletion:
+0053 - duplicate log
+0173 - redundant
+
+ALK0000:0
 ALK0001:1
 ALK0002:2
 ALK0003:2
@@ -22,7 +26,7 @@ ALK0014:1
 ALK0015:1
 ALK0016:1
 ALK0017:1
-ALK0018:2
+ALK0018:1
 ALK0019:2
 ALK0020:1
 ALK0021:1
@@ -33,10 +37,10 @@ ALK0025:1
 ALK0026:2
 ALK0027:2
 ALK0028:3
-ALK0029:1
+ALK0029:2
 ALK0030:2
-ALK0031:1
-ALK0032:1
+ALK0031:2
+ALK0032:2
 ALK0033:1
 ALK0034:1
 ALK0035:1
@@ -57,7 +61,7 @@ ALK0049:1
 ALK0050:1
 ALK0051:1
 ALK0052:1
-ALK0053:1
+ALK0053:0
 ALK0054:1
 ALK0055:1
 ALK0056:1
@@ -177,7 +181,7 @@ ALK0169:1
 ALK0170:1
 ALK0171:1
 ALK0172:1
-ALK0173:1
+ALK0173:0
 ALK0174:1
 ALK0175:1
 ALK0176:1
@@ -185,7 +189,7 @@ ALK0177:1
 ALK0178:1
 ALK0179:1
 ALK0180:1
-ALK0181:1
+ALK0181:0
 ALK0182:1
 ALK0183:1
 ALK0184:2
@@ -206,4 +210,60 @@ ALK0198:1
 ALK0199:1
 ALK0200:1
 ALK0201:1
-ALK0202:1
+ALK0202:1
+ALK0203:1
+ALK0204:1
+ALK0205:1
+ALK0206:1
+ALK0207:1
+ALK0208:1
+ALK0209:1
+ALK0210:1
+ALK0211:1
+ALK0212:1
+ALK0213:1
+ALK0214:1
+ALK0215:1
+ALK0216:1
+ALK0217:1
+ALK0218:1
+ALK0219:1
+ALK0220:1
+ALK0221:1
+ALK0222:1
+ALK0223:0 "Only logs that throw should give a value to \`error\`. Skipping the value."
+ALK0224:1
+ALK0225:1
+ALK0226:1
+ALK0227:1
+ALK0228:1
+ALK0229:1
+ALK0230:1
+ALK0231:1
+ALK0232:1
+ALK0233:1
+ALK0234:1
+ALK0235:1
+ALK0236:1
+ALK0237:1
+ALK0238:1
+ALK0239:1
+ALK0240:1
+ALK0241:1
+ALK0242:1
+ALK0243:1
+ALK0244:1
+ALK0245:1
+ALK0246:1
+ALK0247:1
+ALK0248:1
+ALK0249:1
+ALK0250:1
+ALK0251:1
+ALK0252:1
+ALK0253:1
+ALK0254:1
+ALK0255:1
+ALK0256:1
+ALK0257:1
+ALK0258:1

package/tests/unit_tests/constrained_random_answers/invalid.fixtures.js

@@ -0,0 +1,155 @@
+/** Actual and expected values for INvalid constrained random answer tests. */
+
+module.exports = fixts = {};
+
+/** Research notes
+ * 
+ * Valid: A feature is valid without a description
+ * Valid: A scenario is valid without a description
+ * 
+ * TODO
+ * Research: Can we get back null or undefined without syntax errors? (unlikely)
+ * Research: Can feature.children[0] exist without a `.scenario` prop? (unlikely)
+ * */
+
+fixts.generator_missing_Feature = {};
+fixts.generator_missing_Feature.arg = `Scenario: Invalid feature file`;
+fixts.generator_missing_Feature.expected = null;
+fixts.generator_missing_Feature.error_codes = [ `ALK0232` ];
+fixts.generator_missing_Feature.log_codes = [ `ALK0232` ];
+
+fixts.generator_missing_contents = {};
+fixts.generator_missing_contents.arg = ``;
+fixts.generator_missing_contents.expected = null;
+fixts.generator_missing_contents.error_codes = [ `ALK0233` ];
+fixts.generator_missing_contents.log_codes = [ `ALK0233` ];
+
+fixts.generator_0_Scenarios = {};
+fixts.generator_0_Scenarios.arg = `Feature: No scenarios`;
+fixts.generator_0_Scenarios.expected = null;
+fixts.generator_0_Scenarios.error_codes = [ `ALK0235` ];
+fixts.generator_0_Scenarios.log_codes = [ `ALK0235` ];
+
+
+/**
+ * Missing fore_Step keywords
+ * Missing constraints Step keywords
+ * */
+
+fixts.missing_fore_Step_keywords = {};
+fixts.missing_fore_Step_keywords.arg =
+`Feature: I generate constrained random tests
+
+Scenario: I fail with missing keywords for a fore Step
+  Given ALKiln makes 2
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_kickout"
+  And ALKiln tests get to "end" with:
+    | var | possible_values |
+    | user_choice | correct;; wrong |
+`;
+fixts.missing_fore_Step_keywords.expected = null;
+fixts.missing_fore_Step_keywords.error_codes = [ `ALK0241`, `ALK0261` ];
+fixts.missing_fore_Step_keywords.log_codes = [
+  `ALK0236`,
+  `ALK0241`,
+  `ALK0261`,
+];
+
+
+fixts.missing_constraints_Step_keywords = {};
+fixts.missing_constraints_Step_keywords.arg =
+`Feature: I generate constrained random tests
+
+Scenario: I fail with missing keywords for a constraints Step
+  Given ALKiln makes 2 constrained random answers tests
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_kickout"
+  And tests get to "end" with:
+    | var | possible_values |
+    | user_choice | correct;; wrong |
+`;
+fixts.missing_constraints_Step_keywords.expected = null;
+fixts.missing_constraints_Step_keywords.error_codes = [ `ALK0254`, `ALK0261` ];
+fixts.missing_constraints_Step_keywords.log_codes = [
+  `ALK0236`, `ALK0242`, `ALK0243`,
+  `ALK0244`, `ALK0247`, `ALK0248`,
+  `ALK0254`,
+  `ALK0261`,
+];
+
+
+fixts.missing_table = {};
+fixts.missing_table.arg =
+`Feature: I generate constrained random tests
+
+Scenario: I fail with the first missing table
+  Given ALKiln makes 2 constrained random answers tests
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_kickout"
+  And ALKiln tests get to "end" with:
+  And ALKiln tests get to "end" with:
+`;
+// ======= ENDS ARG ======= //
+fixts.missing_table.expected = null;
+fixts.missing_table.error_codes = [ `ALK0250`, `ALK0261` ];
+fixts.missing_table.log_codes = [
+  `ALK0236`, `ALK0242`, `ALK0243`,
+  `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`,
+  `ALK0250`,
+  `ALK0261`,
+];
+
+
+fixts.missing_ids = {};
+fixts.missing_ids.arg =
+`Feature: I generate constrained random tests
+
+Scenario: I fail with missing quoted ids in constrained random answers
+  Given ALKiln makes 2 constrained random answers tests
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_kickout"
+  And ALKiln will get to end with:
+    | var | possible_values |
+    | user_choice | correct;; wrong |
+`;
+// ======= ENDS ARG ======= //
+fixts.missing_ids.expected = null;
+fixts.missing_ids.error_codes = [ `ALK0249`, `ALK0261` ];
+fixts.missing_ids.log_codes = [
+  `ALK0236`, `ALK0242`, `ALK0243`,
+  `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`,
+  `ALK0249`,
+  `ALK0261`,
+];
+
+
+// A valid generator Scenario, but an erroring test when run in cucumber, so it
+// will error here instead
+fixts.only_row_is_header = {};
+fixts.only_row_is_header.arg =
+`Feature: I generate constrained random tests
+
+Scenario: Non-table row after table header
+  Given ALKiln makes 2 constrained random answers tests
+  Given the final Scenario status should be "failed"
+  Given I start the interview at "test_kickout"
+  And ALKiln will get to "end" with:
+    | var | possible_values |
+    Then non-table line
+    | user_choice | correct;; wrong |
+`;
+fixts.only_row_is_header.expected = null;
+fixts.only_row_is_header.error_codes = [`ALK0251`, `ALK0261`];
+fixts.only_row_is_header.log_codes = [
+  `ALK0236`, `ALK0242`, `ALK0243`,
+  `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`,
+  `ALK0251`,
+  `ALK0261`,
+];
+
+/**
+ * No test for a table interrupted in the middle of regular rows (the parser
+ * can't detect it)
+ * */
+

package/tests/unit_tests/constrained_random_answers/invalid.test.js

@@ -0,0 +1,233 @@
+const chai = require(`chai`);
+const expect = chai.expect;
+
+const TestGenerator = require(`../../../lib/setup/TestGenerator.js`);
+const Log = require(`../../../lib/utils/Log.js`);
+const fixtures = require(`./invalid.fixtures.js`);
+
+
+function get_error_codes({ errors }) {
+  /** Returns a list of ALKiln ALK#### codes from Error stacks
+   * 
+   * @param {object} obj - Named arguments
+   * @param {[Error]} obj.errors - List of Error instances
+   * 
+   * @returns {[str]} - List of ALKiln log codes in the errors
+   * */
+  let codes = [];
+  for ( let error of errors ) {
+    let matches = error.stack.match(/ALK\d\d\d\d/gm);
+    if ( matches ) {
+      codes.push( ...matches );
+    }
+  }
+
+  return codes;
+};
+
+
+function get_log_codes({ logger }) {
+  /** Returns a list of ALKiln codes from arguments sent to a Log instance
+   * 
+   * @params {object} obj - Named arguments
+   * @params {Log} obj.logger - An instance of Log
+   * @params {object} obj.logger.opts - Options passed to the Log instance
+   * @params {str} obj.logger.opts.code - ALK#### formatted log code
+   * @params {[str]} obj.logger.logs - Additional messages to be logged
+   * 
+   * @returns {[str]} - List of ALKiln log codes in the collection of logs
+   * */
+  let codes = [];
+
+  for ( let record of logger.internal_tests_records ) {
+    codes.push( record.opts.code );
+  }
+
+  return codes;
+}
+
+/** Variables that will change their values for each `it()` */
+const logger = new Log({ context: `invalid generators unit tests` });
+let fixture = null;
+let new_contents = null;
+let errors = null;
+let actual = null;
+function mutate_globals_with({ testing_vals }) {
+  /** Set up values for parsing, parse, record values. Mutates in-scope
+   *    variables in a way that helps chaijs (mocha really) get the right
+   *    values to the right places.
+   * 
+   * @params {object} testing_vals - Arguments to pass to the function and
+   *     expected values
+   * 
+   * @returns {undefined}
+   * */
+  // Resets logger
+  logger.internal_tests_records = [];
+
+  // Fresh namespace
+  let generator = new TestGenerator();
+  // Mutates `fixture`
+  fixture = testing_vals;
+  // Mutates `new_contents` and `error`
+  ({ new_contents, errors } = generator.parse_file({
+    file_text: fixture.arg,
+    generator_path: `used_in_warning_logs`,
+    logger
+  }) );
+};
+
+describe(`Constrained random answers parser,`, function () {
+
+  describe(`when given a missing Feature keyword`, function () {
+
+    before(function () { mutate_globals_with({ testing_vals: fixtures.generator_missing_Feature }); });
+    
+    let outer = fixtures.generator_missing_Feature;
+
+    it(`returns \`null\` for contents value`, function () {
+      expect( new_contents ).to.equal( fixture.expected );
+    });
+    it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+      expect( get_error_codes({ errors: errors }) )
+                .to.have.all.members( fixture.error_codes );
+    });
+    it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+      expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+    });
+
+  });
+
+  describe(`when given a empty generator text`, function () {
+
+    before(function () { mutate_globals_with({ testing_vals: fixtures.generator_missing_contents }); });
+    
+    let outer = fixtures.generator_missing_contents;
+
+    it(`returns \`null\` for contents value`, function () {
+      expect( new_contents ).to.equal( fixture.expected );
+    });
+    it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+      expect( get_error_codes({ errors: errors }) )
+                .to.have.all.members( fixture.error_codes );
+    });
+    it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+      expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+    });
+
+  });
+
+  describe(`when given generator text missing Scenarios`, function () {
+
+    before(function () { mutate_globals_with({ testing_vals: fixtures.generator_0_Scenarios }); });
+    
+    let outer = fixtures.generator_0_Scenarios;
+
+    it(`returns \`null\` for contents value`, function () {
+      expect( new_contents ).to.equal( fixture.expected );
+    });
+    it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+      expect( get_error_codes({ errors: errors }) )
+                .to.have.all.members( fixture.error_codes );
+    });
+    it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+      expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+    });
+
+  });
+
+  describe(`when given an invalid generator Scenario`, function () {
+
+    describe(`where a num tests fore Step is missing keywords,`, function () {
+
+      before(function () { mutate_globals_with({ testing_vals: fixtures.missing_fore_Step_keywords }); });
+
+      let outer = fixtures.missing_fore_Step_keywords;
+      
+      it(`returns \`null\` for contents value`, function () {
+        expect( new_contents ).to.equal( fixture.expected );
+      });
+      it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+        expect( get_error_codes({ errors: errors }) ).to.have.all.members( fixture.error_codes );
+      });
+      it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+        expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+      });
+
+    });
+
+    describe(`where a constraints Step is missing keywords,`, function () {
+
+      before(function () { mutate_globals_with({ testing_vals: fixtures.missing_constraints_Step_keywords }); });
+
+      let outer = fixtures.missing_constraints_Step_keywords;
+      
+      it(`returns \`null\` for contents value`, function () {
+        expect( new_contents ).to.equal( fixture.expected );
+      });
+      it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+        expect( get_error_codes({ errors: errors }) ).to.have.all.members( fixture.error_codes );
+      });
+      it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+        expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+      });
+
+    });
+
+    describe(`that is missing a table,`, function () {
+
+      before(function () { mutate_globals_with({ testing_vals: fixtures.missing_table }); });
+
+      let outer = fixtures.missing_table;
+      
+      it(`returns \`null\` for contents value`, function () {
+        expect( new_contents ).to.equal( fixture.expected );
+      });
+      it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+        expect( get_error_codes({ errors: errors }) ).to.have.all.members( fixture.error_codes );
+      });
+      it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+        expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+      });
+
+    });
+
+    describe(`that has no quoted ids,`, function () {
+
+      before(function () { mutate_globals_with({ testing_vals: fixtures.missing_ids }); });
+
+      let outer = fixtures.missing_ids;
+      
+      it(`returns \`null\` for contents value`, function () {
+        expect( new_contents ).to.equal( fixture.expected );
+      });
+      it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+        expect( get_error_codes({ errors: errors }) ).to.have.all.members( fixture.error_codes );
+      });
+      it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+        expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+      });
+
+    });
+
+    describe(`that has no header row,`, function () {
+
+      before(function () { mutate_globals_with({ testing_vals: fixtures.only_row_is_header }); });
+
+      let outer = fixtures.only_row_is_header;
+      
+      it(`returns \`null\` for contents value`, function () {
+        expect( new_contents ).to.equal( fixture.expected );
+      });
+      it(`has these exact error codes: ${ JSON.stringify( outer.error_codes )}`, function () {
+        expect( get_error_codes({ errors: errors }) ).to.have.all.members( fixture.error_codes );
+      });
+      it(`has these exact _log_ codes: ${ JSON.stringify( outer.log_codes )}`, function () {
+        expect( get_log_codes({ logger }) ).to.have.all.members( fixture.log_codes );
+      });
+
+    });
+
+  });
+
+});