@suffolklitlab/alkiln 5.13.3-feat-random-2 → 5.13.3-feat-random-4

package/.github/workflows/github_server.yml

@@ -159,9 +159,7 @@ jobs:
           INSTALL_METHOD: "server"
           #### Developer note: You can probably leave the rest out
           #### To learn more, see https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#optional-inputs
-          #ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
-          # Temp debugging GitHub multi-folder problem
-          ALKILN_TAG_EXPRESSION: "@rp1"
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
           ALKILN_VERSION: random
 
       #### Developer note: Example of making an issue when tests fail

package/.github/workflows/playground.yml

@@ -76,12 +76,10 @@ jobs:
           DOCASSEMBLE_DEVELOPER_API_KEY: "${{ env.DOCASSEMBLE_DEVELOPER_API_KEY }}"
           #### Developer note: See https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#optional-inputs
           #### You can probably leave this out
-          MAX_RANDOM_TESTS_PER_SCENARIO: 10
+          ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO: 10
           # Internal: docassemble changes sometimes break @a11y tests. That is
           # a da problem, not ours. We will trigger these tests manually when we
           # want to check up on this.
-          #ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
-          # Temp debugging GitHub multi-folder problem
-          ALKILN_TAG_EXPRESSION: "@rp1"
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
           #### Developer note: You can probably leave this out
           ALKILN_VERSION: random

package/CHANGELOG.md

@@ -43,7 +43,34 @@ Format:
 - 
 -->
 
-<!-- ## [Unreleased] -->
+## [Unreleased]
+
+### Added
+
+- Add the `constrained random answers` Step. You must put the Step in a "constrained random test template" Scenario which goes in a "constrained random test template" file - a `.feature` file that starts with "alkiln_random". The file **MUST have ONLY 1 Scenario per template file** at the moment. The Step is pretty strict. If there are irregularities, it will probably fail, but it will fail quickly. It is flexible about number of tests, though, and will default to 1. Closes [#962](https://github.com/SuffolkLITLab/ALKiln/issues/962).
+- Add `ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO` optional environment variable for the above `constrained random answers` Step. Default value is `40` until we hear otherwise.
+- Regular tests that have incorrect syntax will fail more quickly now - very near the start. So will invalid "constrained random test template" files. This will be a little slower for GitHub Sandbox tests right now, but still faster than it has been in the past. Closes [#972](https://github.com/SuffolkLITLab/ALKiln/issues/972).
+
+### Changed
+
+- Improve report indentation and tweak styling.
+- Clarify some logs, including addressing bite-sized issues like [#981](https://github.com/SuffolkLITLab/ALKiln/issues/981).
+- The report now shows the name of every Step that gets run in a test.
+
+### Internal
+
+- Added: `lib/globals.js` file to hold some shared constant values across the framework. It should NOT manage state. We may want to change the name to `constants.js` or `global_contants.js`. Let's see how it serves us.
+- Added: executable scripts for more setup steps - managing the artifacts path (at the start of a run) and validating and generating files.
+- Changed: `Log` now prioritizes what folder it stores files in in this order:
+    1. Use the argument to use an existing folder and save that to runtime_config.json at root.
+    2. Use the argument to make a new folder with that name and save that to runtime_config.json at root.
+    3. Use the alread-existing folder name value in runtime_config.json at root
+    4. Create a new folder and save that to runtime_config.json at root
+- Changed: Add `.error()` to `Log` for visual clarity when storing an error to throw later. We need to discuss this behavior and its return value. To make that work Logs can now get the `error` prop without throwing an error.
+- Changed: the Story Table step now accepts multiple ids and stops at whatever id it reaches first. I highly recommend that only ALKiln generated tests use this feature. We may avoid documenting it unless authors ask for it.
+- Changed: `.gitignore` and `.npmignore` more things
+- Fixed: log codes checker obob.
+- Fixed: updated `actions/upload-artifact@v3` to `actions/upload-artifact@v4`
 
 ## [5.13.4] - 2025-01-11
 

package/action.yml

@@ -33,7 +33,7 @@ inputs:
     description: "The version of docassemblecli to install."
     required: false
     default: "0.0.17"
-  MAX_RANDOM_TESTS_PER_SCENARIO:
+  ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO:
     description: "The maximum number of tests ALKiln will generate for every Scenario that can generate tests."
     required: false
     # Default will be internal to avoid duplication of data
@@ -72,7 +72,7 @@ runs:
         MAX_SECONDS_FOR_SETUP: ${{ inputs.MAX_SECONDS_FOR_SETUP }}
         SERVER_RELOAD_TIMEOUT_SECONDS: ${{ inputs.SERVER_RELOAD_TIMEOUT_SECONDS }}
         DOCASSEMBLECLI_VERSION: ${{ inputs.DOCASSEMBLECLI_VERSION }}
-        MAX_RANDOM_TESTS_PER_SCENARIO: ${{ inputs.MAX_RANDOM_TESTS_PER_SCENARIO }}
+        ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO: ${{ inputs.ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO }}
       shell: bash
       run: |
         log="💡 ALK0023 INFO: Set environment variables and create artifacts folder"
@@ -96,7 +96,7 @@ runs:
         echo "DOCASSEMBLE_DEVELOPER_API_KEY=$DOCASSEMBLE_DEVELOPER_API_KEY" >> $GITHUB_ENV
         echo "ALKILN_VERSION=$ALKILN_VERSION" >> $GITHUB_ENV
         echo "MAX_SECONDS_FOR_SETUP=$MAX_SECONDS_FOR_SETUP" >> $GITHUB_ENV
-        echo "MAX_RANDOM_TESTS_PER_SCENARIO=$MAX_RANDOM_TESTS_PER_SCENARIO" >> $GITHUB_ENV
+        echo "ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO=$ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO" >> $GITHUB_ENV
         echo "SERVER_RELOAD_TIMEOUT_SECONDS=$SERVER_RELOAD_TIMEOUT_SECONDS" >> $GITHUB_ENV
         echo "DOCASSEMBLECLI_VERSION=$DOCASSEMBLECLI_VERSION" >> $GITHUB_ENV
         # Hard-coded internal ALKiln vars
@@ -106,7 +106,7 @@ runs:
       run: |
         log="💡 ALK0024 INFO: Confirm environment variables"
         echo -e "\n$log" >> "${{ env.DEBUG_PATH }}"
-        vars_log='\nALKiln version is ${{ env.ALKILN_VERSION }}\nRepo is "${{ env.REPO_URL }}"\nBranch ref is "${{ env.BRANCH_PATH }}"\nCustom MAX_SECONDS_FOR_SETUP is "${{ env.MAX_SECONDS_FOR_SETUP }}"\nCustom SERVER_RELOAD_TIMEOUT_SECONDS is "${{ env.SERVER_RELOAD_TIMEOUT_SECONDS }}"\nCustom MAX_RANDOM_TESTS_PER_SCENARIO is "${{ env.MAX_RANDOM_TESTS_PER_SCENARIO }}"\n'
+        vars_log='\nALKiln version is ${{ env.ALKILN_VERSION }}\nRepo is "${{ env.REPO_URL }}"\nBranch ref is "${{ env.BRANCH_PATH }}"\nCustom MAX_SECONDS_FOR_SETUP is "${{ env.MAX_SECONDS_FOR_SETUP }}"\nCustom SERVER_RELOAD_TIMEOUT_SECONDS is "${{ env.SERVER_RELOAD_TIMEOUT_SECONDS }}"\nCustom ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO is "${{ env.ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO }}"\n'
         echo -e "$vars_log" >> "${{ env.DEBUG_PATH }}" && echo -e "$vars_log"
 
     # Install ALKiln during GitHub workflow

package/docs/constrained_random_tests.md

@@ -252,3 +252,27 @@ Scenario: I visit the zoo
 
 The AST doesn't have a list of lines. For examples, Scenarios and comments are in separate lists. The AST also stores line numbers, so we'd have to keep updating line numbers for every object, which adds complexity. We've currently decided that the more structured model isn't worth that complexity.
 
+
+## Ranges may not be that useful
+
+Focused tests with actual thought out options are so useful. Take these examples:
+
+```gherkin
+# ❌ Too broad: May miss important scenarios
+Then ALKiln will stop at "end" with any combination of:
+  | var | possible values |
+  | income | 0..1000000 |
+  | assets | 0..1000000 |
+
+# ✅ Better: Focused ranges with critical points
+Then ALKiln will stop at "end" with any combination of:
+  | var | possible values |
+  | income | 0;; 30000;; 50000;; 75000;; 100000 |
+  | assets | 0;; 50000;; 100000;; 250000 |
+```
+
+The first one will make a lot of useless tests. Is there a way to have a cake and eat it too? I'm not sure. Example:
+
+```
+| income | 0..100;; 900..1000 |
+```

package/lib/run_cucumber.js

@@ -79,12 +79,6 @@ async function main() {
   log.info({ code: `ALK0049`, context: `pre_run environment` },
     environment
   );
-
-  // TODO: These are not used as far as I can tell. Delete them
-  // // We'll use the user info to construct the possible paths to /sources
-  // let id = session_vars.get_user_id();
-  // // `project` is the name in the server docassemble "Project" page
-  // let project = session_vars.get_user_project_name();
   
   const { runConfiguration } = await cucumber_api.loadConfiguration({
     provided: build_config( argv ),

package/lib/setup/TestGenerator.js

@@ -7,19 +7,18 @@ const get_Gherkin_AST = require(`./get_Gherkin_AST`);
 const get_error_stacks = require(`../utils/get_error_stacks`);
 
 /**
- * TODO: We might give various levels of categories for warnings. For example,
- *     severe warnings (like "will skip the random test") vs. slight warnings
- *     (like "missing number in constraints random test")
+ * TODO: Update ALKilnInThePlayground to let author download the generated
+ *     tests. Also, store them in a unique location. 
  * TODO: Post-MVP, don't repeat tests that the author has already created
  * TODO: Post-MVP, write Steps that warn about fore-Steps and constraints Steps
  *     when they appear in a regular test. It probably means someone named the
  *     file incorrectly.
- * Discuss: Require author to use "Generator Scenario:" as they keyword (Also
- *     "Generator Scenario Outline:") (or "Scenario Generator", maybe harder to
- *     scan) for 2 reasons. 1. Gets the vocab of "generator" in early (though
+ * Discuss: Require author to use "Template Scenario:" as they keyword (Also
+ *     "Template Scenario Outline:") (or "Scenario Template", maybe harder to
+ *     scan) for 2 reasons. 1. Gets the vocab of "template" in early (though
  *     that doesn't match "constraints" Step, which is a whole other language
- *     problem we haven't solved yet.) 2. Saying "MAX_RANDOM_TESTS_PER_SCENARIO"
- *     is confusing when "Scenario" means different things - a generator
+ *     problem we haven't solved yet.) 2. Saying "ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO"
+ *     is confusing when "Scenario" means different things - a template
  *     Scenario and a regular Scenario.
  * Discuss: Now that Log is less fragile and duplicated, shall we create a new
  *     Log instance in here?
@@ -30,6 +29,9 @@ const get_error_stacks = require(`../utils/get_error_stacks`);
  *     do that now, but without the icon.
  * Discuss: If/when we allow multiple Scenarios, research how we find the end
  *     of a Scenario that ends in an `Example`.
+ * Discuss: We might give various levels of categories for warnings. For example,
+ *     severe warnings (like "will skip the random test") vs. slight warnings
+ *     (like "missing number in constraints random test")
  * */
 
 let log = null;
@@ -177,7 +179,7 @@ module.exports = class TestGenerator {
     let scenario_objs = doc_AST.feature.children;
     if ( !scenario_objs[0] ) {
       let syntax_error = log.error({ code: `ALK0235`, context: `generating files`, before: `\n\n`, },
-        `The author's generator .feature file at "${ generator_path }" may be missing its Scenarios.`
+        `It looks like the author's template .feature file at "${ generator_path }" has 0 Scenarios.`
       );
       return { new_contents: null, errors: [new Error( syntax_error )]};
     }
@@ -185,7 +187,7 @@ module.exports = class TestGenerator {
 
     if ( scenario_objs.length > 1 ) {
       let too_many_scenarios = log.error({ code: `ALK0236`, context: `generating files`, },
-        `A generator file can only have 1 generator Scenario. This one has ${ scenario_objs.length }.`
+        `A template file can only have 1 template Scenario. This one has ${ scenario_objs.length }.`
       );
       return { new_contents: ``, errors: [new Error( too_many_scenarios )] };
     }
@@ -250,7 +252,7 @@ module.exports = class TestGenerator {
     let errs_n_warns = new Misbehaviors();
 
     let strs_for_all_new_Scenarios = [];
-    // Soon we want to be able to handle multiple generator Scenarios per file
+    // Soon we want to be able to handle multiple template Scenarios per file
     for ( let scenario_obj_i = 0; scenario_obj_i < scenario_objs.length; scenario_obj_i++ ) {
 
       errs_n_warns.reset_warnings();
@@ -370,11 +372,11 @@ module.exports = class TestGenerator {
 
     let got_fewer_tests_than_desired = new_test_bodies.length < num_to_make;
     if ( got_fewer_tests_than_desired ) {
-      let warn_too_few = log.warn({ code: `ALK0260`, context: `generating files generator-scenario-level`, },
-        `ALKiln could only make ${ new_test_bodies.length } *unique* tests out of the ${ num_to_make } the author wanted for this Scenario. Note that ALKiln compares all generated Scenarios with each other to make sure all are unique. This Scenario may be a duplicate of a Scenario generated by a previous generator. The Scenario:\n      ${ scenario_description_for_logs({ scenario }) }`,
+      let warn_too_few = log.warn({ code: `ALK0260`, context: `generating files template-scenario-level`, },
+        `ALKiln could only make ${ new_test_bodies.length } *unique* tests out of the ${ num_to_make } the author wanted for this Scenario. Note that ALKiln compares all generated Scenarios with each other to make sure all are unique. This Scenario may be a duplicate of a Scenario generated from a previous template. The Scenario:\n      ${ scenario_description_for_logs({ scenario }) }`,
       )
       // Discuss: Should we add more structure to Misbehaviors to give them
-      //     context levels (Feature, generator Scenario, generated Scenario)
+      //     context levels (Feature, template Scenario, generated Scenario)
       misfits.add({ warnings: [ warn_too_few ] });
     }
 
@@ -493,7 +495,7 @@ function get_generator_AST({ file_text, generator_path }) {
 
   if ( generator_errors.length > 0 ) {
     let syntax_error = log.error({ code: `ALK0232`, context: `generating files`, before: `\n\n`, },
-      `The author's generator .feature file at "${ generator_path }" has Gherkin syntax error(s). You might see more warnings above or below this one.\n`,
+      `The author's template .feature file at "${ generator_path }" has Gherkin syntax error(s). You might see more warnings above or below this one.\n`,
       ...get_error_stacks({ errors: generator_errors })
     );
     return { doc_AST: null, errors: [new Error( syntax_error )] };
@@ -503,14 +505,14 @@ function get_generator_AST({ file_text, generator_path }) {
 
   if ( !doc_AST.feature ) {
     let feature_error = log.error({ code: `ALK0233`, context: `generating files`, before: `\n\n`, },
-      `The author's generator .feature file at "${ generator_path }" needs to include the \`Feature\` keyword.`
+      `The author's template .feature file at "${ generator_path }" needs to include the \`Feature\` keyword.`
     );
     return { doc_AST: null, errors: [new Error( feature_error )] };
   }
 
   // Should not get here
   let unknown_error = log.error({ code: `ALK0234`, context: `generating files`, before: `\n\n`, },
-    `The author's generator .feature file at "${ generator_path }" ran into an unanticipated error when ALKiln tried to parse it. If you have a moment, we would love for you to send us your debug.txt.`
+    `The author's template .feature file at "${ generator_path }" ran into an unanticipated error when ALKiln tried to parse it. If you have a moment, we would love for you to send us your debug.txt.`
   );
   return { doc_AST, errors: [new Error( unknown_error )] };
 }
@@ -657,7 +659,7 @@ function get_fore_Steps({ scenario }) {
   if ( there_were_no_fore_Steps ) {
     misfits.add({ errors: [
       new Error( log.error({ code: `ALK0242`, context: `generating files`, },
-        `A generator Scenario's first Step must say how many constrained random answers tests to make. The generator Scenario on line ${ scenario.location.line } is missing that Step. The Scenario's description is:\n${ description }`
+        `A template Scenario's first Step must say how many constrained random answers tests to make. The template Scenario on line ${ scenario.location.line } is missing that Step. The Scenario's description is:\n${ description }`
       ) )
     ] });
   } else {
@@ -724,7 +726,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
     //     a way that is also maintainable?
     warnings.push( 
       log.warn({ code: `ALK0238`, context: `generating files`, },
-        `A generator Scenario has too many Steps that set the number of tests to make. ALKiln will use the final number Step it finds or calculates. The current Step:\n      ${ text }`
+        `A template Scenario has too many Steps that set the number of tests to make. ALKiln will use the final number Step it finds or calculates. The current Step:\n      ${ text }`
       )
     );
   }
@@ -735,7 +737,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
   if ( number_matcher === null ) {
     warnings.push( 
       log.warn({ code: `ALK0239`, context: `generating files`, },
-        `A generator Scenario is missing the number of tests to make. ALKiln will now make 1 constrained random answers test. The current Step:\n      ${ text }`
+        `A template Scenario is missing the number of tests to make. ALKiln will now make 1 constrained random answers test. The current Step:\n      ${ text }`
       )
     );
     return { num_to_make: 1, warnings, errors };
@@ -759,14 +761,14 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
   }
 
   let num_to_make = Math.min(
-    session_vars.get_max_tests_per_generator_scenario(),
+    session_vars.get_max_tests_per_template_scenario(),
     user_num
   );
 
   if ( num_to_make !== user_num ) {
     warnings.push(
       log.warn({ code: `ALK0241`, context: `generating files`, },
-        `The author asked to make ${ user_num } tests, but maximum allowed is ${ num_to_make }. ALKiln will now make ${ num_to_make } tests. If you want to increase the maximum, use the MAX_TESTS_PER_GENERATOR_SCENARIO environment variable. Read more about environment variables at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#env-vars.`
+        `The author asked to make ${ user_num } tests, but maximum allowed is ${ num_to_make }. ALKiln will now make ${ num_to_make } tests. If you want to increase the maximum, use the ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO input. Read more about inputs at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#workflow-inputs. The Step:\n      ${ text }`
       )
     );
   }
@@ -1099,7 +1101,7 @@ function get_final_ids({ step_text }) {
   let ids = step_text.match( CAPTURE_QUESTION_IDS_REGEX );
 
   if ( ids === null ) {
-    let missing_ids_msg = `The constrained random answers Step is missing its question ids. ALKiln is unable to create random tests for this generator Scenario. The Step:\n${ step_text }`;
+    let missing_ids_msg = `The constrained random answers Step is missing its question ids. ALKiln is unable to create random tests for this template Scenario. The Step:\n${ step_text }`;
     let error = log.error({ code: `ALK0250`, context: `generating files`, }, missing_ids_msg );
     errors.push(new Error( error ));
   }
@@ -1348,7 +1350,7 @@ function remove_warning_tag_placeholder({ new_text }) {
 
 
 function add_total_tests({ scenario_group_str, num_tests }) {
-  /** Paste in the total number of tests generated for this generator Scenario.
+  /** Paste in the total number of tests generated for this template Scenario.
    * 
    * @param {object} obj - Named arguments
    * @param {str} obj.scenario_group_str - The combined string of all the
@@ -1369,10 +1371,10 @@ function add_warnings_Scenario({ scenarios_str, warnings }) {
    *     Scenarios. Basically, a whole-generator-scope warning.
    * 
    * Discuss: Add a pre-group Scenario for every group of generated Scenarios
-   *     identifying the generator Scenario of the subsequent new Scenarios.
+   *     identifying the template Scenario of the subsequent new Scenarios.
    *     That is, even when there is no warning to give.
    * 
-   * TODO: If/when we allow multiple generator Scenarios, use the ID here too
+   * TODO: If/when we allow multiple template Scenarios, use the ID here too
    * 
    * @param {object} obj - Named arguments
    * @param {str} obj.scenarios_str - The str of the already existing Scenarios
@@ -1381,7 +1383,7 @@ function add_warnings_Scenario({ scenarios_str, warnings }) {
    * @returns {str} - The text of the combined Scenarios.
    * */
   let tags = `@alkiln_extra_intro ${ ALKILN_WARNING_TAG }`;
-  let keyword_line = `Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.`;
+  let keyword_line = `Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.`;
   let step_text = get_goofy_warnings_Step_text();
 
   let warnings_lines = warnings_as_list_of_lines({ warnings });
@@ -1435,7 +1437,7 @@ function get_warnings_docstring({ warnings_lines, indent }) {
    * */
   let doc_str_lines = [
     `"""`,
-    `――――― Generator file warnings ―――――`,
+    `――――― Template file warnings ―――――`,
     ...warnings_lines,
     `―――――`,
     `"""`,
@@ -1506,7 +1508,7 @@ function save_files({ tests_path, artifacts_path, file_strings }) {
 
   let count = 1;
   for ( let file_str of file_strings ) {
-    let filename = `${ count }_${ Date.now() }.feature`;
+    let filename = `_alkiln_generated_${ count }_${ Date.now() }.feature`;
     count++;
     // Implementation note: If the files don't exist, this creates them
     fs.appendFileSync( `${ tests_path }/${ filename }`, file_str );

package/lib/steps.js

@@ -99,28 +99,12 @@ BeforeAll(async function() {
   // Store path names for files and docs created for the tests
   scope.paths = {};
 
-  // // Make the folder to store all the files for this run of the tests
-  // scope.paths.artifacts = files.make_artifacts_folder();
-  // // Save its name in a file. `session_vars` can get it back later.
-  // // The value we are saving is used by at least run_cucumber.js.
-
   // A little indirect, but the artifacts path should always have already been
   // created by now, either by `run_cucumber.js` or the relevant test
-
-  // // TODO: Is this causing the GitHub tests artifacts trouble? Because
-  // // `.get_artifacts_path_name()` is messing up somehow? That shouldn't be a
-  // // problem. That's the same method `Log` uses. Why does runtime_config not
-  // // have a value here?
-  // scope.paths.artifacts = session_vars.get_artifacts_path_name();
-  // log = new Log({ path: scope.paths.artifacts, context: `steps` });
-
-  console.log(`\n\n`, `scope.paths.artifacts:`, scope.paths.artifacts, `\n\n`);
-  // New attempt avoids passing in a path. For the record, this shouldn't help.
   log = new Log({ context: `steps` });
   scope.set_log({ log_obj: log });
 
   scope.paths.artifacts = session_vars.get_artifacts_path_name();
-
   scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_filename }`;
   scope.paths.report_log = `${ scope.paths.artifacts }/${ log.report_log_filename }`;
   scope.paths.report = `${ scope.paths.artifacts }/${ log.report_filename }`;
@@ -196,7 +180,7 @@ BeforeStep(async function ( step ) {
   //    start of a Step after it's been completed, as cucumber already does. Add
   //    a 'step' type and additional success/failure information, though that
   //    may be challenging.
-  reports.addToReport(scope, { type: `row info`, value: step.pickleStep.text });
+  reports.addToReport(scope, { type: `row`, value: step.pickleStep.text });
   reports.indent();
 });
 

package/lib/utils/Log.js

@@ -147,8 +147,6 @@ class Log {
      * const log = new Log({ path: "_alkiln-123", context: "setup" });
      * */
 
-    console.trace(`\n\nCreating from Log`);
-
     // These records help internal tests check for correct behavior
     this.internal_log_records = [];
 
@@ -164,7 +162,6 @@ class Log {
     if ( context ) { context = context + ` log`; }
     else { context = `log`; }
     let files_msg = `ALKiln will save test results files at "${ this.path }"`;
-    console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. Calculated path:`, this.path);
     if ( is_new_folder ) {
       this.info({ code: `ALK0214`, context, }, files_msg );
     }  else {
@@ -220,30 +217,23 @@ class Log {
      * */
     try {
 
-      console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. Given dir:`, dir);
-
       // GitHub actions pass in a dir, but other processes, like
       //    ALKilnInThePlayground, don't have a path
       if ( !dir ) {
-        console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. No dir given`);
         if ( session_vars.get_artifacts_path_name() ) {
-          console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. Artifacts path already has a name`);
           dir = session_vars.get_artifacts_path_name();
         } else {
-          console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. Making brand new artifacts path`);
           dir = files.make_artifacts_folder();
         }
       }
 
       if ( !fs.existsSync( dir )) {
-        console.log(`🐛🐛🐛🐛🐛 ALKTemp debugging Log path. Artifacts path dir doesn't yet exist`);
         files.make_artifacts_folder( dir );
       }
 
       return { dir, errors: [] };
 
     } catch ( error_making_dir ) {
-      // Logs should fail silently. With no dir, though, we can't save the error
       return { dir, errors: [ error_making_dir ]};
     }
     

package/lib/utils/session_vars.js

@@ -73,7 +73,7 @@ session_vars.get_setup_timeout_ms = function () {
   return timeout;
 };
 
-session_vars.get_max_tests_per_generator_scenario = function() {
+session_vars.get_max_tests_per_template_scenario = function() {
   /** Returns the maximum number of tests to generate for any one generator
    *    Scenario.
    * 
@@ -82,7 +82,7 @@ session_vars.get_max_tests_per_generator_scenario = function() {
    * 
    * @returns {int} - Max number of tests for one generator Scenario
    * */
-  return process.env.MAX_RANDOM_TESTS_PER_SCENARIO || 40;
+  return process.env.ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO || 40;
 }
 
 session_vars.get_da_server_url = function () {
@@ -280,7 +280,6 @@ session_vars.get_runtime_config_var = function ( key ) {
   try {
     let json = JSON.parse( fs.readFileSync( runtime_config_path ));
     let value = json[ key ];
-    console.log(`🐛🐛🐛🐛🐛 ALK00temp config key/value pair:`, key, value);
     if ( session_vars.get_debug() ) {
       console.debug( `🐛 ALK0041 DEBUG: Runtime config var "${ key }" is "${ value }".` );
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "5.13.3-feat-random-2",
+  "version": "5.13.3-feat-random-4",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {

package/tests/unit_tests/constrained_random_answers/valid.fixtures.js

@@ -677,10 +677,10 @@ fixts.warnings.no_number.expected =
 Feature: I generate constrained random tests (generated by ALKiln)
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0239 ${ fixts.WARNING_PLACEHOLDER }
     ―――――
     """
@@ -713,10 +713,10 @@ fixts.warnings.multiple_numbers.expected =
 Feature: I generate constrained random tests (generated by ALKiln)
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0240 ${ fixts.WARNING_PLACEHOLDER }
     ―――――
     """
@@ -750,10 +750,10 @@ fixts.warnings.multiple_number_Steps.expected =
 Feature: I generate constrained random tests (generated by ALKiln)
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0238 ${ fixts.WARNING_PLACEHOLDER }
     ―――――
     """
@@ -787,10 +787,10 @@ fixts.warnings.multiple_number_Steps_and_numbers.expected =
 Feature: I generate constrained random tests (generated by ALKiln)
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0240 ${ fixts.WARNING_PLACEHOLDER }
     🔎 ALK0238 ${ fixts.WARNING_PLACEHOLDER }
     🔎 ALK0240 ${ fixts.WARNING_PLACEHOLDER }
@@ -840,10 +840,10 @@ Feature: I generate valid constrained random answers tests 1 (generated by ALKil
 # The 1st of 2 related tests. This 1st one tests uniqueness within one Scenario.
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0260 ${ fixts.WARNING_PLACEHOLDER }
     ―――――
     """
@@ -868,10 +868,10 @@ Feature: I generate valid constrained random answers tests 2 (generated by ALKil
 # The 2nd of 2 related tests. Because ALKiln will try to only generate unique tests between all the generators, this 2nd one tests uniqueness of tests between generators. Only the body of the Scenario should matter for uniqueness.
 
 @alkiln_extra_intro @alkiln_generator_warning
-Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
   Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
     """
-    ――――― Generator file warnings ―――――
+    ――――― Template file warnings ―――――
     🔎 ALK0260 ${ fixts.WARNING_PLACEHOLDER }
     ―――――
     """
@@ -888,6 +888,43 @@ fixts.warnings.fewer_unique_than_requested.included_log_codes = [
 ];
 
 
+fixts.warnings.over_max = {};
+fixts.warnings.over_max.max_num_allowed = 1;
+fixts.warnings.over_max.arg =
+`Feature: I generate constrained random tests
+
+Scenario: I'm warned of asking for more than the max num tests
+  Given ALKiln makes 2 constrained random tests
+  Given I start the interview at "test_kickout"
+  And ALKiln gets to "end" when I pick from:
+    | var | possible_values |
+    | user_choice | correct;; wrong |`;
+fixts.warnings.over_max.expected =
+`@alkiln @generated @randomized @constraints
+Feature: I generate constrained random tests (generated by ALKiln)
+
+@alkiln_extra_intro @alkiln_generator_warning
+Scenario: ALKiln begs the author's forgiveness. The next tests might be flawed.
+  Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
+    """
+    ――――― Template file warnings ―――――
+    🔎 ALK0241 ${ fixts.WARNING_PLACEHOLDER }
+    ―――――
+    """
+
+@scenario1_of_1
+Scenario: I'm warned of asking for more than the max num tests (1 of 1)
+  Given I start the interview at "test_kickout"
+  And I get to any of the question ids ["end"] with this data:
+    | var | value |
+${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
+fixts.warnings.over_max.exact_log_codes = [
+  `ALK0237`, `ALK0241`, `ALK0243`, `ALK0244`,
+  `ALK0245`, `ALK0248`, `ALK0249`, `ALK0246`, `ALK0246`, `ALK0246`, `ALK0255`, `ALK0257`, `ALK0259`, `ALK0246`,
+  `ALK0261`, `ALK0262`, `ALK0264`,
+];
+
+
 /**
  * ===========
  * Silent warnings

package/tests/unit_tests/constrained_random_answers/valid.test.js

@@ -20,7 +20,7 @@ const TABLE_ROW_REGEX = /^(?!(?:.*\| var \| value \|))(.*\| .* \|)/gm;
 const TABLE_PLACEHOLDER_REGEX = new RegExp(`(${ RANDOM_TABLE_PLACEHOLDER }(\n)?)+`, `g`);
 
 const WARNING_PLACEHOLDER = fixtures.WARNING_PLACEHOLDER;
-const WARNING_MSG_REGEX = /(ALK\d{4} )(.*\d{4}-\d\d-\d\d \d\d:\d\d:\d\dUTC.*(?:\n.+))/g;
+const WARNING_MSG_REGEX = /(ALK\d{4} )(.*\d{4}-\d\d-\d\d \d\d:\d\d:\d\dUTC.*(?!\n    ―)(?:\n)?.+)/g;
 
 const FLUB_PLACEHOLDER = fixtures.FLUB_PLACEHOLDER;
 const FLUB_REGEX = fixtures.FLUB_OPTIONS_REGEX;
@@ -96,6 +96,11 @@ function mutate_globals_with({ testing_vals, do_compare=false }) {
   // Reset log codes
   logger.clear_internal_log_records();
 
+  process.env.ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO = 40;
+  if ( testing_vals.max_num_allowed ) {
+    process.env.ALKILN_MAX_RANDOM_TESTS_PER_SCENARIO = testing_vals.max_num_allowed;
+  }
+
   // Fresh namespace
   let generator = new TestGenerator();
   // Mutates `fixture`
@@ -505,6 +510,23 @@ describe(`Constrained random answers parser, when given a valid generator Scenar
     });
   });
 
+  describe(`that asks for over max # of tests allowed - 1 -`, function () {
+
+    before(function () { mutate_globals_with({ testing_vals: fixtures.warnings.over_max }); });
+
+    let outer = fixtures.warnings.over_max;
+    it(`has these exact log codes with warnings: ${ JSON.stringify( outer.exact_log_codes )}`, function () {
+      expect( get_log_codes({ logger }) ).to.have.all.members( fixture.exact_log_codes );
+    });
+    
+    it(`returns no errors`, function () { expect( errors ).to.have.lengthOf( 0 ); });
+
+    it(`has max # of tests - 1`, function () {
+      expect( comparable_text( new_contents )).to.equal( fixture.expected );
+    });
+
+  });
+
   describe(`that has a blank var value in a generator column`, function () {
 
     before(function () { mutate_globals_with({ testing_vals: fixtures.empty_string_value }); });

package/_alkiln_meta/shift_logs.js

@@ -1,75 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-function shift_logs(folderPath, startNum, endNum) {
-    // Validate inputs
-    if (!folderPath || typeof folderPath !== 'string') {
-        throw new Error('Invalid folder path provided');
-    }
-    if (!startNum || !endNum || isNaN(startNum) || isNaN(endNum)) {
-        throw new Error('Invalid number range provided');
-    }
-
-    // Process directory recursively
-    function processDirectory(dirPath) {
-        // Read all files in the folder
-        const files = fs.readdirSync(dirPath);
-        
-        files.forEach(file => {
-            const some_path = path.join(dirPath, file);
-            const stats = fs.statSync(some_path);
-
-            if (stats.isDirectory()) {
-                // Recursively process subdirectories
-                processDirectory(some_path);
-            } else if (stats.isFile()) {
-                // Only process supported file paths
-                if ( some_path.startsWith(`.`) || some_path.startsWith(`alkiln`) || some_path.startsWith(`_`)) {
-                    console.log(`not editing`, some_path);
-                    return;
-                }
-                if (some_path.match(/(node_modules)|(_alkiln)|(package)|(shrinkwrap)|(runtime)|(alkiln\-)$/)) {
-                    console.log(`not editing`, some_path);
-                    return;
-                }
-
-                let content = fs.readFileSync(some_path, 'utf8');
-                
-                // Create regex pattern for ALK followed by digits
-                const pattern = /\bALK0(\d+)\b/g;
-                
-                // Process matches and adjust numbers
-                content = content.replace(pattern, (match, numStr) => {
-                    console.log(`match in ${ some_path }:`, JSON.stringify(match) );
-                    
-                    const num = parseInt(numStr);
-                    // Only modify numbers within our target range
-                    if (num >= startNum && num <= endNum) {
-                        console.log(`Changing match: previous content:`);
-                        console.log( content );
-                        return `ALK0${num + 1}`;
-                    }
-                    return match;
-                });
-
-                // Write back the modified content
-                fs.writeFileSync(some_path, content);
-                console.log(`Processed file: ${file}`);
-            }
-        });
-    }
-
-    processDirectory(folderPath)
-}
-
-// Example usage
-try {
-    shift_logs(`.`, 236, 276);  //'./your_folder', 123, 456);
-    console.log('Sequence fixing completed');
-} catch (error) {
-    console.error('Error:', error.message);
-}
-
-
-function check_types({ folderPath, startNum, endNum }) {
-}