@suffolklitlab/alkiln 5.13.3-feat-random → 5.13.3-feat-random-3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/unit_tests.yml +1 -1
- package/CHANGELOG.md +28 -1
- package/action.yml +6 -8
- package/action_for_github_server/action.yml +31 -39
- package/lib/docassemble/setup.js +1 -3
- package/lib/run_cucumber.js +15 -54
- package/lib/setup/TestGenerator.js +48 -38
- package/lib/setup/generate.js +1 -1
- package/lib/setup/validate.js +1 -1
- package/lib/steps.js +8 -71
- package/lib/utils/Log.js +12 -9
- package/lib/utils/_alkiln-constrained_random_tests_old.js +49 -0
- package/lib/utils/reports.js +6 -7
- package/lib/utils/session_vars.js +10 -6
- package/package.json +13 -10
- package/tests/log_codes/check_codes.sh +3 -3
- package/tests/log_codes/log_code_expected_instances.txt +23 -1
- package/tests/unit_tests/constrained_random_answers/invalid.fixtures.js +46 -24
- package/tests/unit_tests/constrained_random_answers/invalid.test.js +22 -4
- package/tests/unit_tests/constrained_random_answers/valid.fixtures.js +129 -98
- package/tests/unit_tests/constrained_random_answers/valid.test.js +33 -13
|
@@ -11,6 +11,9 @@ const get_error_stacks = require(`../utils/get_error_stacks`);
|
|
|
11
11
|
* severe warnings (like "will skip the random test") vs. slight warnings
|
|
12
12
|
* (like "missing number in constraints random test")
|
|
13
13
|
* TODO: Post-MVP, don't repeat tests that the author has already created
|
|
14
|
+
* TODO: Post-MVP, write Steps that warn about fore-Steps and constraints Steps
|
|
15
|
+
* when they appear in a regular test. It probably means someone named the
|
|
16
|
+
* file incorrectly.
|
|
14
17
|
* Discuss: Require author to use "Generator Scenario:" as they keyword (Also
|
|
15
18
|
* "Generator Scenario Outline:") (or "Scenario Generator", maybe harder to
|
|
16
19
|
* scan) for 2 reasons. 1. Gets the vocab of "generator" in early (though
|
|
@@ -96,13 +99,13 @@ module.exports = class TestGenerator {
|
|
|
96
99
|
}
|
|
97
100
|
}
|
|
98
101
|
|
|
99
|
-
log.debug({ code: `
|
|
102
|
+
log.debug({ code: `ALK0267`, context: `generating files`, }, `Constrained random answers tests:`, new_files_strings );
|
|
100
103
|
|
|
101
104
|
let tests_path = `${ sources_path }/${ FOLDER_NAME }`;
|
|
102
105
|
let artifacts_path = `${ session_vars.get_artifacts_path_name() }/${ FOLDER_NAME }`;
|
|
103
106
|
save_files({ tests_path, artifacts_path, file_strings: new_files_strings });
|
|
104
107
|
|
|
105
|
-
log.info({ code: `
|
|
108
|
+
log.info({ code: `ALK0271`, context: `generating files`, }, `Tried to save generated files to ${ artifacts_path }` );
|
|
106
109
|
|
|
107
110
|
return { new_strings: new_files_strings, errors: all_errors };
|
|
108
111
|
}
|
|
@@ -140,7 +143,7 @@ module.exports = class TestGenerator {
|
|
|
140
143
|
|
|
141
144
|
let { errors: generated_parsing_errors } = validate_syntax({ file_text: new_contents, generator_path });
|
|
142
145
|
if ( generated_parsing_errors.length > 0 ) {
|
|
143
|
-
let syntax_error = log.error({ code: `
|
|
146
|
+
let syntax_error = log.error({ code: `ALK0266`, context: `generating files`, before: `\n\n`, },
|
|
144
147
|
`The .feature file ALKiln generated from "${ generator_path }" had Gherkin syntax error(s) ALKiln did not anticipate. If you have a moment, we would love you to send us your debug.txt.\n`,
|
|
145
148
|
...get_error_stacks({ errors: generated_parsing_errors })
|
|
146
149
|
);
|
|
@@ -180,6 +183,12 @@ module.exports = class TestGenerator {
|
|
|
180
183
|
}
|
|
181
184
|
// `scenario_objs[0].scenario` seems to always be defined by here
|
|
182
185
|
|
|
186
|
+
if ( scenario_objs.length > 1 ) {
|
|
187
|
+
let too_many_scenarios = log.error({ code: `ALK0236`, context: `generating files`, },
|
|
188
|
+
`A generator file can only have 1 generator Scenario. This one has ${ scenario_objs.length }.`
|
|
189
|
+
);
|
|
190
|
+
return { new_contents: ``, errors: [new Error( too_many_scenarios )] };
|
|
191
|
+
}
|
|
183
192
|
|
|
184
193
|
let generation_errors = [];
|
|
185
194
|
|
|
@@ -197,12 +206,12 @@ module.exports = class TestGenerator {
|
|
|
197
206
|
let count_msg = `ALKiln generated ${ scenarios_strings.length } Scenario or Scenarios`;
|
|
198
207
|
|
|
199
208
|
if ( scenarios_strings.length === 0 ) {
|
|
200
|
-
let error = log.error({ code: `
|
|
209
|
+
let error = log.error({ code: `ALK0263`, context: `generating files`, }, count_msg );
|
|
201
210
|
generation_errors.push( new Error( error ));
|
|
202
211
|
return { new_contents: null, errors: generation_errors };
|
|
203
212
|
}
|
|
204
213
|
|
|
205
|
-
log.info({ code: `
|
|
214
|
+
log.info({ code: `ALK0264`, context: `generating files`, }, count_msg );
|
|
206
215
|
|
|
207
216
|
let scenarios_combined = scenarios_strings.join(`\n\n`);
|
|
208
217
|
let new_file_contents = [ feature_text, scenarios_combined ].join(`\n`);
|
|
@@ -210,7 +219,7 @@ module.exports = class TestGenerator {
|
|
|
210
219
|
if ( !new_file_contents ) {
|
|
211
220
|
// Fallback. Haven't been able to trigger this
|
|
212
221
|
let msg = `Unable to generate a constrained random answers file from "${ generator_path }". See above for warnings and errors.`;
|
|
213
|
-
let error = log.error({ code: `
|
|
222
|
+
let error = log.error({ code: `ALK0265`, context: `generating files`, }, msg );
|
|
214
223
|
return { new_contents: null, errors: [ new Error( error )] };
|
|
215
224
|
}
|
|
216
225
|
|
|
@@ -241,6 +250,7 @@ module.exports = class TestGenerator {
|
|
|
241
250
|
let errs_n_warns = new Misbehaviors();
|
|
242
251
|
|
|
243
252
|
let strs_for_all_new_Scenarios = [];
|
|
253
|
+
// Soon we want to be able to handle multiple generator Scenarios per file
|
|
244
254
|
for ( let scenario_obj_i = 0; scenario_obj_i < scenario_objs.length; scenario_obj_i++ ) {
|
|
245
255
|
|
|
246
256
|
errs_n_warns.reset_warnings();
|
|
@@ -248,7 +258,7 @@ module.exports = class TestGenerator {
|
|
|
248
258
|
let generator_Scenario = scenario_objs[ scenario_obj_i ].scenario;
|
|
249
259
|
|
|
250
260
|
let description = scenario_description_for_logs({ scenario: generator_Scenario });
|
|
251
|
-
log.info({ code: `
|
|
261
|
+
log.info({ code: `ALK0237`, context: `generating files`, },
|
|
252
262
|
`Generating files for Scenario at ${ description }`
|
|
253
263
|
);
|
|
254
264
|
|
|
@@ -275,7 +285,7 @@ module.exports = class TestGenerator {
|
|
|
275
285
|
scenarios_str: new_Scenarios_text, warnings: errs_n_warns.warnings
|
|
276
286
|
});
|
|
277
287
|
}
|
|
278
|
-
log.debug({ code: `
|
|
288
|
+
log.debug({ code: `ALK0262`, context: `generating files`, }, `new_Scenarios_text with ${ errs_n_warns.warnings.length } warnings:\n`, new_Scenarios_text );
|
|
279
289
|
|
|
280
290
|
strs_for_all_new_Scenarios.push( new_Scenarios_text );
|
|
281
291
|
} // ends for every scenario_container
|
|
@@ -360,8 +370,8 @@ module.exports = class TestGenerator {
|
|
|
360
370
|
|
|
361
371
|
let got_fewer_tests_than_desired = new_test_bodies.length < num_to_make;
|
|
362
372
|
if ( got_fewer_tests_than_desired ) {
|
|
363
|
-
let warn_too_few = log.warn({ code: `
|
|
364
|
-
`ALKiln could only make ${ new_test_bodies.length } unique tests out of the ${ num_to_make } the author wanted for this Scenario:\n ${ scenario_description_for_logs({ scenario }) }`,
|
|
373
|
+
let warn_too_few = log.warn({ code: `ALK0260`, context: `generating files generator-scenario-level`, },
|
|
374
|
+
`ALKiln could only make ${ new_test_bodies.length } *unique* tests out of the ${ num_to_make } the author wanted for this Scenario. Note that ALKiln compares all generated Scenarios with each other to make sure all are unique. This Scenario may be a duplicate of a Scenario generated by a previous generator. The Scenario:\n ${ scenario_description_for_logs({ scenario }) }`,
|
|
365
375
|
)
|
|
366
376
|
// Discuss: Should we add more structure to Misbehaviors to give them
|
|
367
377
|
// context levels (Feature, generator Scenario, generated Scenario)
|
|
@@ -373,7 +383,7 @@ module.exports = class TestGenerator {
|
|
|
373
383
|
scenario_group_str: new_Scenarios_text, num_tests: new_test_bodies.length
|
|
374
384
|
});
|
|
375
385
|
|
|
376
|
-
log.debug({ code: `
|
|
386
|
+
log.debug({ code: `ALK0261`, context: `generating files`, }, `new_Scenarios_text before adding warnings:\n`, new_Scenarios_text );
|
|
377
387
|
|
|
378
388
|
return { new_Scenarios_text, ...misfits };
|
|
379
389
|
}
|
|
@@ -400,7 +410,7 @@ module.exports = class TestGenerator {
|
|
|
400
410
|
* */
|
|
401
411
|
let misfits = new Misbehaviors();
|
|
402
412
|
|
|
403
|
-
log.debug({ code: `
|
|
413
|
+
log.debug({ code: `ALK0249`, context: `generating files`, }, `Trying to get unique middle` );
|
|
404
414
|
|
|
405
415
|
let keyword_index = get_keyword_index({ scenario });
|
|
406
416
|
|
|
@@ -436,17 +446,17 @@ module.exports = class TestGenerator {
|
|
|
436
446
|
}
|
|
437
447
|
|
|
438
448
|
let one_test_body = attempt_lines.join(`\n`);
|
|
439
|
-
log.debug({ code: `
|
|
449
|
+
log.debug({ code: `ALK0257`, context: `generating files` }, `Generated test:\n${ one_test_body }` );
|
|
440
450
|
|
|
441
451
|
let is_duplicate = this.all_Scenarios_bodies_map[ one_test_body ];
|
|
442
452
|
if ( is_duplicate ) {
|
|
443
|
-
log.debug({ code: `
|
|
453
|
+
log.debug({ code: `ALK0258`, context: `generating files` }, `Test is a duplicate on attempt ${ attempts.count }.` );
|
|
444
454
|
if ( attempts.count >= attempts.max ) { clear_inner_state(); }
|
|
445
455
|
continue;
|
|
446
456
|
}
|
|
447
457
|
|
|
448
458
|
// Otherwise, add to the global map and finish
|
|
449
|
-
log.debug({ code: `
|
|
459
|
+
log.debug({ code: `ALK0259`, context: `generating files` }, `This test is unique on attempt number ${ attempts.count }.` );
|
|
450
460
|
is_unique = true;
|
|
451
461
|
this.all_Scenarios_bodies_map[ one_test_body ] = true;
|
|
452
462
|
|
|
@@ -646,18 +656,18 @@ function get_fore_Steps({ scenario }) {
|
|
|
646
656
|
let description = scenario_description_for_logs({ scenario });
|
|
647
657
|
if ( there_were_no_fore_Steps ) {
|
|
648
658
|
misfits.add({ errors: [
|
|
649
|
-
new Error( log.error({ code: `
|
|
659
|
+
new Error( log.error({ code: `ALK0242`, context: `generating files`, },
|
|
650
660
|
`A generator Scenario's first Step must say how many constrained random answers tests to make. The generator Scenario on line ${ scenario.location.line } is missing that Step. The Scenario's description is:\n${ description }`
|
|
651
661
|
) )
|
|
652
662
|
] });
|
|
653
663
|
} else {
|
|
654
|
-
log.debug({ code: `
|
|
664
|
+
log.debug({ code: `ALK0243`, context: `generating files` },
|
|
655
665
|
`The final "first Step" Gherkin index was ${ last_valid_fore_step_index }. The final file index of that Step was ${ as_index(steps[ last_valid_fore_step_index ].location.line) }. The Scenario description:\n${ description }`
|
|
656
666
|
);
|
|
657
667
|
}
|
|
658
668
|
|
|
659
669
|
if ( first_vals.num_to_make ) {
|
|
660
|
-
log.debug({ code: `
|
|
670
|
+
log.debug({ code: `ALK0244`, context: `generating files` },
|
|
661
671
|
`Will try to make ${ first_vals.num_to_make } tests.`
|
|
662
672
|
);
|
|
663
673
|
}
|
|
@@ -713,7 +723,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
|
|
|
713
723
|
// also possibly repeat this message. Can we do something about that in
|
|
714
724
|
// a way that is also maintainable?
|
|
715
725
|
warnings.push(
|
|
716
|
-
log.warn({ code: `
|
|
726
|
+
log.warn({ code: `ALK0238`, context: `generating files`, },
|
|
717
727
|
`A generator Scenario has too many Steps that set the number of tests to make. ALKiln will use the final number Step it finds or calculates. The current Step:\n ${ text }`
|
|
718
728
|
)
|
|
719
729
|
);
|
|
@@ -724,7 +734,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
|
|
|
724
734
|
/** Missing number */
|
|
725
735
|
if ( number_matcher === null ) {
|
|
726
736
|
warnings.push(
|
|
727
|
-
log.warn({ code: `
|
|
737
|
+
log.warn({ code: `ALK0239`, context: `generating files`, },
|
|
728
738
|
`A generator Scenario is missing the number of tests to make. ALKiln will now make 1 constrained random answers test. The current Step:\n ${ text }`
|
|
729
739
|
)
|
|
730
740
|
);
|
|
@@ -737,7 +747,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
|
|
|
737
747
|
// than the first
|
|
738
748
|
user_num = parseInt( number_matcher[ number_matcher.length - 1 ]);
|
|
739
749
|
warnings.push(
|
|
740
|
-
log.warn({ code: `
|
|
750
|
+
log.warn({ code: `ALK0240`, context: `generating files`, },
|
|
741
751
|
`This constrained random answers Step is ambiguous because it has more than 1 number. ALKiln will now use the last number in the Step (${ user_num }). The Step:\n ${ text }`,
|
|
742
752
|
)
|
|
743
753
|
);
|
|
@@ -755,7 +765,7 @@ function get_number_of_Scenarios_to_generate2({ step, prev_num_to_make }) {
|
|
|
755
765
|
|
|
756
766
|
if ( num_to_make !== user_num ) {
|
|
757
767
|
warnings.push(
|
|
758
|
-
log.warn({ code: `
|
|
768
|
+
log.warn({ code: `ALK0241`, context: `generating files`, },
|
|
759
769
|
`The author asked to make ${ user_num } tests, but maximum allowed is ${ num_to_make }. ALKiln will now make ${ num_to_make } tests. If you want to increase the maximum, use the MAX_TESTS_PER_GENERATOR_SCENARIO environment variable. Read more about environment variables at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#env-vars.`
|
|
760
770
|
)
|
|
761
771
|
);
|
|
@@ -794,7 +804,7 @@ function get_definitely_unqiue_Scenario_lines({
|
|
|
794
804
|
|
|
795
805
|
let unqiue_Scenario_lines = [];
|
|
796
806
|
|
|
797
|
-
log.debug({ code: `
|
|
807
|
+
log.debug({ code: `ALK0245`, context: `generating files`, },
|
|
798
808
|
`Trying to get get_definitely_unqiue_Scenario_lines`
|
|
799
809
|
);
|
|
800
810
|
|
|
@@ -823,7 +833,7 @@ function get_definitely_unqiue_Scenario_lines({
|
|
|
823
833
|
|
|
824
834
|
unqiue_Scenario_lines.push( ...tags_tweens, keyword_text );
|
|
825
835
|
|
|
826
|
-
log.debug({ code: `
|
|
836
|
+
log.debug({ code: `ALK0248`, context: `generating files`, },
|
|
827
837
|
`Unique lines:`, unqiue_Scenario_lines
|
|
828
838
|
);
|
|
829
839
|
|
|
@@ -949,7 +959,7 @@ function get_possibly_unique_lines_data({
|
|
|
949
959
|
} // Ends loop for every step
|
|
950
960
|
|
|
951
961
|
if ( !found_at_least_1_significant_step ) {
|
|
952
|
-
let error = log.error({ code: `
|
|
962
|
+
let error = log.error({ code: `ALK0256`, context: `generating files`, },
|
|
953
963
|
`ALKiln found 0 valid constraints Steps for the Scenario at line ${ scenario.location.line }: "${ scenario_description_for_logs({ scenario }) }". Check the Step docs at https://assemblyline.suffolklitlab.org/docs/alkiln/writing#constrained-random-answers for the correct language and syntax.`
|
|
954
964
|
);
|
|
955
965
|
misfits.add({ errors: [new Error( error )] });
|
|
@@ -1011,14 +1021,14 @@ function get_random_Story_Table({
|
|
|
1011
1021
|
if ( misfits.errors.length > 0 ) { return { table: null, ...misfits }; }
|
|
1012
1022
|
|
|
1013
1023
|
if ( !step.dataTable ) {
|
|
1014
|
-
let error = log.error({ code: `
|
|
1024
|
+
let error = log.error({ code: `ALK0251`, context: `generating files`, },
|
|
1015
1025
|
`The table is missing for the constrained random answers Step at line ${ step.location.line }. The Step:`,
|
|
1016
1026
|
get_flatter_table_Step({ step })
|
|
1017
1027
|
);
|
|
1018
1028
|
misfits.add({ errors: [ new Error( error ) ] });
|
|
1019
1029
|
|
|
1020
1030
|
} else if ( step.dataTable.rows.length === 1 ) {
|
|
1021
|
-
let error = log.error({ code: `
|
|
1031
|
+
let error = log.error({ code: `ALK0252`, context: `generating files`, },
|
|
1022
1032
|
`The body of the table is missing for the table of the constrained random answers Step at line ${ step.location.line }. There is only a header row. The Step:`,
|
|
1023
1033
|
get_flatter_table_Step({ step }) );
|
|
1024
1034
|
misfits.add({ errors: [new Error( error )] });
|
|
@@ -1058,7 +1068,7 @@ function get_random_Story_Table({
|
|
|
1058
1068
|
}
|
|
1059
1069
|
|
|
1060
1070
|
let table = story_table_parts.join(`\n`);
|
|
1061
|
-
log.debug({ code: `
|
|
1071
|
+
log.debug({ code: `ALK0255`, context: `generating files`, }, `Table:`, table );
|
|
1062
1072
|
return { table, ...misfits };
|
|
1063
1073
|
}
|
|
1064
1074
|
|
|
@@ -1090,7 +1100,7 @@ function get_final_ids({ step_text }) {
|
|
|
1090
1100
|
|
|
1091
1101
|
if ( ids === null ) {
|
|
1092
1102
|
let missing_ids_msg = `The constrained random answers Step is missing its question ids. ALKiln is unable to create random tests for this generator Scenario. The Step:\n${ step_text }`;
|
|
1093
|
-
let error = log.error({ code: `
|
|
1103
|
+
let error = log.error({ code: `ALK0250`, context: `generating files`, }, missing_ids_msg );
|
|
1094
1104
|
errors.push(new Error( error ));
|
|
1095
1105
|
}
|
|
1096
1106
|
|
|
@@ -1193,7 +1203,7 @@ function pick_random_answer({ answers_str=`` }) {
|
|
|
1193
1203
|
// with problems that may not exist. We can always make the warnings
|
|
1194
1204
|
// louder later. Should we make them loud instead? And suggest the author
|
|
1195
1205
|
// should put the empty one in the middle?
|
|
1196
|
-
log.debug({ code: `
|
|
1206
|
+
log.debug({ code: `ALK0254`, context: `generating files` },
|
|
1197
1207
|
`**Quiet warning** - One of the constrained random answer choices in a row in this table is "". It could be on purpose or it could be a misplaced separator (";;"). The choices: "${ answers_str }"`
|
|
1198
1208
|
);
|
|
1199
1209
|
}
|
|
@@ -1425,9 +1435,9 @@ function get_warnings_docstring({ warnings_lines, indent }) {
|
|
|
1425
1435
|
* */
|
|
1426
1436
|
let doc_str_lines = [
|
|
1427
1437
|
`"""`,
|
|
1428
|
-
|
|
1438
|
+
`――――― Generator file warnings ―――――`,
|
|
1429
1439
|
...warnings_lines,
|
|
1430
|
-
|
|
1440
|
+
`―――――`,
|
|
1431
1441
|
`"""`,
|
|
1432
1442
|
];
|
|
1433
1443
|
|
|
@@ -1490,7 +1500,7 @@ function save_files({ tests_path, artifacts_path, file_strings }) {
|
|
|
1490
1500
|
fs.mkdirSync( tests_path, { recursive: true });
|
|
1491
1501
|
fs.mkdirSync( artifacts_path, { recursive: true });
|
|
1492
1502
|
|
|
1493
|
-
log.debug({ code: `
|
|
1503
|
+
log.debug({ code: `ALK0269`, context: `generating files`, },
|
|
1494
1504
|
`Saving ${ file_strings.length } newly generated files in "${ tests_path }"`
|
|
1495
1505
|
);
|
|
1496
1506
|
|
|
@@ -1503,7 +1513,7 @@ function save_files({ tests_path, artifacts_path, file_strings }) {
|
|
|
1503
1513
|
fs.appendFileSync( `${ artifacts_path }/${ filename }`, file_str );
|
|
1504
1514
|
}
|
|
1505
1515
|
|
|
1506
|
-
log.debug({ code: `
|
|
1516
|
+
log.debug({ code: `ALK0270`, context: `generating files`, },
|
|
1507
1517
|
`Saved new generated files:`, fs.readdirSync( tests_path )
|
|
1508
1518
|
);
|
|
1509
1519
|
}
|
|
@@ -1526,7 +1536,7 @@ function delete_previous_tests_from_sources_folders_for_ALKiln_developers({ test
|
|
|
1526
1536
|
* @returns {undefined}
|
|
1527
1537
|
* */
|
|
1528
1538
|
if ( !fs.existsSync( tests_path )) { return; }
|
|
1529
|
-
log.debug({ code: `
|
|
1539
|
+
log.debug({ code: `ALK0268`, context: `generating files`, },
|
|
1530
1540
|
`Deleting previously generated files:`, fs.readdirSync( tests_path )
|
|
1531
1541
|
);
|
|
1532
1542
|
fs.rmSync( tests_path, { recursive: true, force: true });
|
|
@@ -1708,13 +1718,13 @@ function get_tweens({
|
|
|
1708
1718
|
tweens.push( generator_lines[ line_i ]);
|
|
1709
1719
|
}
|
|
1710
1720
|
|
|
1711
|
-
log.debug({ code: `
|
|
1721
|
+
log.debug({ code: `ALK0246`, context: `generating files`, },
|
|
1712
1722
|
`Got tween lines. Starting line (excluded):`, generator_lines[ end_index_of_prev ],
|
|
1713
1723
|
`\nTweens:`, tweens,
|
|
1714
1724
|
`\nEnding line (excluded):`, generator_lines[ start_index_of_next ]
|
|
1715
1725
|
);
|
|
1716
1726
|
} catch ( tween_error ) {
|
|
1717
|
-
errors.push( new Error( log.error({ code: `
|
|
1727
|
+
errors.push( new Error( log.error({ code: `ALK0247`, context: `generating files`, },
|
|
1718
1728
|
`ALKiln got an error it is unfamiliar with when generating lines in between other lines starting on Gherkin AST line ${ start }. We would love to see your debug.txt`
|
|
1719
1729
|
)));
|
|
1720
1730
|
errors.push( tween_error );
|
|
@@ -1735,7 +1745,7 @@ function get_authors_indent({ located }) {
|
|
|
1735
1745
|
* given Gherkin location object.
|
|
1736
1746
|
* */
|
|
1737
1747
|
if ( typeof located !== `object` || !located.location ) {
|
|
1738
|
-
debug
|
|
1748
|
+
log.debug({ code: `ALK0253`, context: `generating files`, }
|
|
1739
1749
|
`Skipped get_authors_indent. Given argument:\n`,
|
|
1740
1750
|
located
|
|
1741
1751
|
);
|
package/lib/setup/generate.js
CHANGED
|
@@ -34,7 +34,7 @@ function generate_constrained_random_test_files() {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
if ( all_errors.length > 0 ) {
|
|
37
|
-
log.throw({ code: `
|
|
37
|
+
log.throw({ code: `ALK0272`, context: `generating files`,
|
|
38
38
|
error: new Error( `ALKiln got error(s) when it tried to generate constrained random answers tests. See warnings and errors above.` )
|
|
39
39
|
});
|
|
40
40
|
};
|
package/lib/setup/validate.js
CHANGED
|
@@ -77,7 +77,7 @@ function parse_all_feature_files_in({ sources_path }) {
|
|
|
77
77
|
|
|
78
78
|
if ( these_AST_errors.length > 0 ) {
|
|
79
79
|
all_AST_errors.push( ...these_AST_errors );
|
|
80
|
-
log.warn({ code: `
|
|
80
|
+
log.warn({ code: `ALK0276`, context: `validating files`, },
|
|
81
81
|
`There is a Gherkin syntax error(s) in "${ feature_path }"`,
|
|
82
82
|
...get_error_stacks({ errors: these_AST_errors })
|
|
83
83
|
);
|
package/lib/steps.js
CHANGED
|
@@ -99,17 +99,12 @@ BeforeAll(async function() {
|
|
|
99
99
|
// Store path names for files and docs created for the tests
|
|
100
100
|
scope.paths = {};
|
|
101
101
|
|
|
102
|
-
// // Make the folder to store all the files for this run of the tests
|
|
103
|
-
// scope.paths.artifacts = files.make_artifacts_folder();
|
|
104
|
-
// // Save its name in a file. `session_vars` can get it back later.
|
|
105
|
-
// // The value we are saving is used by at least run_cucumber.js.
|
|
106
|
-
|
|
107
102
|
// A little indirect, but the artifacts path should always have already been
|
|
108
103
|
// created by now, either by `run_cucumber.js` or the relevant test
|
|
109
|
-
|
|
110
|
-
log = new Log({ path: scope.paths.artifacts, context: `steps` });
|
|
104
|
+
log = new Log({ context: `steps` });
|
|
111
105
|
scope.set_log({ log_obj: log });
|
|
112
106
|
|
|
107
|
+
scope.paths.artifacts = session_vars.get_artifacts_path_name();
|
|
113
108
|
scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_filename }`;
|
|
114
109
|
scope.paths.report_log = `${ scope.paths.artifacts }/${ log.report_log_filename }`;
|
|
115
110
|
scope.paths.report = `${ scope.paths.artifacts }/${ log.report_filename }`;
|
|
@@ -181,10 +176,10 @@ Before(async (scenario) => {
|
|
|
181
176
|
// Add a check for an error page before each step? After each step?
|
|
182
177
|
|
|
183
178
|
BeforeStep(async function ( step ) {
|
|
184
|
-
//
|
|
185
|
-
// after it's been completed, as cucumber already does. Add
|
|
186
|
-
// and additional success/failure information, though that
|
|
187
|
-
// challenging
|
|
179
|
+
// Discuss: In the report, add a way for the reports to add "✔" or "✖" to the
|
|
180
|
+
// start of a Step after it's been completed, as cucumber already does. Add
|
|
181
|
+
// a 'step' type and additional success/failure information, though that
|
|
182
|
+
// may be challenging.
|
|
188
183
|
reports.addToReport(scope, { type: `row`, value: step.pickleStep.text });
|
|
189
184
|
reports.indent();
|
|
190
185
|
});
|
|
@@ -352,18 +347,8 @@ Given(/the max(?:imum)? sec(?:ond)?s for each step (?:in this scenario )?is (\d+
|
|
|
352
347
|
// Story
|
|
353
348
|
//#####################################
|
|
354
349
|
//#####################################
|
|
355
|
-
// TODO: Consider more permissive sentences:
|
|
356
|
-
// with(?: this data| these answers)?:?
|
|
357
|
-
// with(?:.*):?
|
|
358
|
-
// ^(?:the user|I)? gets? to ?.*?(".*") ?with .*?
|
|
359
|
-
// .*gets? to ?.*?(".*") ?with.*?
|
|
360
|
-
// More readable?:
|
|
361
|
-
// I? ?gets? to ?.*?(".*") ?with.*?(?:this data:)?
|
|
362
|
-
// get to + stop at (?:gets? to|stops? at)
|
|
363
|
-
// `before "a", "b", "c" and after` -> `"a", "b", "c"`
|
|
364
|
-
// Impossible with cucumber `before "a", "b", "c" and after` -> `["a", "b", "c"]`
|
|
365
350
|
const INCLUDING_QUOTES_REGEX = /"([^"]*)"/g;
|
|
366
|
-
When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)
|
|
351
|
+
When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/i, {timeout: -1}, async (target_ids, raw_var_data) => {
|
|
367
352
|
/* NO CUCUMBER TIMEOUT. Must handle timeout ourselves:
|
|
368
353
|
* https://github.com/cucumber/cucumber-js/blob/master/docs/support_files/timeouts.md#disable-timeouts
|
|
369
354
|
* Iterate in here to keep timeout reasonable */
|
|
@@ -439,7 +424,7 @@ When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/, {timeout: -1}, async (t
|
|
|
439
424
|
if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
|
|
440
425
|
resolve();
|
|
441
426
|
} catch ( err ) {
|
|
442
|
-
let err_msg = `
|
|
427
|
+
let err_msg = `ALKiln ran into an error it tried to answer fields on the page with the question id "${ id }". The text above might have more information.`
|
|
443
428
|
reports.addToReport( scope, { type: `error`, code: `ALK0062`, value: err_msg });
|
|
444
429
|
// Throw error instead?
|
|
445
430
|
reject( err );
|
|
@@ -498,54 +483,6 @@ When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/, {timeout: -1}, async (t
|
|
|
498
483
|
reports.addToReport(scope, { type: `plain`, value: `` });
|
|
499
484
|
});
|
|
500
485
|
|
|
501
|
-
/**
|
|
502
|
-
*
|
|
503
|
-
* | var | possible_values |
|
|
504
|
-
* | favorite_color | blue;; green;; red |
|
|
505
|
-
*
|
|
506
|
-
* @param raw_var_data { [{ var, possible_values }] } List of table row objects
|
|
507
|
-
* @param raw_var_data[i].var {str} Name of the variable to set
|
|
508
|
-
* @param raw_var_data[i].possible_values {str} Format is ";;" separated
|
|
509
|
-
* possible values for the variable (lawyers use ";" grammatically)
|
|
510
|
-
* */
|
|
511
|
-
|
|
512
|
-
/** Need a sentence for:
|
|
513
|
-
* And I will go to the interview at the address ("")
|
|
514
|
-
* And I will run (100) constrained random tests
|
|
515
|
-
* And those tests should get to one of these page ids - [("lsdjf", "sldkjf")]
|
|
516
|
-
* And I will get there with random choices from this data:
|
|
517
|
-
* | var | possible_values |
|
|
518
|
-
* | favorite_color | blue;; green;; red |
|
|
519
|
-
*
|
|
520
|
-
* Write files
|
|
521
|
-
* 1. Make an object from the given constraints table (split by single _and_
|
|
522
|
-
* double quotes? Just double quotes? Document author _must_ use double
|
|
523
|
-
* quotes?)
|
|
524
|
-
* 1. Make sure that it's not the same as previous objects
|
|
525
|
-
* 1. Far in the future: Make sure it's not the same as a test the author
|
|
526
|
-
* already wrote (need to wrap cucumber for this to run all random tests
|
|
527
|
-
* last)
|
|
528
|
-
* 1. Make a Scenario description using the iteration number
|
|
529
|
-
* 1. Add to one big file (name includes datetime) for all these Scenarios
|
|
530
|
-
* 1. Increment iteration number
|
|
531
|
-
* 1. Leave them there to get run later
|
|
532
|
-
*
|
|
533
|
-
* Supposedly more performant to write one big file [^citation needed]. Not
|
|
534
|
-
* sure it matters that much yet. If we do want to optimize, maybe using
|
|
535
|
-
* a write stream directly is the way to go.
|
|
536
|
-
*
|
|
537
|
-
* Command:
|
|
538
|
-
* I run (\d*) random tests for/at "interview.yml" that will reach one of these question/page ids (\[".*"]) with these constraints: (table)
|
|
539
|
-
* (\d*) random at "interview.yml" will reach one/any of (\[.*]) with: (table)
|
|
540
|
-
*
|
|
541
|
-
* Given I generate 100 random tests
|
|
542
|
-
* And I test the interview at ""
|
|
543
|
-
* And I reach any of these question/page ids: ""
|
|
544
|
-
* And I answer with/give these possible answers: (table)
|
|
545
|
-
* I generate 100 random tests for the interview at "" to reach any of the question ids [] with these possible answers
|
|
546
|
-
*
|
|
547
|
-
*/
|
|
548
|
-
|
|
549
486
|
|
|
550
487
|
//#####################################
|
|
551
488
|
//#####################################
|
package/lib/utils/Log.js
CHANGED
|
@@ -10,7 +10,7 @@ const files = require(`./files`);
|
|
|
10
10
|
/**
|
|
11
11
|
* Future issues:
|
|
12
12
|
* - [ ] Save logs from GitHub actions as well? Users can download them
|
|
13
|
-
* themselves, but this might not be clear to them
|
|
13
|
+
* themselves, but this might not be clear to them.
|
|
14
14
|
* - [ ] Capture cucumber ProgressFormatter logs instead of making them
|
|
15
15
|
* ourselves?
|
|
16
16
|
* - [ ] keeping report separate for now (can just be an issue to integrate)
|
|
@@ -18,9 +18,8 @@ const files = require(`./files`);
|
|
|
18
18
|
* message to prepend the accompanying log code, etc (all except the `before`
|
|
19
19
|
* prop). Before this, do UX research to see if the current format is as
|
|
20
20
|
* confusing to authors as it seems from this end.
|
|
21
|
-
* - [ ] Discuss:
|
|
22
|
-
*
|
|
23
|
-
* - [ ] Discuss: Should `.error()` return an Error?
|
|
21
|
+
* - [ ] Discuss: Should `.error()` return an Error? See discussion:
|
|
22
|
+
* https://github.com/SuffolkLITLab/ALKiln/discussions/988
|
|
24
23
|
* */
|
|
25
24
|
|
|
26
25
|
/**
|
|
@@ -149,7 +148,7 @@ class Log {
|
|
|
149
148
|
* */
|
|
150
149
|
|
|
151
150
|
// These records help internal tests check for correct behavior
|
|
152
|
-
this.
|
|
151
|
+
this.internal_log_records = [];
|
|
153
152
|
|
|
154
153
|
let dir = path; // eventually, convert to this.dir instead of this.path
|
|
155
154
|
|
|
@@ -166,7 +165,7 @@ class Log {
|
|
|
166
165
|
if ( is_new_folder ) {
|
|
167
166
|
this.info({ code: `ALK0214`, context, }, files_msg );
|
|
168
167
|
} else {
|
|
169
|
-
this.debug({ code: `
|
|
168
|
+
this.debug({ code: `ALK0277`, context, }, files_msg );
|
|
170
169
|
}
|
|
171
170
|
|
|
172
171
|
// Has this `Log` ever failed to append to the debug file?
|
|
@@ -174,6 +173,10 @@ class Log {
|
|
|
174
173
|
|
|
175
174
|
}
|
|
176
175
|
|
|
176
|
+
clear_internal_log_records() {
|
|
177
|
+
this.internal_log_records = [];
|
|
178
|
+
}
|
|
179
|
+
|
|
177
180
|
_is_new_folder({ dir }) {
|
|
178
181
|
/** Check if the artifacts folder is a new folder.
|
|
179
182
|
*
|
|
@@ -373,8 +376,8 @@ class Log {
|
|
|
373
376
|
|
|
374
377
|
// Throw whatever we can throw
|
|
375
378
|
if ( do_throw ) {
|
|
376
|
-
// WARNING: If someone catches the error we throw, these
|
|
377
|
-
//
|
|
379
|
+
// WARNING: If someone catches the error we throw, these strings will get
|
|
380
|
+
// logged. The final resulting logs may become confusing.
|
|
378
381
|
try {
|
|
379
382
|
console.error( `\n\n\n${before}${metadata_str}:`, ...logs );
|
|
380
383
|
} catch ( pre_throw_error ) {
|
|
@@ -469,7 +472,7 @@ class Log {
|
|
|
469
472
|
} = debug_opts;
|
|
470
473
|
logs = logs || [];
|
|
471
474
|
|
|
472
|
-
this.
|
|
475
|
+
this.internal_log_records.push({ opts: debug_opts, logs });
|
|
473
476
|
|
|
474
477
|
// "warning" level and icon for for internal devs we use to avoid disrupting
|
|
475
478
|
// the info authors are looking for
|
|
@@ -1,3 +1,52 @@
|
|
|
1
|
+
|
|
2
|
+
/**
|
|
3
|
+
*
|
|
4
|
+
* | var | possible_values |
|
|
5
|
+
* | favorite_color | blue;; green;; red |
|
|
6
|
+
*
|
|
7
|
+
* @param raw_var_data { [{ var, possible_values }] } List of table row objects
|
|
8
|
+
* @param raw_var_data[i].var {str} Name of the variable to set
|
|
9
|
+
* @param raw_var_data[i].possible_values {str} Format is ";;" separated
|
|
10
|
+
* possible values for the variable (lawyers use ";" grammatically)
|
|
11
|
+
* */
|
|
12
|
+
|
|
13
|
+
/** Need a sentence for:
|
|
14
|
+
* And I will go to the interview at the address ("")
|
|
15
|
+
* And I will run (100) constrained random tests
|
|
16
|
+
* And those tests should get to one of these page ids - [("lsdjf", "sldkjf")]
|
|
17
|
+
* And I will get there with random choices from this data:
|
|
18
|
+
* | var | possible_values |
|
|
19
|
+
* | favorite_color | blue;; green;; red |
|
|
20
|
+
*
|
|
21
|
+
* Write files
|
|
22
|
+
* 1. Make an object from the given constraints table (split by single _and_
|
|
23
|
+
* double quotes? Just double quotes? Document author _must_ use double
|
|
24
|
+
* quotes?)
|
|
25
|
+
* 1. Make sure that it's not the same as previous objects
|
|
26
|
+
* 1. Far in the future: Make sure it's not the same as a test the author
|
|
27
|
+
* already wrote (need to wrap cucumber for this to run all random tests
|
|
28
|
+
* last)
|
|
29
|
+
* 1. Make a Scenario description using the iteration number
|
|
30
|
+
* 1. Add to one big file (name includes datetime) for all these Scenarios
|
|
31
|
+
* 1. Increment iteration number
|
|
32
|
+
* 1. Leave them there to get run later
|
|
33
|
+
*
|
|
34
|
+
* Supposedly more performant to write one big file [^citation needed]. Not
|
|
35
|
+
* sure it matters that much yet. If we do want to optimize, maybe using
|
|
36
|
+
* a write stream directly is the way to go.
|
|
37
|
+
*
|
|
38
|
+
* Command:
|
|
39
|
+
* I run (\d*) random tests for/at "interview.yml" that will reach one of these question/page ids (\[".*"]) with these constraints: (table)
|
|
40
|
+
* (\d*) random at "interview.yml" will reach one/any of (\[.*]) with: (table)
|
|
41
|
+
*
|
|
42
|
+
* Given I generate 100 random tests
|
|
43
|
+
* And I test the interview at ""
|
|
44
|
+
* And I reach any of these question/page ids: ""
|
|
45
|
+
* And I answer with/give these possible answers: (table)
|
|
46
|
+
* I generate 100 random tests for the interview at "" to reach any of the question ids [] with these possible answers
|
|
47
|
+
*
|
|
48
|
+
*/
|
|
49
|
+
|
|
1
50
|
/**
|
|
2
51
|
* If not doing a 1-Scenario requirement
|
|
3
52
|
*
|
package/lib/utils/reports.js
CHANGED
|
@@ -26,14 +26,14 @@ reports.addReportHeading = function (scope, { scenario }) {
|
|
|
26
26
|
tag_names.push( tag.name );
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
let heading = `\n
|
|
29
|
+
let heading = `\n━━━━━━━━━━━━━━━\n`
|
|
30
30
|
+ `Scenario: ${ scenario.pickle.name }`;
|
|
31
31
|
|
|
32
32
|
if ( tag_names.length > 0 ) {
|
|
33
33
|
heading += `\nTags: ${ tag_names.join(' ') }`;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
heading += `\n
|
|
36
|
+
heading += `\n━━━━━━━━━━━━━━━`;
|
|
37
37
|
|
|
38
38
|
reports.addToReport( scope, {
|
|
39
39
|
code: `ALK0176`,
|
|
@@ -279,7 +279,7 @@ let getLineText = function ({ type=`no type`, code=`ALK00rl`, value=``, indent=`
|
|
|
279
279
|
return result;
|
|
280
280
|
}; // Ends getLine()
|
|
281
281
|
|
|
282
|
-
let getDebugLine = function ({ type=``, code=`
|
|
282
|
+
let getDebugLine = function ({ type=``, code=`ALK0278`, value=``, indent=`` }) {
|
|
283
283
|
/** In debug log lines, include all the information possible. */
|
|
284
284
|
|
|
285
285
|
// `plain` is usually used for visual clarity and shouldn't have
|
|
@@ -299,12 +299,11 @@ let getDebugLine = function ({ type=``, code=`ALK00rd`, value=``, indent=`` }) {
|
|
|
299
299
|
|
|
300
300
|
start += ` ${ code }`;
|
|
301
301
|
|
|
302
|
-
|
|
302
|
+
let description_parts = [];
|
|
303
303
|
let extra_types = type.replace(/debug|info|success|warning|error/i, ``).replace(/ +/g, ' ').trim();
|
|
304
|
-
if ( extra_types.length > 0 ) {
|
|
304
|
+
if ( extra_types.length > 0 ) { description_parts = extra_types.split(` `); }
|
|
305
305
|
|
|
306
|
-
// Get all the
|
|
307
|
-
let description_parts = [];
|
|
306
|
+
// Get all the main descriptive text
|
|
308
307
|
if ( type.includes(`debug`) ) { description_parts.push( `DEBUG` ); }
|
|
309
308
|
if ( type.includes(`info`) ) { description_parts.push( `INFO` ); }
|
|
310
309
|
if ( type.includes(`success`) ) { description_parts.push( `SUCCESS` ); }
|