@suffolklitlab/alkiln 5.13.2 → 5.13.3-feat-random

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/.github/workflows/github_server.yml +7 -7
  2. package/.github/workflows/playground.yml +2 -1
  3. package/CHANGELOG.md +32 -1
  4. package/action.yml +89 -29
  5. package/action_for_github_server/action.yml +73 -29
  6. package/docs/4_environments.md +243 -0
  7. package/docs/artifacts_folder_name.md +135 -0
  8. package/docs/constrained_random_tests.md +254 -0
  9. package/docs/decisions/006_which_log_level_to_pick_2024_09_5.md +70 -0
  10. package/docs/decisions/007_getting_and_storing_logs.md +179 -0
  11. package/docs/decisions/move_runtime_config_to_sources_folder.md +5 -0
  12. package/docs/decisions/reports_and_logs_2024_08_11.md +161 -0
  13. package/docs/kinds_of_errors.md +26 -0
  14. package/docs/prioritize_constrained_random_tests.md +60 -0
  15. package/lib/docassemble/docassemble_api_interface.js +77 -51
  16. package/lib/docassemble/server_install.js +32 -9
  17. package/lib/docassemble/setup.js +41 -24
  18. package/lib/docassemble/takedown.js +33 -10
  19. package/lib/globals.js +10 -0
  20. package/lib/run_cucumber.js +155 -50
  21. package/lib/scope.js +101 -77
  22. package/lib/setup/TestGenerator.js +1745 -0
  23. package/lib/setup/artifacts.js +20 -0
  24. package/lib/setup/generate.js +42 -0
  25. package/lib/setup/get_Gherkin_AST.js +26 -0
  26. package/lib/setup/validate.js +89 -0
  27. package/lib/steps.js +176 -51
  28. package/lib/utils/Log.js +1132 -0
  29. package/lib/utils/_alkiln-constrained_random_tests_old.js +1727 -0
  30. package/lib/utils/files.js +18 -11
  31. package/lib/utils/get_error_stacks.js +16 -0
  32. package/lib/utils/reports.js +95 -28
  33. package/lib/utils/session_vars.js +102 -73
  34. package/lib/utils/set_sources_paths.js +98 -0
  35. package/lib/utils/time.js +44 -0
  36. package/package.json +16 -8
  37. package/tests/log_codes/check_codes.sh +19 -14
  38. package/tests/log_codes/log_code_expected_instances.txt +69 -9
  39. package/tests/unit_tests/constrained_random_answers/invalid.fixtures.js +155 -0
  40. package/tests/unit_tests/constrained_random_answers/invalid.test.js +233 -0
  41. package/tests/unit_tests/constrained_random_answers/valid.fixtures.js +905 -0
  42. package/tests/unit_tests/constrained_random_answers/valid.test.js +521 -0
  43. package/tests/unit_tests/example_of_chai_test_lifecycle.js +78 -0
  44. package/tests/unit_tests/getAllFields.test.js +5 -0
  45. package/tests/unit_tests/getMatchingRows.test.js +5 -0
  46. package/tests/unit_tests/log.test.js +557 -0
  47. package/tests/unit_tests/normalizeTable.test.js +5 -0
  48. package/lib/utils/log.js +0 -96
  49. /package/docs/{expected_inernal_results.md → expected_internal_results.md} +0 -0
@@ -0,0 +1,905 @@
1
+ /** Actual and expected values for valid constrained random answer tests. */
2
+
3
+ /**
4
+ * TODO:
5
+ * - When we allow multiple Scenarios in one file, test when a Scenario ends in
6
+ * a docstring before another Scenario, especially with comments and blank
7
+ * lines after the docstring and with comment-like text in the docstring
8
+ * itself.
9
+ * */
10
+
11
+
12
+ module.exports = fixts = {};
13
+
14
+ fixts.RANDOM_TABLE_PLACEHOLDER = `@@@ ALK_RANDOM_TABLE_BODY_PLACEHOLDER @@@`;
15
+ fixts.WARNING_PLACEHOLDER = `@@@ ALK_WARNING_PLACEHOLDER @@@`;
16
+ fixts.FLUB_PLACEHOLDER = `@@@ ALK_FLUB_PLACEHOLDER @@@`;
17
+ fixts.FLUB_OPTIONS_REGEX = /(hiccup|snafu|bumble|stumble|fumble|slip|goof|flub)/g;
18
+
19
+ /**
20
+ * ===========
21
+ * Simplest test
22
+ * =========== */
23
+ fixts.simple = simple = {};
24
+ simple.arg =
25
+ `Feature: I generate constrained random tests
26
+
27
+ Scenario: simplest random answers
28
+ Given ALKiln makes 1 constrained random answers test
29
+ Given I start the interview at "test_kickout"
30
+ And ALKiln can get to one of "kickout screen", "success screen" with:
31
+ | var | possible_values |
32
+ | user_choice | correct;; wrong |
33
+ | row_2 | answer1;; answer2;; answer3 |`;
34
+ // ======= ENDS ARG ======= //
35
+ simple.expected =
36
+ `@alkiln @generated @randomized @constraints
37
+ Feature: I generate constrained random tests (generated by ALKiln)
38
+
39
+ @scenario1_of_1
40
+ Scenario: simplest random answers (1 of 1)
41
+ Given I start the interview at "test_kickout"
42
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
43
+ | var | value |
44
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
45
+ // ======= ENDS EXPECTED ======= //
46
+ simple.exact_log_codes = [
47
+ `ALK0236`, `ALK0242`, `ALK0243`,
48
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
49
+ `ALK0259`, `ALK0260`, `ALK0262`,
50
+ ];
51
+ simple.num_Scenarios = 1;
52
+
53
+
54
+ /** TODO: 2 generator Steps in one Scenario */
55
+
56
+ `Feature: I generate constrained random tests
57
+
58
+ Scenario: I fail with 2 generator Steps in 1 Scenario
59
+ Given ALKiln makes 2 constrained answers tests
60
+ Given I start the interview at "form_entrypoint_file_name"
61
+ And I generate 2 constrained random tests that get to "end" when I pick from:
62
+ | var | possible_values |
63
+ | other_var | val1;; val2;; val3 |
64
+ And I generate 1 constrained random test that gets to "end" when I pick from:
65
+ | var | possible_values |
66
+ | user_choice | correct;; wrong |
67
+ `;
68
+
69
+
70
+ /**
71
+ * ===========
72
+ * Minimal required Step text:
73
+ * - Required: "constrain", "ALKiln"
74
+ * - Required: "generat" or "rand" or "mak" or "mad"
75
+ * =========== */
76
+ fixts.generat = {};
77
+ fixts.generat.arg =
78
+ `Feature: I generate constrained random tests
79
+
80
+ Scenario: I use minimal text (constrain and generat)
81
+ Given ALKiln constrain generat 1
82
+ Given I start the interview at "test_kickout"
83
+ And ALKiln can/will get to/reach "end" with:
84
+ | var | possible_values |
85
+ | user_choice | correct;; wrong |`;
86
+ fixts.generat.expected =
87
+ `@alkiln @generated @randomized @constraints
88
+ Feature: I generate constrained random tests (generated by ALKiln)
89
+
90
+ @scenario1_of_1
91
+ Scenario: I use minimal text (constrain and generat) (1 of 1)
92
+ Given I start the interview at "test_kickout"
93
+ And I get to any of the question ids ["end"] with this data:
94
+ | var | value |
95
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
96
+ fixts.generat.exact_log_codes = [
97
+ `ALK0236`, `ALK0242`, `ALK0243`,
98
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
99
+ `ALK0259`, `ALK0260`, `ALK0262`,
100
+ ];
101
+
102
+
103
+ // ===============
104
+ fixts.rand = {};
105
+ fixts.rand.arg =
106
+ `Feature: I generate constrained random tests
107
+
108
+ Scenario: I use minimal text (constrain and rand)
109
+ Given 1 rand constrain ALKiln
110
+ Given I start the interview at "test_kickout"
111
+ And ALKiln can/will get to/reach "end" with:
112
+ | var | possible_values |
113
+ | user_choice | correct;; wrong |`;
114
+ fixts.rand.expected =
115
+ `@alkiln @generated @randomized @constraints
116
+ Feature: I generate constrained random tests (generated by ALKiln)
117
+
118
+ @scenario1_of_1
119
+ Scenario: I use minimal text (constrain and rand) (1 of 1)
120
+ Given I start the interview at "test_kickout"
121
+ And I get to any of the question ids ["end"] with this data:
122
+ | var | value |
123
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
124
+ fixts.rand.exact_log_codes = [
125
+ `ALK0236`, `ALK0242`, `ALK0243`,
126
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
127
+ `ALK0259`, `ALK0260`, `ALK0262`,
128
+ ];
129
+
130
+
131
+ // ===============
132
+ fixts.make = {};
133
+ fixts.make.arg =
134
+ `Feature: I generate constrained random tests
135
+
136
+ Scenario: I use minimal text (constrain and make)
137
+ Given 1 constrain make ALKiln
138
+ Given I start the interview at "test_kickout"
139
+ And ALKiln can/will get to/reach "end" with:
140
+ | var | possible_values |
141
+ | user_choice | correct;; wrong |`;
142
+ fixts.make.expected =
143
+ `@alkiln @generated @randomized @constraints
144
+ Feature: I generate constrained random tests (generated by ALKiln)
145
+
146
+ @scenario1_of_1
147
+ Scenario: I use minimal text (constrain and make) (1 of 1)
148
+ Given I start the interview at "test_kickout"
149
+ And I get to any of the question ids ["end"] with this data:
150
+ | var | value |
151
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
152
+ fixts.make.exact_log_codes = [
153
+ `ALK0236`, `ALK0242`, `ALK0243`,
154
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
155
+ `ALK0259`, `ALK0260`, `ALK0262`,
156
+ ];
157
+
158
+
159
+ // ===============
160
+ fixts.made = {};
161
+ fixts.made.arg =
162
+ `Feature: I generate constrained random tests
163
+
164
+ Scenario: I use minimal text (constrain and made)
165
+ Given 1 made ALKiln constrain
166
+ Given I start the interview at "test_kickout"
167
+ And ALKiln will get to "end" with:
168
+ | var | possible_values |
169
+ | user_choice | correct;; wrong |`;
170
+ fixts.made.expected =
171
+ `@alkiln @generated @randomized @constraints
172
+ Feature: I generate constrained random tests (generated by ALKiln)
173
+
174
+ @scenario1_of_1
175
+ Scenario: I use minimal text (constrain and made) (1 of 1)
176
+ Given I start the interview at "test_kickout"
177
+ And I get to any of the question ids ["end"] with this data:
178
+ | var | value |
179
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
180
+ fixts.made.exact_log_codes = [
181
+ `ALK0236`, `ALK0242`, `ALK0243`,
182
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
183
+ `ALK0259`, `ALK0260`, `ALK0262`,
184
+ ];
185
+
186
+
187
+ /**
188
+ * ===========
189
+ * Complex:
190
+ * - 3 Scenarios with extra "Scenario:" keyword not counted (from 2 generator Scenarios)
191
+ * - Able to generate just one Scenario
192
+ * - Text under Feature
193
+ * - Inconsistent tag spacing
194
+ * - Inconsistent table indentation
195
+ * - A Step after the table
196
+ * - Comments in between Scenarios repeat for each generated Scenario
197
+ * - Comments in the Scenario repeats
198
+ * - Comments in the table
199
+ * - Comment has generator Step text and it doesn't get changed
200
+ * =========== */
201
+ fixts.complex = complex = {};
202
+ complex.arg =
203
+ `Feature: I generate constrained random tests
204
+
205
+ Line 1 after Feature that includes Scenario: keyword. Keep. No comments allowed above this line.
206
+ # Comment 0 keep
207
+
208
+ Scenario: generate a complex Scenario
209
+ # Comment 1 keep and repeat
210
+ Given ALKiln makes 2 constrained answers tests
211
+ Given I start the interview at "test_kickout"
212
+ # Comment 2 keep and repeat
213
+ And ALKiln can get to any of "kickout screen", "success screen" with:
214
+ # Comment 3 keep and repeat
215
+ | var | possible_values |
216
+ # Comment 4 keep and repeat and same with blank line below
217
+
218
+
219
+ | user_choice | correct;; wrong |
220
+ # Comment 5 keep and repeat
221
+ And I wait 0.1 second
222
+ # Comment 6 keep and repeat
223
+ And a docstring containing keywords
224
+ """
225
+ @not_really_a_tag
226
+ Scenario: not really a Scenario
227
+ # Not really a comment. Should test table, but would have to improve test infrastructure for that.
228
+ """
229
+ # Comment 7 keep and repeat. Would belong to different Scenario in future.
230
+ `;
231
+ // // Restore if/when we allow multiple Scenarios
232
+ // Scenario: generate 1st complex Scenario
233
+ // Scenario: generate 2nd complex Scenario
234
+ // # Comment 7 keep
235
+ // Given I start the interview at "interview_2"
236
+ // # And ALKiln will get to "end" with:
237
+ // And ALKiln will get to "end" with:
238
+ // | var | possible_values |
239
+ // | question_1 | opt1;; opt2;; opt3 |
240
+ // | question_2 | optA;; optB;; optC |
241
+ // // ======= ENDS ARG ======= //
242
+ complex.expected =
243
+ `@alkiln @generated @randomized @constraints
244
+ Feature: I generate constrained random tests (generated by ALKiln)
245
+
246
+ Line 1 after Feature that includes Scenario: keyword. Keep. No comments allowed above this line.
247
+ # Comment 0 keep
248
+
249
+ @scenario1_of_2
250
+ Scenario: generate a complex Scenario (1 of 2)
251
+ # Comment 1 keep and repeat
252
+ Given I start the interview at "test_kickout"
253
+ # Comment 2 keep and repeat
254
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
255
+ # Comment 3 keep and repeat
256
+ | var | value |
257
+ # Comment 4 keep and repeat and same with blank line below
258
+
259
+
260
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
261
+ # Comment 5 keep and repeat
262
+ And I wait 0.1 second
263
+ # Comment 6 keep and repeat
264
+ And a docstring containing keywords
265
+ """
266
+ @not_really_a_tag
267
+ Scenario: not really a Scenario
268
+ # Not really a comment. Should test table, but would have to improve test infrastructure for that.
269
+ """
270
+ # Comment 7 keep and repeat. Would belong to different Scenario in future.
271
+
272
+
273
+ @scenario2_of_2
274
+ Scenario: generate a complex Scenario (2 of 2)
275
+ # Comment 1 keep and repeat
276
+ Given I start the interview at "test_kickout"
277
+ # Comment 2 keep and repeat
278
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
279
+ # Comment 3 keep and repeat
280
+ | var | value |
281
+ # Comment 4 keep and repeat and same with blank line below
282
+
283
+
284
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
285
+ # Comment 5 keep and repeat
286
+ And I wait 0.1 second
287
+ # Comment 6 keep and repeat
288
+ And a docstring containing keywords
289
+ """
290
+ @not_really_a_tag
291
+ Scenario: not really a Scenario
292
+ # Not really a comment. Should test table, but would have to improve test infrastructure for that.
293
+ """
294
+ # Comment 7 keep and repeat. Would belong to different Scenario in future.
295
+ `;
296
+ // // Restore if/when we allow multiple Scenarios
297
+ // `@scenario1_of_2
298
+ // @scenario2_of_2`
299
+ // `@scenario1_of_1
300
+ // Scenario: generate complex 2 (1 of 1)
301
+ // # Comment 7 keep
302
+ // Given I start the interview at "interview_2"
303
+ // # And ALKiln will get to "end" with:
304
+ // And I get to any of the question ids ["end"] with this data:
305
+ // | var | value |
306
+ // ${ fixts.RANDOM_TABLE_PLACEHOLDER }
307
+ // `;
308
+ // ======= ENDS EXPECTED ======= //
309
+ complex.num_Scenarios = 2;
310
+ complex.included_log_codes = [
311
+ `ALK0236`, `ALK0242`, `ALK0243`,
312
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`,
313
+ `ALK0259`, `ALK0260`, `ALK0262`,
314
+ ];
315
+
316
+
317
+ /** Existing Feature and Scenario tags are preserved */
318
+ fixts.tags = tags = {};
319
+ tags.arg =
320
+ `@feature_tag1 @feature_tag2
321
+ Feature: I generate constrained random tests
322
+
323
+ @scenario_tag1 @scenario_tag2
324
+ Scenario: simplest random answers
325
+ Given ALKiln makes 1 constrained answers test
326
+ Given I start the interview at "test_kickout"
327
+ And ALKiln will get to any of ["kickout screen", "success screen"] when I pick from these constraints:
328
+ | var | possible_values |
329
+ | user_choice | correct;; wrong |`;
330
+ // ======= ENDS ARG ======= //
331
+ tags.expected =
332
+ `@feature_tag1 @feature_tag2 @alkiln @generated @randomized @constraints
333
+ Feature: I generate constrained random tests (generated by ALKiln)
334
+
335
+ @scenario_tag1 @scenario_tag2 @scenario1_of_1
336
+ Scenario: simplest random answers (1 of 1)
337
+ Given I start the interview at "test_kickout"
338
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
339
+ | var | value |
340
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
341
+ tags.exact_log_codes = [
342
+ `ALK0236`, `ALK0242`, `ALK0243`,
343
+ `ALK0245`, `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
344
+ `ALK0259`, `ALK0260`, `ALK0262`,
345
+ ];
346
+
347
+
348
+ /**
349
+ * WEIRD SPACING
350
+ * - [x] Feature tags have extra spaces and are indented
351
+ * - [x] Scenario tags have extra spaces and are indented
352
+ * - [x] Weird indention, including table
353
+ * - [x] No space between choices (no expected value)
354
+ * */
355
+
356
+ fixts.weird_spacing = {};
357
+
358
+ /** Weird spacing is preserved in all tag types */
359
+ fixts.weird_spacing.tags = {};
360
+ fixts.weird_spacing.tags.arg =
361
+ ` @feature_tag @spacing
362
+ Feature: I generate constrained random tests
363
+
364
+ @scenario_tag @spacing
365
+ Scenario: a scenario
366
+ Given ALKiln makes 1 constrained answers test
367
+ Given I start the interview at "test_kickout"
368
+ And ALKiln will get to any of ["kickout screen", "success screen"] when I pick from these constraints:
369
+ | var | possible_values |
370
+ | user_choice | correct;; wrong |
371
+ `;
372
+ // ======= ENDS ARG ======= //
373
+ fixts.weird_spacing.tags.expected =
374
+ ` @feature_tag @spacing @alkiln @generated @randomized @constraints
375
+ Feature: I generate constrained random tests (generated by ALKiln)
376
+
377
+ @scenario_tag @spacing @scenario1_of_1
378
+ Scenario: a scenario (1 of 1)
379
+ Given I start the interview at "test_kickout"
380
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
381
+ | var | value |
382
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
383
+ `;
384
+ // ======= ENDS EXPECTED ======= //
385
+ fixts.weird_spacing.tags.exact_log_codes = [
386
+ `ALK0236`, `ALK0242`, `ALK0243`,
387
+ `ALK0245`, `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
388
+ `ALK0259`, `ALK0260`, `ALK0262`,
389
+ ];
390
+
391
+
392
+ /** Weird spacing is preserved in all tag types */
393
+ fixts.weird_spacing.tags_line_2 = {};
394
+ fixts.weird_spacing.tags_line_2.arg =
395
+ `
396
+ @line_2_tag
397
+ Feature: I generate constrained random tests
398
+
399
+ Scenario: a scenario
400
+ Given ALKiln makes 1 constrained answers test
401
+ Given I start the interview at "test_kickout"
402
+ And ALKiln will get to any of ["kickout screen", "success screen"] when I pick from these constraints:
403
+ | var | possible_values |
404
+ | user_choice | correct;; wrong |
405
+ `;
406
+ // ======= ENDS ARG ======= //
407
+ fixts.weird_spacing.tags_line_2.expected =
408
+ `
409
+ @line_2_tag @alkiln @generated @randomized @constraints
410
+ Feature: I generate constrained random tests (generated by ALKiln)
411
+
412
+ @scenario1_of_1
413
+ Scenario: a scenario (1 of 1)
414
+ Given I start the interview at "test_kickout"
415
+ And I get to any of the question ids ["kickout screen", "success screen"] with this data:
416
+ | var | value |
417
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
418
+ `;
419
+ // ======= ENDS EXPECTED ======= //
420
+ fixts.weird_spacing.tags_line_2.exact_log_codes = [
421
+ `ALK0236`, `ALK0242`, `ALK0243`,
422
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
423
+ `ALK0259`, `ALK0260`, `ALK0262`,
424
+ ];
425
+
426
+
427
+ /** Various lines are not indented the usual way */
428
+ fixts.weird_spacing.indents = {};
429
+ fixts.weird_spacing.indents.arg =
430
+ ` Feature: I generate constrained random tests
431
+
432
+ Scenario: a scenario
433
+ # Comment line
434
+ Given ALKiln makes 1 constrained answers tests
435
+ Given I start the interview at "interview_2"
436
+ And ALKiln will get to "end" with:
437
+ | var | possible_values |
438
+ | question_1 | opt1;; opt2;; opt3 |
439
+ | question_2 | optA;; optB;; optC |
440
+ `;
441
+ // ======= ENDS ARG ======= //
442
+ fixts.weird_spacing.indents.expected =
443
+ `@alkiln @generated @randomized @constraints
444
+ Feature: I generate constrained random tests (generated by ALKiln)
445
+
446
+ @scenario1_of_1
447
+ Scenario: a scenario (1 of 1)
448
+ # Comment line
449
+ Given I start the interview at "interview_2"
450
+ And I get to any of the question ids ["end"] with this data:
451
+ | var | value |
452
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
453
+ `;
454
+ // ======= ENDS EXPECTED ======= //
455
+ fixts.weird_spacing.indents.exact_log_codes = [
456
+ `ALK0236`, `ALK0242`, `ALK0243`,
457
+ `ALK0245`, `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
458
+ `ALK0259`, `ALK0260`, `ALK0262`,
459
+ ];
460
+
461
+
462
+ /** No spacing between choices (No `.expected` val. Different kind of test) */
463
+ fixts.weird_spacing.choices = {};
464
+ fixts.weird_spacing.choices.arg =
465
+ `Feature: I generate constrained random tests
466
+
467
+ Scenario: a scenario
468
+ Given ALKiln makes 1 constrained answers test
469
+ Given I start the interview at "test_kickout"
470
+ And ALKiln will get to "end" with:
471
+ | var | possible_values |
472
+ | user_choice | correct;;wrong |
473
+ `;
474
+ // ======= ENDS ARG ======= //
475
+ fixts.weird_spacing.choices.expected = `Not appropriate to use use expected value for this test.`;
476
+ // ======= ENDS EXPECTED ======= //
477
+ fixts.weird_spacing.choices.find_1 = /(?:correct)|(?:wrong)/;
478
+ // ======= ENDS OPTIONS TO FIND ONE OF ======= //
479
+ fixts.weird_spacing.choices.exact_log_codes = [
480
+ `ALK0236`, `ALK0242`, `ALK0243`,
481
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
482
+ `ALK0259`, `ALK0260`, `ALK0262`,
483
+ ];
484
+
485
+
486
+ /**
487
+ * ===========
488
+ * Column shenanigans
489
+ * =========== */
490
+ fixts.columns_3 = {};
491
+ fixts.columns_3.arg =
492
+ `Feature: I generate constrained random tests
493
+
494
+ Scenario: 3 columns
495
+ Given ALKiln makes 1 constrained answers test
496
+ Given I start the interview at "file.yml"
497
+ And ALKiln will get to "end" with:
498
+ | var | possible_values | column 3 becomes "trigger" |
499
+ | users[0].has_bear | True | users[0].name.first |`;
500
+ // ======= ENDS ARG ======= //
501
+ fixts.columns_3.expected =
502
+ `@alkiln @generated @randomized @constraints
503
+ Feature: I generate constrained random tests (generated by ALKiln)
504
+
505
+ @scenario1_of_1
506
+ Scenario: 3 columns (1 of 1)
507
+ Given I start the interview at "file.yml"
508
+ And I get to any of the question ids ["end"] with this data:
509
+ | var | value | trigger |
510
+ | users[0].has_bear | True | users[0].name.first |`;
511
+ // ======= ENDS EXPECTED ======= //
512
+ fixts.columns_3.exact_log_codes = [
513
+ `ALK0236`, `ALK0242`, `ALK0243`,
514
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
515
+ `ALK0259`, `ALK0260`, `ALK0262`,
516
+ ];
517
+
518
+
519
+ fixts.columns_4 = {};
520
+ fixts.columns_4.arg =
521
+ `Feature: I generate constrained random tests
522
+
523
+ Scenario: 4 columns
524
+ Given ALKiln makes 1 constrained answers test
525
+ Given I start the interview at "file.yml"
526
+ And ALKiln will get to "end" with:
527
+ | var | possible_values | column 3 becomes "target" | column 4 is ignored |
528
+ | users[0].has_bear | True | users[0].name.first | is_ignored |`;
529
+ // ======= ENDS ARG ======= //
530
+ fixts.columns_4.expected =
531
+ `@alkiln @generated @randomized @constraints
532
+ Feature: I generate constrained random tests (generated by ALKiln)
533
+
534
+ @scenario1_of_1
535
+ Scenario: 4 columns (1 of 1)
536
+ Given I start the interview at "file.yml"
537
+ And I get to any of the question ids ["end"] with this data:
538
+ | var | value | trigger |
539
+ | users[0].has_bear | True | users[0].name.first |`;
540
+ // ======= ENDS EXPECTED ======= //
541
+ fixts.columns_4.exact_log_codes = [
542
+ `ALK0236`, `ALK0242`, `ALK0243`,
543
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
544
+ `ALK0259`, `ALK0260`, `ALK0262`,
545
+ ];
546
+
547
+
548
+ /**
549
+ * ===========
550
+ * Row shenanigans
551
+ * =========== */
552
+ fixts.rows_comments = {};
553
+ fixts.rows_comments.arg =
554
+ `Feature: I generate constrained random tests
555
+
556
+ Scenario: Keeps comments before and in table
557
+ Given ALKiln makes 1 constrained answers test
558
+ Given I start the interview at "file.yml"
559
+ And ALKiln will get to "end" with:
560
+ # Comment 1 keep and repeat
561
+ | var | possible_values |
562
+ # Comment 2 keep and repeat
563
+ | user_choice | correct;; wrong |`;
564
+ fixts.rows_comments.expected =
565
+ `@alkiln @generated @randomized @constraints
566
+ Feature: I generate constrained random tests (generated by ALKiln)
567
+
568
+ @scenario1_of_1
569
+ Scenario: Keeps comments before and in table (1 of 1)
570
+ Given I start the interview at "file.yml"
571
+ And I get to any of the question ids ["end"] with this data:
572
+ # Comment 1 keep and repeat
573
+ | var | value |
574
+ # Comment 2 keep and repeat
575
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
576
+ fixts.rows_comments.exact_log_codes = [
577
+ `ALK0236`, `ALK0242`, `ALK0243`,
578
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
579
+ `ALK0259`, `ALK0260`, `ALK0262`,
580
+ ];
581
+
582
+
583
+ fixts.comment_after_last_row = {};
584
+ fixts.comment_after_last_row.arg =
585
+ `Feature: I generate constrained random tests
586
+
587
+ Scenario: Keeps comment after table
588
+ Given ALKiln makes 2 constrained answers tests
589
+ Given I start the interview at "file.yml"
590
+ And ALKiln will get to "end" with:
591
+ | var | possible_values |
592
+ | user_choice | correct;; wrong |
593
+ # Comment 1 keep and repeat
594
+
595
+ # Comment 2 keep and repeat`;
596
+ fixts.comment_after_last_row.expected =
597
+ `@alkiln @generated @randomized @constraints
598
+ Feature: I generate constrained random tests (generated by ALKiln)
599
+
600
+ @scenario1_of_2
601
+ Scenario: Keeps comment after table (1 of 2)
602
+ Given I start the interview at "file.yml"
603
+ And I get to any of the question ids ["end"] with this data:
604
+ | var | value |
605
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
606
+ # Comment 1 keep and repeat
607
+
608
+ # Comment 2 keep and repeat
609
+
610
+ @scenario2_of_2
611
+ Scenario: Keeps comment after table (2 of 2)
612
+ Given I start the interview at "file.yml"
613
+ And I get to any of the question ids ["end"] with this data:
614
+ | var | value |
615
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
616
+ # Comment 1 keep and repeat
617
+
618
+ # Comment 2 keep and repeat`;
619
+ fixts.comment_after_last_row.included_log_codes = [
620
+ `ALK0236`, `ALK0242`, `ALK0243`,
621
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`,
622
+ `ALK0259`, `ALK0260`, `ALK0262`,
623
+ ];
624
+
625
+
626
+ fixts.rows_empty = {};
627
+ fixts.rows_empty.arg =
628
+ `Feature: I generate constrained random tests
629
+
630
+ Scenario: Keeps empty rows before and in table
631
+ Given ALKiln makes 1 constrained answers test
632
+ Given I start the interview at "file.yml"
633
+ And ALKiln will get to "end" with:
634
+
635
+ | var | possible_values |
636
+
637
+ | user_choice | correct;; wrong |`;
638
+ fixts.rows_empty.expected =
639
+ `@alkiln @generated @randomized @constraints
640
+ Feature: I generate constrained random tests (generated by ALKiln)
641
+
642
+ @scenario1_of_1
643
+ Scenario: Keeps empty rows before and in table (1 of 1)
644
+ Given I start the interview at "file.yml"
645
+ And I get to any of the question ids ["end"] with this data:
646
+
647
+ | var | value |
648
+
649
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
650
+ fixts.rows_empty.exact_log_codes = [
651
+ `ALK0236`, `ALK0242`, `ALK0243`,
652
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
653
+ `ALK0259`, `ALK0260`, `ALK0262`,
654
+ ];
655
+
656
+
657
+ /**
658
+ * ===========
659
+ * Warnings
660
+ * Discuss: add a "strict mode" to fail on warnings?
661
+ * =========== */
662
+ fixts.warnings = {};
663
+
664
+
665
+ fixts.warnings.no_number = {};
666
+ fixts.warnings.no_number.arg =
667
+ `Feature: I generate constrained random tests
668
+
669
+ Scenario: I'm warned with missing number in random answers generator
670
+ Given ALKiln makes not a number constrained random tests
671
+ Given I start the interview at "test_kickout"
672
+ And ALKiln gets to "end" when I pick from:
673
+ | var | possible_values |
674
+ | user_choice | correct;; wrong |`;
675
+ fixts.warnings.no_number.expected =
676
+ `@alkiln @generated @randomized @constraints
677
+ Feature: I generate constrained random tests (generated by ALKiln)
678
+
679
+ @alkiln_extra_intro @alkiln_generator_warning
680
+ Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
681
+ Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
682
+ """
683
+ ━━━━━ Generator file warning or warnings ━━━━━
684
+ 🔎 ALK0238 ${ fixts.WARNING_PLACEHOLDER }
685
+ ━━━━━
686
+ """
687
+
688
+ @scenario1_of_1
689
+ Scenario: I'm warned with missing number in random answers generator (1 of 1)
690
+ Given I start the interview at "test_kickout"
691
+ And I get to any of the question ids ["end"] with this data:
692
+ | var | value |
693
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
694
+ fixts.warnings.no_number.exact_log_codes = [
695
+ `ALK0236`, `ALK0238`, `ALK0242`, `ALK0243`,
696
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
697
+ `ALK0259`, `ALK0260`, `ALK0262`,
698
+ ];
699
+
700
+
701
+ fixts.warnings.multiple_numbers = {};
702
+ fixts.warnings.multiple_numbers.arg =
703
+ `Feature: I generate constrained random tests
704
+
705
+ Scenario: I'm warned with too many numbers in random answers tests
706
+ Given ALKiln makes 4 1 constrained random tests
707
+ Given I start the interview at "test_kickout"
708
+ And ALKiln gets to "end" when I pick from:
709
+ | var | possible_values |
710
+ | user_choice | correct;; wrong |`;
711
+ fixts.warnings.multiple_numbers.expected =
712
+ `@alkiln @generated @randomized @constraints
713
+ Feature: I generate constrained random tests (generated by ALKiln)
714
+
715
+ @alkiln_extra_intro @alkiln_generator_warning
716
+ Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
717
+ Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
718
+ """
719
+ ━━━━━ Generator file warning or warnings ━━━━━
720
+ 🔎 ALK0239 ${ fixts.WARNING_PLACEHOLDER }
721
+ ━━━━━
722
+ """
723
+
724
+ @scenario1_of_1
725
+ Scenario: I'm warned with too many numbers in random answers tests (1 of 1)
726
+ Given I start the interview at "test_kickout"
727
+ And I get to any of the question ids ["end"] with this data:
728
+ | var | value |
729
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
730
+ fixts.warnings.multiple_numbers.exact_log_codes = [
731
+ `ALK0236`, `ALK0239`, `ALK0242`, `ALK0243`,
732
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
733
+ `ALK0259`, `ALK0260`, `ALK0262`,
734
+ ];
735
+
736
+
737
+ fixts.warnings.multiple_number_Steps = {};
738
+ fixts.warnings.multiple_number_Steps.arg =
739
+ `Feature: I generate constrained random tests
740
+
741
+ Scenario: I'm warned with too many number Steps in random answers tests
742
+ Given ALKiln makes 7 constrained random tests
743
+ Given ALKiln makes 1 constrained random tests
744
+ Given I start the interview at "test_kickout"
745
+ And ALKiln gets to "end" when I pick from:
746
+ | var | possible_values |
747
+ | user_choice | correct;; wrong |`;
748
+ fixts.warnings.multiple_number_Steps.expected =
749
+ `@alkiln @generated @randomized @constraints
750
+ Feature: I generate constrained random tests (generated by ALKiln)
751
+
752
+ @alkiln_extra_intro @alkiln_generator_warning
753
+ Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
754
+ Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
755
+ """
756
+ ━━━━━ Generator file warning or warnings ━━━━━
757
+ 🔎 ALK0237 ${ fixts.WARNING_PLACEHOLDER }
758
+ ━━━━━
759
+ """
760
+
761
+ @scenario1_of_1
762
+ Scenario: I'm warned with too many number Steps in random answers tests (1 of 1)
763
+ Given I start the interview at "test_kickout"
764
+ And I get to any of the question ids ["end"] with this data:
765
+ | var | value |
766
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
767
+ fixts.warnings.multiple_number_Steps.exact_log_codes = [
768
+ `ALK0236`, `ALK0237`, `ALK0242`, `ALK0243`,
769
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
770
+ `ALK0259`, `ALK0260`, `ALK0262`,
771
+ ];
772
+
773
+
774
+ fixts.warnings.multiple_number_Steps_and_numbers = {};
775
+ fixts.warnings.multiple_number_Steps_and_numbers.arg =
776
+ `Feature: I generate constrained random tests
777
+
778
+ Scenario: I'm warned with too many numbers in too many numbers Steps in random answers tests
779
+ Given ALKiln makes 3 2 constrained random tests
780
+ Given ALKiln makes 4 1 constrained random tests
781
+ Given I start the interview at "test_kickout"
782
+ And ALKiln gets to "end" when I pick from:
783
+ | var | possible_values |
784
+ | user_choice | correct;; wrong |`;
785
+ fixts.warnings.multiple_number_Steps_and_numbers.expected =
786
+ `@alkiln @generated @randomized @constraints
787
+ Feature: I generate constrained random tests (generated by ALKiln)
788
+
789
+ @alkiln_extra_intro @alkiln_generator_warning
790
+ Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
791
+ Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
792
+ """
793
+ ━━━━━ Generator file warning or warnings ━━━━━
794
+ 🔎 ALK0239 ${ fixts.WARNING_PLACEHOLDER }
795
+ 🔎 ALK0237 ${ fixts.WARNING_PLACEHOLDER }
796
+ 🔎 ALK0239 ${ fixts.WARNING_PLACEHOLDER }
797
+ ━━━━━
798
+ """
799
+
800
+ @scenario1_of_1
801
+ Scenario: I'm warned with too many numbers in too many numbers Steps in random answers tests (1 of 1)
802
+ Given I start the interview at "test_kickout"
803
+ And I get to any of the question ids ["end"] with this data:
804
+ | var | value |
805
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
806
+ fixts.warnings.multiple_number_Steps_and_numbers.exact_log_codes = [
807
+ `ALK0236`, `ALK0239`, `ALK0237`, `ALK0239`, `ALK0242`, `ALK0243`,
808
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0245`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`, `ALK0245`,
809
+ `ALK0259`, `ALK0260`, `ALK0262`,
810
+ ];
811
+
812
+
813
+ fixts.warnings.fewer_unique_than_requested = {};
814
+ fixts.warnings.fewer_unique_than_requested.arg =
815
+ `Feature: I generate constrained random tests
816
+
817
+ Scenario: I'm warned when I ask for too many unique tests
818
+ Given ALKiln makes 3 constrained random answers tests
819
+ Given I start the interview at "test_kickout"
820
+ And ALKiln can get to "end" when I pick from:
821
+ | var | possible_values |
822
+ | user_choice | correct;; wrong |`;
823
+ fixts.warnings.fewer_unique_than_requested.expected =
824
+ `@alkiln @generated @randomized @constraints
825
+ Feature: I generate constrained random tests (generated by ALKiln)
826
+
827
+ @alkiln_extra_intro @alkiln_generator_warning
828
+ Scenario: ALKiln humbly begs the author's forgiveness. The next tests might be flawed.
829
+ Then ALKiln warns the author about a generator ${ fixts.FLUB_PLACEHOLDER }:
830
+ """
831
+ ━━━━━ Generator file warning or warnings ━━━━━
832
+ 🔎 ALK0258 ${ fixts.WARNING_PLACEHOLDER }
833
+ ━━━━━
834
+ """
835
+
836
+ @scenario1_of_2
837
+ Scenario: I'm warned when I ask for too many unique tests (1 of 2)
838
+ Given I start the interview at "test_kickout"
839
+ And I get to any of the question ids ["end"] with this data:
840
+ | var | value |
841
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
842
+
843
+ @scenario2_of_2
844
+ Scenario: I'm warned when I ask for too many unique tests (2 of 2)
845
+ Given I start the interview at "test_kickout"
846
+ And I get to any of the question ids ["end"] with this data:
847
+ | var | value |
848
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
849
+ fixts.warnings.fewer_unique_than_requested.find_1_of = [/\| correct \|/, /\| wrong \|/];
850
+ // The third one is the warning Scenario
851
+ fixts.warnings.fewer_unique_than_requested.num_Scenarios = 3;
852
+ fixts.warnings.fewer_unique_than_requested.repeat = { code: `ALK0255`, count: 10 };
853
+ fixts.warnings.fewer_unique_than_requested.included_log_codes = [
854
+ `ALK0236`, `ALK0242`, `ALK0243`,
855
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0253`, `ALK0255`, `ALK0257`,
856
+ `ALK0258`, `ALK0259`, `ALK0260`, `ALK0262`,
857
+ ];
858
+
859
+
860
+ /**
861
+ * ===========
862
+ * Silent warnings
863
+ *
864
+ * Discuss: Should we have author warnings that are only in the debug logs?
865
+ * These would be warnings for tests that aren't invalid, but may (or may
866
+ * not) have unexpected behavior. Should they have a different icon and
867
+ * keyword?
868
+ *
869
+ * Discuss: Would a "strict mode" to fail on this warning?
870
+ * =========== */
871
+
872
+ fixts.empty_string_value = {};
873
+ fixts.empty_string_value.arg =
874
+ `Feature: I generate constrained random tests
875
+
876
+ Scenario: An empty string as a value causes silent log warnings
877
+ Given ALKiln makes 2 constrained answers tests
878
+ Given I start the interview at "test_kickout"
879
+ And ALKiln will get to "end" with:
880
+ | var | possible_values |
881
+ | user_choice | correct;; |`;
882
+ fixts.empty_string_value.expected =
883
+ `@alkiln @generated @randomized @constraints
884
+ Feature: I generate constrained random tests (generated by ALKiln)
885
+
886
+ @scenario1_of_2
887
+ Scenario: An empty string as a value causes silent log warnings (1 of 2)
888
+ Given I start the interview at "test_kickout"
889
+ And I get to any of the question ids ["end"] with this data:
890
+ | var | value |
891
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }
892
+
893
+ @scenario2_of_2
894
+ Scenario: An empty string as a value causes silent log warnings (2 of 2)
895
+ Given I start the interview at "test_kickout"
896
+ And I get to any of the question ids ["end"] with this data:
897
+ | var | value |
898
+ ${ fixts.RANDOM_TABLE_PLACEHOLDER }`;
899
+ fixts.empty_string_value.find_1_of = [/\| correct \|/, /\| \|/];
900
+ fixts.empty_string_value.num_Scenarios = 2;
901
+ fixts.empty_string_value.included_log_codes = [
902
+ `ALK0236`, `ALK0242`, `ALK0243`,
903
+ `ALK0244`, `ALK0247`, `ALK0248`, `ALK0245`, `ALK0252`, `ALK0253`, `ALK0255`, `ALK0257`,
904
+ `ALK0259`, `ALK0260`, `ALK0262`,
905
+ ];