@suffolklitlab/alkiln 5.13.2 → 5.13.3-feat-random-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/github_server.yml +10 -8
- package/.github/workflows/playground.yml +5 -2
- package/.github/workflows/unit_tests.yml +1 -1
- package/CHANGELOG.md +32 -1
- package/_alkiln_meta/shift_logs.js +75 -0
- package/action.yml +89 -31
- package/action_for_github_server/action.yml +65 -29
- package/docs/4_environments.md +243 -0
- package/docs/artifacts_folder_name.md +135 -0
- package/docs/constrained_random_tests.md +254 -0
- package/docs/decisions/006_which_log_level_to_pick_2024_09_5.md +70 -0
- package/docs/decisions/007_getting_and_storing_logs.md +179 -0
- package/docs/decisions/move_runtime_config_to_sources_folder.md +5 -0
- package/docs/decisions/reports_and_logs_2024_08_11.md +161 -0
- package/docs/kinds_of_errors.md +26 -0
- package/docs/prioritize_constrained_random_tests.md +60 -0
- package/lib/docassemble/docassemble_api_interface.js +77 -51
- package/lib/docassemble/server_install.js +32 -9
- package/lib/docassemble/setup.js +40 -25
- package/lib/docassemble/takedown.js +33 -10
- package/lib/globals.js +10 -0
- package/lib/run_cucumber.js +141 -69
- package/lib/scope.js +101 -77
- package/lib/setup/TestGenerator.js +1755 -0
- package/lib/setup/artifacts.js +20 -0
- package/lib/setup/generate.js +42 -0
- package/lib/setup/get_Gherkin_AST.js +26 -0
- package/lib/setup/validate.js +89 -0
- package/lib/steps.js +132 -54
- package/lib/utils/Log.js +1145 -0
- package/lib/utils/_alkiln-constrained_random_tests_old.js +1776 -0
- package/lib/utils/files.js +18 -11
- package/lib/utils/get_error_stacks.js +16 -0
- package/lib/utils/reports.js +100 -34
- package/lib/utils/session_vars.js +107 -73
- package/lib/utils/set_sources_paths.js +98 -0
- package/lib/utils/time.js +44 -0
- package/package.json +18 -7
- package/tests/log_codes/check_codes.sh +18 -13
- package/tests/log_codes/log_code_expected_instances.txt +91 -9
- package/tests/unit_tests/constrained_random_answers/invalid.fixtures.js +177 -0
- package/tests/unit_tests/constrained_random_answers/invalid.test.js +251 -0
- package/tests/unit_tests/constrained_random_answers/valid.fixtures.js +936 -0
- package/tests/unit_tests/constrained_random_answers/valid.test.js +541 -0
- package/tests/unit_tests/example_of_chai_test_lifecycle.js +78 -0
- package/tests/unit_tests/getAllFields.test.js +5 -0
- package/tests/unit_tests/getMatchingRows.test.js +5 -0
- package/tests/unit_tests/log.test.js +557 -0
- package/tests/unit_tests/normalizeTable.test.js +5 -0
- package/lib/utils/log.js +0 -96
- /package/docs/{expected_inernal_results.md → expected_internal_results.md} +0 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
const session_vars = require(`../utils/session_vars`);
|
|
4
|
+
const files = require(`../utils/files`);
|
|
5
|
+
const Log = require(`../setup/artifacts.js`);
|
|
6
|
+
|
|
7
|
+
/** Reset and refresh artifacts path name */
|
|
8
|
+
|
|
9
|
+
session_vars.delete_artifacts_path_name();
|
|
10
|
+
|
|
11
|
+
// process.argv is a list of strings of commands and args.
|
|
12
|
+
// If using `npm run cucumber` (as opposed to the bin commands),
|
|
13
|
+
// use `--` before `--sources=./foo`
|
|
14
|
+
const argv = require(`minimist`)( process.argv.slice(2) );
|
|
15
|
+
const artifacts_path = files.make_artifacts_folder( argv.path );
|
|
16
|
+
|
|
17
|
+
session_vars.save_artifacts_path_name( artifacts_path );
|
|
18
|
+
|
|
19
|
+
// Leave the config Project name value as it is. Local developers find it
|
|
20
|
+
// useful to re-use that value.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
const Log = require(`../utils/Log`);
|
|
4
|
+
const set_sources_paths = require(`../utils/set_sources_paths`);
|
|
5
|
+
const TestGenerator = require(`./TestGenerator.js`);
|
|
6
|
+
|
|
7
|
+
// process.argv is a list of strings of commands and args.
|
|
8
|
+
// If using `npm run cucumber` (as opposed to the bin commands),
|
|
9
|
+
// use `--` before `--sources=./foo`
|
|
10
|
+
const argv = require(`minimist`)( process.argv.slice(2) );
|
|
11
|
+
const log = new Log({ path: argv.path, context: `generating files` });
|
|
12
|
+
// Run script
|
|
13
|
+
generate_constrained_random_test_files();
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
function generate_constrained_random_test_files() {
|
|
17
|
+
/** Executable that generates constrained random answers tests and throws
|
|
18
|
+
* errors.
|
|
19
|
+
*
|
|
20
|
+
* Takes no arguments - uses command line arguments instead.
|
|
21
|
+
* */
|
|
22
|
+
let all_errors = [];
|
|
23
|
+
|
|
24
|
+
// Every file that's executed separately needs to figure out the location of
|
|
25
|
+
// the sources folders.
|
|
26
|
+
let sources_paths = set_sources_paths({ log });
|
|
27
|
+
let generator = new TestGenerator();
|
|
28
|
+
for ( let sources_path of sources_paths ) {
|
|
29
|
+
let { errors: generating_errors } = generator.create({
|
|
30
|
+
sources_path,
|
|
31
|
+
logger: log
|
|
32
|
+
});
|
|
33
|
+
all_errors.push( ...generating_errors );
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if ( all_errors.length > 0 ) {
|
|
37
|
+
log.throw({ code: `ALK0272`, context: `generating files`,
|
|
38
|
+
error: new Error( `ALKiln got error(s) when it tried to generate constrained random answers tests. See warnings and errors above.` )
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
const Gherkin = require(`@cucumber/gherkin`);
|
|
2
|
+
const Messages = require(`@cucumber/messages`);
|
|
3
|
+
|
|
4
|
+
module.exports = function get_Gherkin_AST({ file_text }) {
|
|
5
|
+
/** Return Gherkin parser's abstract syntax tree of the given generator file
|
|
6
|
+
*
|
|
7
|
+
* @param {object} obj - Named arguments
|
|
8
|
+
* @param {string} file_text - The generator file's text contents
|
|
9
|
+
*
|
|
10
|
+
* @returns {object} - Gherkin parser's AST of the given generator file
|
|
11
|
+
* */
|
|
12
|
+
let AST = null;
|
|
13
|
+
let parse_errors = [];
|
|
14
|
+
try {
|
|
15
|
+
let uuidFn = Messages.IdGenerator.uuid();
|
|
16
|
+
let builder = new Gherkin.AstBuilder(uuidFn);
|
|
17
|
+
let matcher = new Gherkin.GherkinClassicTokenMatcher(); // or Gherkin.GherkinInMarkdownTokenMatcher()
|
|
18
|
+
|
|
19
|
+
let parser = new Gherkin.Parser(builder, matcher);
|
|
20
|
+
AST = parser.parse( file_text );
|
|
21
|
+
} catch ( gherkin_error ) {
|
|
22
|
+
parse_errors.push( gherkin_error );
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return { AST, errors: parse_errors };
|
|
26
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
const fs = require(`fs`);
|
|
4
|
+
const fast_glob = require(`fast-glob`);
|
|
5
|
+
|
|
6
|
+
const globals = require(`../globals`);
|
|
7
|
+
const Log = require(`../utils/Log`);
|
|
8
|
+
const set_sources_paths = require(`../utils/set_sources_paths`);
|
|
9
|
+
const get_Gherkin_AST = require(`./get_Gherkin_AST`);
|
|
10
|
+
const get_error_stacks = require(`../utils/get_error_stacks`);
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* TODO: Document that generator files need to have valid Gherkin syntax
|
|
15
|
+
* TODO: Document how to read Gherkin syntax errors. E.g.
|
|
16
|
+
* (3:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'Scenario: bar'
|
|
17
|
+
* (6:1): expected: #EOF, #TableRow, #DocStringSeparator, #StepLine, #TagLine, #ExamplesLine, #ScenarioLine, #RuleLine, #Comment, #Empty, got 'Feature: zoo'
|
|
18
|
+
* */
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
// process.argv is a list of strings of commands and args.
|
|
22
|
+
// If using `npm run cucumber` (as opposed to the bin commands),
|
|
23
|
+
// use `--` before `--sources=./foo`
|
|
24
|
+
const argv = require(`minimist`)( process.argv.slice(2) );
|
|
25
|
+
const log = new Log({ path: argv.path, context: `validating files` });
|
|
26
|
+
// Run script
|
|
27
|
+
validate_Gherkin_files();
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
function validate_Gherkin_files() {
|
|
31
|
+
/** Accumulates syntax errors for all `.feature`` test files, including
|
|
32
|
+
* generator files. If there are any, it logs those errors and then finally
|
|
33
|
+
* throws an error. Uses the values in command line arguments.
|
|
34
|
+
* */
|
|
35
|
+
|
|
36
|
+
let sources_paths = set_sources_paths({ log });
|
|
37
|
+
|
|
38
|
+
let all_errors = [];
|
|
39
|
+
for ( let sources_path of sources_paths ) {
|
|
40
|
+
let { errors: files_errors } = parse_all_feature_files_in({ sources_path });
|
|
41
|
+
all_errors.push( ...files_errors );
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Discuss: If an author has only some files with syntax errors, should we try
|
|
45
|
+
// to still run the tests for the files that had valid syntax?
|
|
46
|
+
if ( all_errors.length > 0 ) {
|
|
47
|
+
log.throw({ code: `ALK0230`, context: `validating files`,
|
|
48
|
+
error: new Error( `ALKiln ran into Gherkin syntax error(s) when parsing your test feature files. See warnings and errors above.` )
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
function parse_all_feature_files_in({ sources_path }) {
|
|
55
|
+
/** Returns accumulated list of Gherkin errors from all files in the
|
|
56
|
+
* docassemble "sources" folder (or an empty list).
|
|
57
|
+
*
|
|
58
|
+
* @param {object} obj - Named arguments
|
|
59
|
+
* @param {object} obj.sources_path - Path to the docassemble "sources" folder
|
|
60
|
+
*
|
|
61
|
+
* @returns {{errors: [Error]}} Obj with an `error` prop that is a list of
|
|
62
|
+
* Error instances. Follows the format of other functions that return
|
|
63
|
+
* accumulated errors.
|
|
64
|
+
* */
|
|
65
|
+
let all_AST_errors = [];
|
|
66
|
+
|
|
67
|
+
let feature_file_paths = fast_glob.sync(
|
|
68
|
+
`${ sources_path }/**/*.feature`,
|
|
69
|
+
// These files were generated last time and will get deleted before the run
|
|
70
|
+
{ ignore: [`${ sources_path }/${ globals.generated_files_folder_name }/*`] }
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
for ( let feature_path of feature_file_paths ) {
|
|
74
|
+
|
|
75
|
+
let file_text = fs.readFileSync( feature_path, { encoding: `utf8` });
|
|
76
|
+
let { AST, errors: these_AST_errors } = get_Gherkin_AST({ file_text });
|
|
77
|
+
|
|
78
|
+
if ( these_AST_errors.length > 0 ) {
|
|
79
|
+
all_AST_errors.push( ...these_AST_errors );
|
|
80
|
+
log.warn({ code: `ALK0276`, context: `validating files`, },
|
|
81
|
+
`There is a Gherkin syntax error(s) in "${ feature_path }"`,
|
|
82
|
+
...get_error_stacks({ errors: these_AST_errors })
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
} // ends for each file path
|
|
87
|
+
|
|
88
|
+
return { errors: all_AST_errors };
|
|
89
|
+
}
|
package/lib/steps.js
CHANGED
|
@@ -5,6 +5,8 @@ const {
|
|
|
5
5
|
Given,
|
|
6
6
|
After,
|
|
7
7
|
Before,
|
|
8
|
+
BeforeStep,
|
|
9
|
+
AfterStep,
|
|
8
10
|
BeforeAll,
|
|
9
11
|
AfterAll,
|
|
10
12
|
setDefaultTimeout //,
|
|
@@ -18,7 +20,7 @@ const { v4: uuidv4 } = require('uuid');
|
|
|
18
20
|
|
|
19
21
|
// Ours
|
|
20
22
|
const scope = require('./scope');
|
|
21
|
-
const
|
|
23
|
+
const Log = require(`./utils/Log`);
|
|
22
24
|
const session_vars = require('./utils/session_vars');
|
|
23
25
|
const files = require('./utils/files' );
|
|
24
26
|
const reports = require(`./utils/reports`);
|
|
@@ -82,7 +84,9 @@ let click_with = {
|
|
|
82
84
|
pc: 'click',
|
|
83
85
|
};
|
|
84
86
|
|
|
85
|
-
|
|
87
|
+
let log = null;
|
|
88
|
+
|
|
89
|
+
BeforeAll(async function() {
|
|
86
90
|
scope.showif_timeout = 600;
|
|
87
91
|
// TODO: Do these really need to be maps? As sets, they may make the output
|
|
88
92
|
// more predictable to the user, but they add complexity.
|
|
@@ -95,29 +99,41 @@ BeforeAll(async () => {
|
|
|
95
99
|
// Store path names for files and docs created for the tests
|
|
96
100
|
scope.paths = {};
|
|
97
101
|
|
|
98
|
-
//
|
|
99
|
-
//
|
|
102
|
+
// // Make the folder to store all the files for this run of the tests
|
|
103
|
+
// scope.paths.artifacts = files.make_artifacts_folder();
|
|
104
|
+
// // Save its name in a file. `session_vars` can get it back later.
|
|
105
|
+
// // The value we are saving is used by at least run_cucumber.js.
|
|
106
|
+
|
|
107
|
+
// A little indirect, but the artifacts path should always have already been
|
|
108
|
+
// created by now, either by `run_cucumber.js` or the relevant test
|
|
109
|
+
|
|
110
|
+
// // TODO: Is this causing the GitHub tests artifacts trouble? Because
|
|
111
|
+
// // `.get_artifacts_path_name()` is messing up somehow? That shouldn't be a
|
|
112
|
+
// // problem. That's the same method `Log` uses. Why does runtime_config not
|
|
113
|
+
// // have a value here?
|
|
114
|
+
// scope.paths.artifacts = session_vars.get_artifacts_path_name();
|
|
115
|
+
// log = new Log({ path: scope.paths.artifacts, context: `steps` });
|
|
116
|
+
|
|
117
|
+
console.log(`\n\n`, `scope.paths.artifacts:`, scope.paths.artifacts, `\n\n`);
|
|
118
|
+
// New attempt avoids passing in a path. For the record, this shouldn't help.
|
|
119
|
+
log = new Log({ context: `steps` });
|
|
120
|
+
scope.set_log({ log_obj: log });
|
|
100
121
|
|
|
101
|
-
|
|
102
|
-
scope.paths.artifacts = files.make_artifacts_folder();
|
|
103
|
-
// Save its name in a file. `session_vars` can get it back later.
|
|
104
|
-
// The value we are saving is used by at least run_cucumber.js.
|
|
105
|
-
session_vars.save_artifacts_path_name( scope.paths.artifacts );
|
|
122
|
+
scope.paths.artifacts = session_vars.get_artifacts_path_name();
|
|
106
123
|
|
|
107
|
-
scope.paths.
|
|
108
|
-
scope.paths.
|
|
109
|
-
scope.paths.
|
|
110
|
-
|
|
124
|
+
scope.paths.debug_log = `${ scope.paths.artifacts }/${ log.debug_log_filename }`;
|
|
125
|
+
scope.paths.report_log = `${ scope.paths.artifacts }/${ log.report_log_filename }`;
|
|
126
|
+
scope.paths.report = `${ scope.paths.artifacts }/${ log.report_filename }`;
|
|
127
|
+
scope.paths.debug_of_unexpected = `${ scope.paths.artifacts }/${ log.temp_unexpected_debug_log_filename }`;
|
|
128
|
+
scope.paths.report_of_unexpected = `${ scope.paths.artifacts }/${ log.unexpected_filename }`;
|
|
111
129
|
|
|
112
130
|
reports.create( scope );
|
|
113
131
|
});
|
|
114
132
|
|
|
115
133
|
Before(async (scenario) => {
|
|
116
134
|
|
|
117
|
-
//
|
|
118
|
-
|
|
119
|
-
console.log('');
|
|
120
|
-
process.stdout.write(`Scenario: ${ scenario.pickle.name }: `);
|
|
135
|
+
// Start the running "progress bar" for the Scenario
|
|
136
|
+
log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `);
|
|
121
137
|
|
|
122
138
|
// Create browser, which can't be created in BeforeAll() where .driver doesn't exist for some reason
|
|
123
139
|
if (!scope.browser) {
|
|
@@ -175,6 +191,14 @@ Before(async (scenario) => {
|
|
|
175
191
|
|
|
176
192
|
// Add a check for an error page before each step? After each step?
|
|
177
193
|
|
|
194
|
+
BeforeStep(async function ( step ) {
|
|
195
|
+
// Discuss: In the report, add a way for the reports to add "✔" or "✖" to the
|
|
196
|
+
// start of a Step after it's been completed, as cucumber already does. Add
|
|
197
|
+
// a 'step' type and additional success/failure information, though that
|
|
198
|
+
// may be challenging.
|
|
199
|
+
reports.addToReport(scope, { type: `row info`, value: step.pickleStep.text });
|
|
200
|
+
reports.indent();
|
|
201
|
+
});
|
|
178
202
|
|
|
179
203
|
//#####################################
|
|
180
204
|
//#####################################
|
|
@@ -339,17 +363,17 @@ Given(/the max(?:imum)? sec(?:ond)?s for each step (?:in this scenario )?is (\d+
|
|
|
339
363
|
// Story
|
|
340
364
|
//#####################################
|
|
341
365
|
//#####################################
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
// with(?:.*):?
|
|
345
|
-
When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)" with this data:?/, {timeout: -1}, async (target_id, raw_var_data) => {
|
|
366
|
+
const INCLUDING_QUOTES_REGEX = /"([^"]*)"/g;
|
|
367
|
+
When(/I? ?gets? to ?.*?(".*").*?with.*?(?:this data:)?/i, {timeout: -1}, async (target_ids, raw_var_data) => {
|
|
346
368
|
/* NO CUCUMBER TIMEOUT. Must handle timeout ourselves:
|
|
347
369
|
* https://github.com/cucumber/cucumber-js/blob/master/docs/support_files/timeouts.md#disable-timeouts
|
|
348
370
|
* Iterate in here to keep timeout reasonable */
|
|
349
371
|
|
|
350
372
|
// Note: Why do we not just stop when we've used all the vars in the table or
|
|
351
373
|
// reach the last var? Vars are not required to be given in sequence and pages that
|
|
352
|
-
// have non-button fields don't
|
|
374
|
+
// have non-button fields often don't have a var assigned to the 'continue' button.
|
|
375
|
+
// It is optional for the table to use all the vars and some vars may be used
|
|
376
|
+
// multiple times.
|
|
353
377
|
// Alternative: require vars even for 'continue' pages. Then in loop require at least one var
|
|
354
378
|
// to be found on a page before hitting 'continue'. That requires detecting navigation because
|
|
355
379
|
// pages with fields will not have a var for the continue button, so there will
|
|
@@ -357,10 +381,18 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
357
381
|
// It also seems harsh, but more reliable.
|
|
358
382
|
// Ask developers what they'd like.
|
|
359
383
|
|
|
384
|
+
let ids_as_list = Array.from(
|
|
385
|
+
target_ids.matchAll( INCLUDING_QUOTES_REGEX ),
|
|
386
|
+
( match ) => { return match[ 1 ]; }
|
|
387
|
+
);
|
|
388
|
+
let sanitized_ids = ids_as_list.map(( risky_id ) => {
|
|
389
|
+
return scope.to_docassemble_sanitized_id( risky_id );
|
|
390
|
+
});
|
|
360
391
|
|
|
361
392
|
let supported_table = await scope.normalizeTable( scope, { raw_var_data });
|
|
362
393
|
|
|
363
|
-
let
|
|
394
|
+
let id = await scope.get_page_id( scope );
|
|
395
|
+
let id_matches = id && sanitized_ids.includes( id );
|
|
364
396
|
// Until we reach the target page, hit an error, or time out (take too long)
|
|
365
397
|
while ( !id_matches ) {
|
|
366
398
|
let custom_timeout = new Promise(function( resolve, reject ) {
|
|
@@ -368,8 +400,8 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
368
400
|
// Set up the timeout that avoids infinite loops
|
|
369
401
|
let on_page_timeout = async function() {
|
|
370
402
|
|
|
371
|
-
let non_reload_report_msg = `The target id
|
|
372
|
-
+ `too long to try to reach
|
|
403
|
+
let non_reload_report_msg = `The target id or ids were ${ target_ids }, but it took `
|
|
404
|
+
+ `too long to try to reach a valid id (over ${ scope.timeout/1000/60 } min). The `
|
|
373
405
|
+ `test got stuck on "${ id }".`
|
|
374
406
|
let error = non_reload_report_msg;
|
|
375
407
|
await scope.handle_page_timeout_error( scope, { non_reload_report_data: {
|
|
@@ -386,7 +418,7 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
386
418
|
// Maybe list of timeout and/or interval ids in state that can all be cleared at once?
|
|
387
419
|
(async function() {
|
|
388
420
|
try {
|
|
389
|
-
let
|
|
421
|
+
let id = await scope.get_page_id( scope );
|
|
390
422
|
|
|
391
423
|
// THIS FUNCTION ACTUALLY MATTERS
|
|
392
424
|
// Add special rows. Basically, do weirder things.
|
|
@@ -396,20 +428,19 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
396
428
|
from_story_table: true,
|
|
397
429
|
});
|
|
398
430
|
|
|
399
|
-
log.debug({
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
pre: `Rows, including special rows:, ${JSON.stringify( ensured_var_data )}`,
|
|
403
|
-
});
|
|
431
|
+
log.debug({ code: `ALK0061`, context: `table`, },
|
|
432
|
+
`Rows, including special rows:`, ensured_var_data
|
|
433
|
+
);
|
|
404
434
|
|
|
405
435
|
// THIS FUNCTION ACTUALLY MATTERS
|
|
406
436
|
// Some errors happen in there to be closer to the relevant code. We'd have to try/catch anyway.
|
|
407
437
|
let error = await scope.setFields(scope, { var_data: ensured_var_data, id, ensure_navigation: true });
|
|
408
438
|
|
|
439
|
+
// TODO: use Log class to throw this error?
|
|
409
440
|
if ( error.was_found ) { await scope.throwPageError( scope, { id: id }); }
|
|
410
441
|
resolve();
|
|
411
442
|
} catch ( err ) {
|
|
412
|
-
let err_msg = `
|
|
443
|
+
let err_msg = `ALKiln ran into an error it tried to answer fields on the page with the question id "${ id }". The text above might have more information.`
|
|
413
444
|
reports.addToReport( scope, { type: `error`, code: `ALK0062`, value: err_msg });
|
|
414
445
|
// Throw error instead?
|
|
415
446
|
reject( err );
|
|
@@ -422,7 +453,8 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
422
453
|
await custom_timeout;
|
|
423
454
|
|
|
424
455
|
// Prep for next loop
|
|
425
|
-
|
|
456
|
+
id = await scope.get_page_id( scope );
|
|
457
|
+
id_matches = id && sanitized_ids.includes( id );
|
|
426
458
|
|
|
427
459
|
} // ends while !id_matches
|
|
428
460
|
// Can anything unsatisfactory get through here?
|
|
@@ -441,7 +473,7 @@ When(/^(?:the user gets)?(?:I get)? to ?(?:the)?(?: ?question)?(?: id)? "([^"]+)
|
|
|
441
473
|
// Add full used table to report so it can be copied if needed.
|
|
442
474
|
// TODO: Should we/How do we get this in the final scenario report instead of here?
|
|
443
475
|
reports.addToReport( scope, { type: `note info`, code: `ALK0063`, value: `\n${ ' '.repeat(2) }Rows that got set:` });
|
|
444
|
-
reports.addToReport( scope, { type: `step info`, code: `ALK0064`, value: `${ ' '.repeat(4) }And I get to the question
|
|
476
|
+
reports.addToReport( scope, { type: `step info`, code: `ALK0064`, value: `${ ' '.repeat(4) }And I get to any of the question ids ${ target_ids } with this data:` });
|
|
445
477
|
reports.addToReport(scope, { type: `row info`, code: `ALK0065`, value: `${ ' '.repeat(6) }| var | value | trigger |`, });
|
|
446
478
|
let at_least_one_row_was_NOT_used = false;
|
|
447
479
|
for ( let row of sorted_rows ) {
|
|
@@ -972,7 +1004,7 @@ When(/I answer randomly for at most ([0-9]+) (?:page(?:s)?|screen(?:s)?)/, { tim
|
|
|
972
1004
|
} // ends if !has_content
|
|
973
1005
|
|
|
974
1006
|
if ( buttons_exist ) {
|
|
975
|
-
|
|
1007
|
+
log.stdout({}, `\x1b[36m${ 'v' }\x1b[0m`); // pressed button
|
|
976
1008
|
}
|
|
977
1009
|
|
|
978
1010
|
resolve();
|
|
@@ -1150,6 +1182,22 @@ When(/I set the address of ?(?:the var(?:iable)?)? "([^"]+)" to "([^"]+)"/, { ti
|
|
|
1150
1182
|
// Next macro - download sequence?
|
|
1151
1183
|
|
|
1152
1184
|
|
|
1185
|
+
//#####################################
|
|
1186
|
+
//#####################################
|
|
1187
|
+
// Messages from ALKiln to author
|
|
1188
|
+
//#####################################
|
|
1189
|
+
//#####################################
|
|
1190
|
+
|
|
1191
|
+
const LOG_CODE_REGEX = /ALKiln\d\d\d\d/;
|
|
1192
|
+
Given(/ALKiln warns the author/i, async ( warnings_text ) => {
|
|
1193
|
+
/** Create a warning message for the author files they wrote to generate
|
|
1194
|
+
* constrained random answers tests.
|
|
1195
|
+
*
|
|
1196
|
+
* @param {str} warnings_text - Warning text generated by ALKiln
|
|
1197
|
+
* */
|
|
1198
|
+
reports.addToReport( scope, { type: `as_is`, value: warnings_text });
|
|
1199
|
+
})
|
|
1200
|
+
|
|
1153
1201
|
|
|
1154
1202
|
//#####################################
|
|
1155
1203
|
//#####################################
|
|
@@ -1187,6 +1235,8 @@ Given(/the Scenario report should not include/i, async ( report_text ) => {
|
|
|
1187
1235
|
Given(/the final Scenario status should be "([^"]+)"/i, async ( expected_status ) => {
|
|
1188
1236
|
/** WARNING: If a Step will fail, this Step must come first.
|
|
1189
1237
|
*
|
|
1238
|
+
* TODO: Status "A" ambiguous (internal)
|
|
1239
|
+
*
|
|
1190
1240
|
* Prepare to test that the Scenario completes with the correct status. */
|
|
1191
1241
|
scope.expected_status = expected_status.toUpperCase();
|
|
1192
1242
|
});
|
|
@@ -1263,11 +1313,24 @@ Then(/I fail to delete (\d) detected interview(?:s)? and get no error __internal
|
|
|
1263
1313
|
//#####################################
|
|
1264
1314
|
//#####################################
|
|
1265
1315
|
|
|
1316
|
+
AfterStep(async function({ result }) {
|
|
1317
|
+
// TODO: Abstract this
|
|
1318
|
+
if ( result.status === `PASSED` ) { log.stdout({ records_only: true }, `.`); }
|
|
1319
|
+
else if ( result.status === `FAILED` ) { log.stdout({ records_only: true }, `F`); }
|
|
1320
|
+
else if ( result.status === `UNDEFINED` ) { log.stdout({ records_only: true }, `U`); }
|
|
1321
|
+
else if ( result.status === `SKIPPED` ) { log.stdout({ records_only: true }, `-`); }
|
|
1322
|
+
else if ( result.status === `PENDING` ) { log.stdout({ records_only: true }, `P`); }
|
|
1323
|
+
|
|
1324
|
+
reports.outdent();
|
|
1325
|
+
});
|
|
1326
|
+
|
|
1266
1327
|
After(async function(scenario) {
|
|
1267
1328
|
|
|
1268
1329
|
// Log errors
|
|
1269
1330
|
if ( scenario.result.message ) {
|
|
1270
|
-
log.debug({
|
|
1331
|
+
log.debug({ code: `ALK0091`, context: `scenario`, },
|
|
1332
|
+
scenario.result.message
|
|
1333
|
+
);
|
|
1271
1334
|
}
|
|
1272
1335
|
|
|
1273
1336
|
let changeable_test_status = scenario.result.status;
|
|
@@ -1297,11 +1360,9 @@ After(async function(scenario) {
|
|
|
1297
1360
|
reports.saveFinalScenarioData( scope, { status: changeable_test_status });
|
|
1298
1361
|
|
|
1299
1362
|
if (changeable_test_status !== `PASSED`) {
|
|
1300
|
-
log.debug({
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
pre: `Non-passed status occurred while running test: ${changeable_test_status}`,
|
|
1304
|
-
});
|
|
1363
|
+
log.debug({ code: `ALK0094`, level: `warn`, },
|
|
1364
|
+
`Non-passed status occurred while running test: ${ changeable_test_status }`
|
|
1365
|
+
);
|
|
1305
1366
|
|
|
1306
1367
|
if ( scope.page ) {
|
|
1307
1368
|
|
|
@@ -1397,8 +1458,9 @@ After(async function(scenario) {
|
|
|
1397
1458
|
+ `Expected status: ${scope.expected_status}. `
|
|
1398
1459
|
+ `See the report for details.`;
|
|
1399
1460
|
let code = `ALK0099`;
|
|
1400
|
-
log.debug({
|
|
1401
|
-
|
|
1461
|
+
expected_status_err_msg = log.debug({ code: code, level: `error`, },
|
|
1462
|
+
msg
|
|
1463
|
+
);
|
|
1402
1464
|
}
|
|
1403
1465
|
}
|
|
1404
1466
|
scope.expected_status = null; // reset for next Scenario
|
|
@@ -1418,17 +1480,25 @@ After(async function(scenario) {
|
|
|
1418
1480
|
// Make sure that tests which ALKiln has calculated as failed
|
|
1419
1481
|
// cause a cucumber error later. Just changing the status to `FAILED`
|
|
1420
1482
|
// causes unhelpful cucumber errors. TODO: Add to internal docs.
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
});
|
|
1483
|
+
// TODO: Should we move this to where the error is thrown?
|
|
1484
|
+
log.debug({ code: force_failure_code, level: `error`, },
|
|
1485
|
+
force_failure_msg
|
|
1486
|
+
);
|
|
1426
1487
|
// TODO: Look into setDefinitionFunctionWrapper to catch errors
|
|
1427
1488
|
// of failing tests that should indeed be failing:
|
|
1428
1489
|
// https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/api_reference.md#setdefinitionfunctionwrapperwrapper
|
|
1429
1490
|
need_to_force_failed_status = true;
|
|
1430
1491
|
}
|
|
1431
1492
|
|
|
1493
|
+
// Not sure if this is necessary
|
|
1494
|
+
if ( changeable_test_status === 'PASSED' ) {
|
|
1495
|
+
log.stdout({ records_only: true }, `.` );
|
|
1496
|
+
} else if ( changeable_test_status === `SKIPPED` ) {
|
|
1497
|
+
log.stdout({ records_only: true }, `-` );
|
|
1498
|
+
} else {
|
|
1499
|
+
log.stdout({ records_only: true }, changeable_test_status[0].toUpperCase() );
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1432
1502
|
let signout_succeeded = true;
|
|
1433
1503
|
let signout_error_msg = ``;
|
|
1434
1504
|
// If there is a page open, then sign out and close it
|
|
@@ -1482,9 +1552,10 @@ After(async function(scenario) {
|
|
|
1482
1552
|
// going to the wrong file). If it becomes a problem we can track it down later.
|
|
1483
1553
|
if ( visible_keys.length > 0 ) {
|
|
1484
1554
|
// Can an API key ever see an interview that it can't delete?
|
|
1485
|
-
log.
|
|
1486
|
-
|
|
1487
|
-
|
|
1555
|
+
log.debug({ code: `ALK0197`, context: `internal`, level: `warn`, }
|
|
1556
|
+
`The API key can see ${ visible_keys.length } interviews that were not deleted:`,
|
|
1557
|
+
visible_keys
|
|
1558
|
+
);
|
|
1488
1559
|
}
|
|
1489
1560
|
} catch ( error ) {
|
|
1490
1561
|
// Deleting interviews is nice, but not non-critical. Don't bother the
|
|
@@ -1494,7 +1565,9 @@ After(async function(scenario) {
|
|
|
1494
1565
|
// interview. If it becomes a problem we can track it down later.
|
|
1495
1566
|
// Consider add "internal" to docs: "If you see this, make an
|
|
1496
1567
|
// issue in ALKiln with your debug/verbose log"
|
|
1497
|
-
log.
|
|
1568
|
+
log.debug({ code: `ALK0198`, context: `internal`, level: `warn`, error },
|
|
1569
|
+
`Cleaning up interviews incomplete, but unimportant.`
|
|
1570
|
+
);
|
|
1498
1571
|
}
|
|
1499
1572
|
}
|
|
1500
1573
|
|
|
@@ -1513,11 +1586,16 @@ After(async function(scenario) {
|
|
|
1513
1586
|
throw new Error( `🤕 ${ force_failure_code } unreachable ERROR: ${ force_failure_msg }` );
|
|
1514
1587
|
|
|
1515
1588
|
} else if ( internal_errors.length > 0 ) {
|
|
1516
|
-
log.
|
|
1589
|
+
log.debug({ code: `ALK0203`, context: `internal`, level: `warn`, },
|
|
1590
|
+
`There was no need to force failed status and yet there are error messages.`,
|
|
1591
|
+
internal_errors
|
|
1592
|
+
);
|
|
1517
1593
|
}
|
|
1518
1594
|
|
|
1519
1595
|
// Find the race condition.
|
|
1520
|
-
log.debug({
|
|
1596
|
+
log.debug({ code: `ALK0100`, context: `scenario`, },
|
|
1597
|
+
`Scenario After() message:`, scenario.result.message
|
|
1598
|
+
);
|
|
1521
1599
|
|
|
1522
1600
|
});
|
|
1523
1601
|
|