@suffolklitlab/alkiln 5.13.0-logs3 → 5.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/github_server.yml +8 -8
- package/.github/workflows/playground.yml +3 -3
- package/CHANGELOG.md +1 -0
- package/action.yml +28 -70
- package/action_for_github_server/action.yml +29 -61
- package/lib/docassemble/docassemble_api_interface.js +34 -63
- package/lib/docassemble/server_install.js +9 -23
- package/lib/docassemble/setup.js +20 -33
- package/lib/docassemble/takedown.js +7 -27
- package/lib/run_cucumber.js +33 -81
- package/lib/scope.js +75 -61
- package/lib/steps.js +41 -70
- package/lib/utils/files.js +6 -8
- package/lib/utils/log.js +84 -477
- package/lib/utils/reports.js +4 -13
- package/lib/utils/session_vars.js +18 -25
- package/lib/utils/time.js +0 -44
- package/npm-shrinkwrap.json +19 -15
- package/package.json +2 -2
- package/tests/log_codes/check_codes.sh +1 -2
- package/tests/log_codes/log_code_expected_instances.txt +7 -11
- package/tests/unit_tests/getAllFields.test.js +0 -5
- package/tests/unit_tests/getMatchingRows.test.js +0 -5
- package/tests/unit_tests/normalizeTable.test.js +0 -5
- package/docs/decisions/006_which_log_level_to_pick_2024_09_5.md +0 -46
- package/docs/decisions/reports_and_logs_2024_08_11.md +0 -141
- package/tests/unit_tests/log.test.js +0 -20
|
@@ -3,22 +3,19 @@ const fg = require('fast-glob');
|
|
|
3
3
|
const path = require('path');
|
|
4
4
|
require('dotenv').config();
|
|
5
5
|
|
|
6
|
-
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
* We use `.get_foo()` functions partly because it makes the code
|
|
20
|
-
* easier to test.
|
|
21
|
-
* */
|
|
6
|
+
/**
|
|
7
|
+
* This code assumes that this module is run as part of a GitHub Action. If the
|
|
8
|
+
* customer is running it as a CLI tool, then they won't get useful messages yet.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* This file provides user info variables. E.g. server address. They're either
|
|
12
|
+
* `process.env` vars or vars that have been built from `process.env` vars.
|
|
13
|
+
* It is meant to avoid duplication, make testing easier, and leave room for
|
|
14
|
+
* future manipulation. No other file should need to use `process.env`.
|
|
15
|
+
*
|
|
16
|
+
* We have a bunch of `.get_foo()` functions partly because it makes the code
|
|
17
|
+
* much easier to test.
|
|
18
|
+
*/
|
|
22
19
|
|
|
23
20
|
const session_vars = {};
|
|
24
21
|
|
|
@@ -139,7 +136,7 @@ session_vars.save_project_name = function ( project_name ) {
|
|
|
139
136
|
let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
|
|
140
137
|
|
|
141
138
|
if ( session_vars.get_debug() ) {
|
|
142
|
-
console.
|
|
139
|
+
console.log( `🐞 ALK0036 DEBUG: Stored name of Project "${ project_name }" locally.` );
|
|
143
140
|
}
|
|
144
141
|
return file;
|
|
145
142
|
}; // Ends save_project_name()
|
|
@@ -151,7 +148,7 @@ session_vars.get_project_name = function () {
|
|
|
151
148
|
}
|
|
152
149
|
let json = JSON.parse( fs.readFileSync( runtime_config_path ));
|
|
153
150
|
let project_name = json[ project_name_key ] || null;
|
|
154
|
-
if ( session_vars.get_debug() ) { console.
|
|
151
|
+
if ( session_vars.get_debug() ) { console.log( `🐞 ALK0037 DEBUG: Project name from file is "${ project_name }".` ); }
|
|
155
152
|
return project_name;
|
|
156
153
|
}; // Ends get_project_name()
|
|
157
154
|
|
|
@@ -175,11 +172,7 @@ session_vars.delete_project_name = function () {
|
|
|
175
172
|
// const file_storing_artifact_path = `_alkiln_test_artifacts_dir_name.json`;
|
|
176
173
|
const artifacts_path_key = `artifacts_path`;
|
|
177
174
|
session_vars.save_artifacts_path_name = function ( path_name ) {
|
|
178
|
-
|
|
179
|
-
* in case it doesn't exist. Should we create the path in here
|
|
180
|
-
* if it doesn't exist? Should we reserve this file for getting
|
|
181
|
-
* and setting?
|
|
182
|
-
* */
|
|
175
|
+
/* Save a key with the name of the current artifacts folder in case it doesn't exist. */
|
|
183
176
|
return session_vars.save_runtime_config_var({ key: artifacts_path_key, value: path_name });
|
|
184
177
|
}; // Ends save_artifacts_path_name()
|
|
185
178
|
session_vars.get_artifacts_path_name = function () {
|
|
@@ -233,7 +226,7 @@ session_vars.save_runtime_config_var = function ({ key, value }) {
|
|
|
233
226
|
let file = fs.writeFileSync( runtime_config_path, JSON.stringify( json, null, 2 ) );
|
|
234
227
|
|
|
235
228
|
if ( session_vars.get_debug() ) {
|
|
236
|
-
console.
|
|
229
|
+
console.log( `🐞 ALK0040 DEBUG: Stored runtime config var "${key}" locally as "${value}".` );
|
|
237
230
|
}
|
|
238
231
|
return file;
|
|
239
232
|
}; // Ends session_vars.save_runtime_config_var()
|
|
@@ -249,7 +242,7 @@ session_vars.get_runtime_config_var = function ( key ) {
|
|
|
249
242
|
let json = JSON.parse( fs.readFileSync( runtime_config_path ));
|
|
250
243
|
let value = json[ key ];
|
|
251
244
|
if ( session_vars.get_debug() ) {
|
|
252
|
-
console.
|
|
245
|
+
console.log( `🐞 ALK0041 DEBUG: Runtime config var "${key}" is "${value}".` );
|
|
253
246
|
}
|
|
254
247
|
|
|
255
248
|
return value;
|
package/lib/utils/time.js
CHANGED
|
@@ -24,47 +24,3 @@ time.waitForTimeout = function( milliseconds ) {
|
|
|
24
24
|
setTimeout(resolve, milliseconds);
|
|
25
25
|
});
|
|
26
26
|
};
|
|
27
|
-
time.sleep = time.waitForTimeout;
|
|
28
|
-
time.wait = time.waitForTimeout;
|
|
29
|
-
|
|
30
|
-
time.filename_date = function() {
|
|
31
|
-
/** Return the current timestamp in a human-readable
|
|
32
|
-
* filename-safe format to match the workflow artifact
|
|
33
|
-
* date format.
|
|
34
|
-
* See https://stackoverflow.com/a/48729396/14144258 and https://stackoverflow.com/a/6040556/14144258
|
|
35
|
-
*/
|
|
36
|
-
// let date = new Date(); // now
|
|
37
|
-
// let day = ("0" + date.getUTCDate()).slice(-2);
|
|
38
|
-
// let month = ("0" + (date.getUTCMonth() + 1)).slice(-2);
|
|
39
|
-
// let year = date.getUTCFullYear();
|
|
40
|
-
// let hours = ("0" + date.getUTCHours()).slice(-2);
|
|
41
|
-
// let mins = ("0" + date.getUTCMinutes()).slice(-2);
|
|
42
|
-
// let secs = ("0" + date.getUTCSeconds()).slice(-2);
|
|
43
|
-
|
|
44
|
-
let dt = time.get_time_parts();
|
|
45
|
-
let safe_date = `${ dt.y }-${ dt.mon }-${ dt.d } at ${ dt.h }h${ dt.min }m${ dt.s }sUTC`;
|
|
46
|
-
return safe_date;
|
|
47
|
-
}; // Ends time.filename_date()
|
|
48
|
-
|
|
49
|
-
time.log_date = function () {
|
|
50
|
-
/** Return the current timestamp in a human-readable
|
|
51
|
-
* more compact log format.
|
|
52
|
-
* See https://stackoverflow.com/a/48729396/14144258 and https://stackoverflow.com/a/6040556/14144258
|
|
53
|
-
*/
|
|
54
|
-
let dt = time.get_time_parts();
|
|
55
|
-
let short_date = `${ dt.y }-${ dt.mon }-${ dt.d } ${ dt.h }:${ dt.min }:${ dt.s }UTC`;
|
|
56
|
-
return short_date;
|
|
57
|
-
}; // Ends time.log_date()
|
|
58
|
-
|
|
59
|
-
time.get_time_parts = function () {
|
|
60
|
-
/** Get all the parts of the date as numbers. */
|
|
61
|
-
let date = new Date(); // now
|
|
62
|
-
return ({
|
|
63
|
-
y: date.getUTCFullYear(),
|
|
64
|
-
mon: ("0" + (date.getUTCMonth() + 1)).slice(-2),
|
|
65
|
-
d: ("0" + date.getUTCDate()).slice(-2),
|
|
66
|
-
h: ("0" + date.getUTCHours()).slice(-2),
|
|
67
|
-
min: ("0" + date.getUTCMinutes()).slice(-2),
|
|
68
|
-
s: ("0" + date.getUTCSeconds()).slice(-2),
|
|
69
|
-
});
|
|
70
|
-
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"minimatch": "3.0.5",
|
|
23
23
|
"minimist": "1.2.8",
|
|
24
24
|
"mocha": "9.2.2",
|
|
25
|
-
"pdfjs-dist": "3.
|
|
25
|
+
"pdfjs-dist": "3.11.174",
|
|
26
26
|
"puppeteer": "22.15.0",
|
|
27
27
|
"qs": "6.10.3",
|
|
28
28
|
"sanitize-filename": "1.6.3",
|
|
@@ -2908,6 +2908,16 @@
|
|
|
2908
2908
|
"node": ">=0.10.0"
|
|
2909
2909
|
}
|
|
2910
2910
|
},
|
|
2911
|
+
"node_modules/path2d-polyfill": {
|
|
2912
|
+
"version": "2.0.1",
|
|
2913
|
+
"resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz",
|
|
2914
|
+
"integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==",
|
|
2915
|
+
"license": "MIT",
|
|
2916
|
+
"optional": true,
|
|
2917
|
+
"engines": {
|
|
2918
|
+
"node": ">=8"
|
|
2919
|
+
}
|
|
2920
|
+
},
|
|
2911
2921
|
"node_modules/pathval": {
|
|
2912
2922
|
"version": "1.1.1",
|
|
2913
2923
|
"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
|
|
@@ -2917,14 +2927,16 @@
|
|
|
2917
2927
|
}
|
|
2918
2928
|
},
|
|
2919
2929
|
"node_modules/pdfjs-dist": {
|
|
2920
|
-
"version": "3.
|
|
2921
|
-
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.
|
|
2922
|
-
"integrity": "sha512-
|
|
2923
|
-
"
|
|
2924
|
-
|
|
2930
|
+
"version": "3.11.174",
|
|
2931
|
+
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.11.174.tgz",
|
|
2932
|
+
"integrity": "sha512-TdTZPf1trZ8/UFu5Cx/GXB7GZM30LT+wWUNfsi6Bq8ePLnb+woNKtDymI2mxZYBpMbonNFqKmiz684DIfnd8dA==",
|
|
2933
|
+
"license": "Apache-2.0",
|
|
2934
|
+
"engines": {
|
|
2935
|
+
"node": ">=18"
|
|
2925
2936
|
},
|
|
2926
2937
|
"optionalDependencies": {
|
|
2927
|
-
"canvas": "^2.11.
|
|
2938
|
+
"canvas": "^2.11.2",
|
|
2939
|
+
"path2d-polyfill": "^2.0.1"
|
|
2928
2940
|
}
|
|
2929
2941
|
},
|
|
2930
2942
|
"node_modules/pend": {
|
|
@@ -3782,14 +3794,6 @@
|
|
|
3782
3794
|
"extsprintf": "^1.2.0"
|
|
3783
3795
|
}
|
|
3784
3796
|
},
|
|
3785
|
-
"node_modules/web-streams-polyfill": {
|
|
3786
|
-
"version": "3.3.3",
|
|
3787
|
-
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
|
|
3788
|
-
"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
|
|
3789
|
-
"engines": {
|
|
3790
|
-
"node": ">= 8"
|
|
3791
|
-
}
|
|
3792
|
-
},
|
|
3793
3797
|
"node_modules/webidl-conversions": {
|
|
3794
3798
|
"version": "3.0.1",
|
|
3795
3799
|
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@suffolklitlab/alkiln",
|
|
3
|
-
"version": "5.13.
|
|
3
|
+
"version": "5.13.1",
|
|
4
4
|
"description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"minimatch": "3.0.5",
|
|
48
48
|
"minimist": "1.2.8",
|
|
49
49
|
"mocha": "9.2.2",
|
|
50
|
-
"pdfjs-dist": "3.
|
|
50
|
+
"pdfjs-dist": "3.11.174",
|
|
51
51
|
"puppeteer": "22.15.0",
|
|
52
52
|
"qs": "6.10.3",
|
|
53
53
|
"sanitize-filename": "1.6.3",
|
|
@@ -55,8 +55,7 @@ fi
|
|
|
55
55
|
|
|
56
56
|
# removed and are no longer used. Syntax used works with GitHub cli - [[:digit:]]
|
|
57
57
|
# https://stackoverflow.com/a/6901221
|
|
58
|
-
|
|
59
|
-
instances=$(find "$where_to_look" -type f ! -name "check_codes.sh" ! -name "CONTRIBUTING.md" ! -name "$expected_instances_file" ! -name "debug_log.txt" ! -name "cucumber-report.txt" ! -path "*/node_modules/*" ! -path "*/ALKilnTests/*" ! -path "*/alkiln-*/*" ! -path "*/_alkiln*/*" ! -path "*/docs/decisions/*" ! -path "*/\.*" -print0 | xargs -0 -P 4 grep -0 -ro 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' | grep -v -- '--' )
|
|
58
|
+
instances=$(find "$where_to_look" -type f ! -name "check_codes.sh" ! -name "CONTRIBUTING.md" ! -name "$expected_instances_file" ! -name "debug_log.txt" ! -name "cucumber-report.txt" ! -path "*/node_modules/*" ! -path "*/ALKilnTests/*" ! -path "*/alkiln-*/*" ! -path "*/_alkiln*/*" -print0 | xargs -0 -P 4 grep -0 -ro 'ALK[[:digit:]][[:digit:]][[:digit:]][[:digit:]]' | grep -v -- '--' )
|
|
60
59
|
|
|
61
60
|
if [[ "$loudness" != "0" ]]; then
|
|
62
61
|
total_instances=$(echo "$instances" | wc -l)
|
|
@@ -4,11 +4,7 @@ Why we need duplicate codes sometimes: GitHub actions sometimes don't have ways
|
|
|
4
4
|
|
|
5
5
|
The script that tests the log codes will ignore this file (or any file with this name), and any file with these names: check_codes.sh, debug_log.txt, cucumber-report.txt. It will also ignore files in folders with these names:, node_modules, ALKilnTests, alkiln-*, _alkiln*. That list might get updated, so for an up-to-date list, always check the file check_codes.sh where we use `--exclude=` and such.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
0053 - duplicate log
|
|
9
|
-
0173 - redundant
|
|
10
|
-
|
|
11
|
-
ALK0000:0
|
|
7
|
+
ALK0000:8
|
|
12
8
|
ALK0001:1
|
|
13
9
|
ALK0002:2
|
|
14
10
|
ALK0003:2
|
|
@@ -26,7 +22,7 @@ ALK0014:1
|
|
|
26
22
|
ALK0015:1
|
|
27
23
|
ALK0016:1
|
|
28
24
|
ALK0017:1
|
|
29
|
-
ALK0018:
|
|
25
|
+
ALK0018:2
|
|
30
26
|
ALK0019:2
|
|
31
27
|
ALK0020:1
|
|
32
28
|
ALK0021:1
|
|
@@ -37,10 +33,10 @@ ALK0025:1
|
|
|
37
33
|
ALK0026:2
|
|
38
34
|
ALK0027:2
|
|
39
35
|
ALK0028:3
|
|
40
|
-
ALK0029:
|
|
36
|
+
ALK0029:1
|
|
41
37
|
ALK0030:2
|
|
42
|
-
ALK0031:
|
|
43
|
-
ALK0032:
|
|
38
|
+
ALK0031:1
|
|
39
|
+
ALK0032:1
|
|
44
40
|
ALK0033:1
|
|
45
41
|
ALK0034:1
|
|
46
42
|
ALK0035:1
|
|
@@ -61,7 +57,7 @@ ALK0049:1
|
|
|
61
57
|
ALK0050:1
|
|
62
58
|
ALK0051:1
|
|
63
59
|
ALK0052:1
|
|
64
|
-
ALK0053:
|
|
60
|
+
ALK0053:1
|
|
65
61
|
ALK0054:1
|
|
66
62
|
ALK0055:1
|
|
67
63
|
ALK0056:1
|
|
@@ -181,7 +177,7 @@ ALK0169:1
|
|
|
181
177
|
ALK0170:1
|
|
182
178
|
ALK0171:1
|
|
183
179
|
ALK0172:1
|
|
184
|
-
ALK0173:
|
|
180
|
+
ALK0173:1
|
|
185
181
|
ALK0174:1
|
|
186
182
|
ALK0175:1
|
|
187
183
|
ALK0176:1
|
|
@@ -7,11 +7,6 @@ chai.use(deepEqualInAnyOrder);
|
|
|
7
7
|
const html = require('./html.fixtures.js');
|
|
8
8
|
const fields = require('./fields.fixtures.js');
|
|
9
9
|
const scope = require('../../lib/scope.js');
|
|
10
|
-
|
|
11
|
-
const Log = require(`../../lib/utils/log.js`);
|
|
12
|
-
const log = new Log({ path: `_alkiln-misc_artifacts`, context: `unit_tests getAllFields` });
|
|
13
|
-
scope.set_log({ log_obj: log });
|
|
14
|
-
|
|
15
10
|
const getAllFields = scope.getAllFields;
|
|
16
11
|
|
|
17
12
|
chai.config.truncateThreshold = 0
|
|
@@ -6,11 +6,6 @@ const tables = require('./tables.fixtures.js');
|
|
|
6
6
|
const fields = require('./fields.fixtures.js');
|
|
7
7
|
const matches = require('./matches.fixtures.js');
|
|
8
8
|
const scope = require('../../lib/scope.js');
|
|
9
|
-
|
|
10
|
-
const Log = require(`../../lib/utils/log.js`);
|
|
11
|
-
const log = new Log({ path: `_alkiln-misc_artifacts`, context: `unit_tests getMatchingRows` });
|
|
12
|
-
scope.set_log({ log_obj: log });
|
|
13
|
-
|
|
14
9
|
const getMatchingRows = scope.getMatchingRows;
|
|
15
10
|
// Match state var instantiated in steps.js. TODO: Add tests for such rows.
|
|
16
11
|
scope.row_needs_trigger_flag = `ALKiln: no trigger variable needed`;
|
|
@@ -5,11 +5,6 @@ const expect = chai.expect;
|
|
|
5
5
|
|
|
6
6
|
const tables = require('./tables.fixtures.js');
|
|
7
7
|
const scope = require('../../lib/scope.js');
|
|
8
|
-
|
|
9
|
-
const Log = require(`../../lib/utils/log.js`);
|
|
10
|
-
const log = new Log({ path: `_alkiln-misc_artifacts`, context: `unit_tests normalizeTable` });
|
|
11
|
-
scope.set_log({ log_obj: log });
|
|
12
|
-
|
|
13
8
|
const normalizeTable = scope.normalizeTable;
|
|
14
9
|
|
|
15
10
|
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
# Which log levels to pick
|
|
2
|
-
|
|
3
|
-
## Context and Problem Statement
|
|
4
|
-
|
|
5
|
-
The log levels we're giving access to in Log are `success` (`info` with a special icon), `info`, `warn`, and `throw` (`error` and throw the error). Sometimes, like in the log file itself, we hard-code the icons and such. Those places have access to other levels, like `error`.
|
|
6
|
-
|
|
7
|
-
We want to avoid distracting our users by logging errors everywhere, even when an error is what triggered the log. Does that hold true in the debug log? Successes can also be misleading about whether tests have passed or not.
|
|
8
|
-
|
|
9
|
-
When picking the log level for a log, how do we know which level to choose?
|
|
10
|
-
|
|
11
|
-
## Considered Options
|
|
12
|
-
|
|
13
|
-
- Stay as true to technical behavior as possible.
|
|
14
|
-
- Stay as clear to users as possible - use levels that will help users pay attention to what affects them.
|
|
15
|
-
|
|
16
|
-
For example:
|
|
17
|
-
|
|
18
|
-
`success`:
|
|
19
|
-
- Use whenever anything completes successfully.
|
|
20
|
-
- Only use for success of major operations.
|
|
21
|
-
- Use for every test success.
|
|
22
|
-
- Use for success of all the tests.
|
|
23
|
-
`error`:
|
|
24
|
-
- Always use `error` when an error happens.
|
|
25
|
-
- Use `error` when an unexpected error happens.
|
|
26
|
-
- Only use `error` for user-relevant errors.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
## Decision Outcome
|
|
30
|
-
|
|
31
|
-
Distract the user as little as possible everywhere.
|
|
32
|
-
|
|
33
|
-
`success`:
|
|
34
|
-
- Only log success once for success of all tests.
|
|
35
|
-
- Include the `success` icon in the Scenario "Passed" section of the report.
|
|
36
|
-
`throw`:
|
|
37
|
-
- Use this level to throw errors for the user's tests (or our own internal tests).
|
|
38
|
-
- Use this if ALKiln itself needs to abort a test or the test suite.
|
|
39
|
-
`error`:
|
|
40
|
-
- Avoid using `error` explicitly. Only the debug logs should use the `error` level, and only to record critical errors that come in with `throw`.
|
|
41
|
-
`info`:
|
|
42
|
-
- Log expected errors as `info`. For example, the aborted promises when awaiting navigation.
|
|
43
|
-
`warn`:
|
|
44
|
-
- Avoid using the word "error" in warnings if possible.
|
|
45
|
-
- Use `warn` in the console for messages the user should investigate.
|
|
46
|
-
- Use `warn` in the debug log for messages ALKiln devs might investigate if necessary. If the warning comes from non-critical unexpected behavior, do try to be clear about what happened, but also try to avoid using the word "error" if possible. E.g. "Skipped creating metadata for the logs" instead of "Error while creating metadata for the logs".
|
|
@@ -1,141 +0,0 @@
|
|
|
1
|
-
# Shape and behavior of the report and log objects
|
|
2
|
-
|
|
3
|
-
## Context and scope
|
|
4
|
-
|
|
5
|
-
We have reports and logs we show to the user in the console and as artifacts. We have logs we use for development. cucumber also has logs we want to include in the above until we are confident we are handling all cases ourselves.
|
|
6
|
-
|
|
7
|
-
Right now, the system for logging is fractured and scattered - reports log to the console and save completely separately from other logs. Logs that are not from the report don't get saved anywhere. Some logs only get shown in debug mode, which can be frustrating and unhelpful.
|
|
8
|
-
|
|
9
|
-
## Goals
|
|
10
|
-
|
|
11
|
-
Overview:
|
|
12
|
-
|
|
13
|
-
- Unify the logging and report systems
|
|
14
|
-
- Clarify their flow
|
|
15
|
-
- Make log information more visible
|
|
16
|
-
|
|
17
|
-
Details:
|
|
18
|
-
|
|
19
|
-
- Log objects have statuses, icons, log codes (e.g. ALK0105), contextual info (e.g. `setup`, `takedown`, `invalid answer`), timestamps, descriptive text, and data (like full error traces).
|
|
20
|
-
- These are the broad purposes of various logs:
|
|
21
|
-
1. Setup and takedown messages.
|
|
22
|
-
2. Test progress (a compact way to see what's happening/happened in each test):
|
|
23
|
-
|
|
24
|
-
Test1: .............F--
|
|
25
|
-
|
|
26
|
-
Test 2: ..
|
|
27
|
-
3. A pretty report at the end with details about each test. It has sections, like "failed" and "passed", headings, etc.
|
|
28
|
-
4. cucumber's various console logs
|
|
29
|
-
5. Lots of other informational logs for internal development and debugging purposes.
|
|
30
|
-
I want to do different things with different logs:
|
|
31
|
-
- During the tests, always log items 1, 2, 3, and 4 to the console.
|
|
32
|
-
- During the tests, save items 2, 3, and 4 to a log as they come with all the info in the log objects - `report_log.txt`/`running_report.txt`/`report_run.txt` maybe. If we have to stop the tests early because of an infinite loop or something, this will at least show some information.
|
|
33
|
-
- At the end of the tests, format items 2, 3, and 4, often with only part of the log object info, in a file called `report.txt`.
|
|
34
|
-
- During the tests, save all of the items to a `verbose_log.txt` as they happen.
|
|
35
|
-
- In debug mode, log all that stuff to the console while the tests are running.
|
|
36
|
-
Right now, a few of the logs, like for the test progress (item 2), are printed with `stdout` inline while all others are logged with `console.log`.
|
|
37
|
-
- Logs should accept multiple arguments to match console.log/etc. behavior.
|
|
38
|
-
|
|
39
|
-
Note: The progress logs printed to the console have color codes. We also may want a plain text version without any colors to save in the report log and show in the Playground[^1] "console" and/or a version converted to HTML that does have styles. The latter may be for something easier to digest in a browser and/or to show visually in the Playground version of the tests.
|
|
40
|
-
|
|
41
|
-
Questions:
|
|
42
|
-
- Should `log` throw errors? Otherwise everywhere we need to error, we need to use `log` and `throw`, but I am unable to find a way to remove the current function from an error stack/trace.
|
|
43
|
-
- How do we capture "errors" that are more subtle? Keep the word "error" in the "context"/"types" list? For example, expected errors:
|
|
44
|
-
- Invalid answers on the form that are then assessed as valid behavior?
|
|
45
|
-
- `AbortError`s of promises that should indeed have been aborted.
|
|
46
|
-
- Should we allow "in-between"s for all parts of the log? All together: pre-everything (new lines, dividers, etc), icon/context/etc., pre-logs, logs (the actual messages to print), pre-data, data, post-data? We can default to nothing for all of those.
|
|
47
|
-
- How do we differentiate between methods that log to the console vs. methods that just store data?
|
|
48
|
-
|
|
49
|
-
[^1]: Don't go down this rabbit hole. Broadly - with the "Playground" version, users see the results as a web page instead of in the GitHub job console or in the artifacts. The users run the tests on their own server. This has to do with the platform we built ALKiln for - [docassemble](https://docassemble.org/). It gives users a faster iteration cycle.
|
|
50
|
-
|
|
51
|
-
## Non/anti-goals
|
|
52
|
-
|
|
53
|
-
- Spend a month on this
|
|
54
|
-
- Create a complex and heavy system to try to abstract absolutely everything
|
|
55
|
-
|
|
56
|
-
## The actual design
|
|
57
|
-
|
|
58
|
-
This design still seems a little complex and potentially too coupled with reports.
|
|
59
|
-
|
|
60
|
-
### System-context diagram
|
|
61
|
-
|
|
62
|
-
```
|
|
63
|
-
logic code -> logger -> report -----------> console
|
|
64
|
-
^ |-> console |-> stdout/progress
|
|
65
|
-
cucumber ---| |-> running report |-> final report.txt
|
|
66
|
-
|-> verbose log
|
|
67
|
-
```
|
|
68
|
-
|
|
69
|
-
```
|
|
70
|
-
logic code -> logger ------> report ------> final report.txt
|
|
71
|
-
^ |-> console
|
|
72
|
-
cucumber ---| |-> stdout/inline/progress
|
|
73
|
-
|-> running report
|
|
74
|
-
|-> verbose log
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
```
|
|
78
|
-
|------------------------------------<|
|
|
79
|
-
v |
|
|
80
|
-
logic code -> logger -> report -----------> format prettily
|
|
81
|
-
^ |-> console
|
|
82
|
-
cucumber ---| |-> running report
|
|
83
|
-
|-> verbose log
|
|
84
|
-
|-> stdout/progress
|
|
85
|
-
|-> final report.txt
|
|
86
|
-
```
|
|
87
|
-
|
|
88
|
-
### APIs
|
|
89
|
-
|
|
90
|
-
Everything saves to the "verbose"/"debug" log. Aside from that:
|
|
91
|
-
|
|
92
|
-
`log.info/warning/success/error` methods log to console. This includes, e.g, setup, takedown which show up in GitHub workflow job logs.
|
|
93
|
-
|
|
94
|
-
`log.report` saves to the running report then uses a report obj focused on cosmetics that has `.heading/table/header/error/etc` methods. Maybe a `.stdout/inlne` method, which then prints to the console immediately.
|
|
95
|
-
|
|
96
|
-
`log.verbose/debug` (name?) - Only save to verbose/debug log with log codes, etc.
|
|
97
|
-
|
|
98
|
-
### Data storage
|
|
99
|
-
|
|
100
|
-
- `report.txt` - final pretty output for the user with headings, etc.
|
|
101
|
-
- Each Scenario's `report.txt` - pretty output in each test's folder.
|
|
102
|
-
- `running_report_log.txt` (name?) - ugly, but still sparse, giving the user some info about what happened during the tests without overwhelming them.
|
|
103
|
-
- `verbose.txt` (name? `debug`?) - very ugly and full of all the information from everywhere else and more. Useful for internal development and troubleshooting. Includes setup and takedown.
|
|
104
|
-
|
|
105
|
-
It may be complex to store setup and takedown output in the verbose/debug file. `console.Console` cannot append to a file[^2], so we would have to figure out a work-around. One approach could be to always first copy from the existing verbose file and paste those contents back into the file as the first log. That may take setting up some fallbacks to avoid errors.
|
|
106
|
-
|
|
107
|
-
[^2]: `fs.appendFile()` only takes strings and doesn't log exactly like the console. I want to log exactly like the console to make it easy to search for words I and others see in the console output.
|
|
108
|
-
|
|
109
|
-
### Code and pseudo-code
|
|
110
|
-
|
|
111
|
-
For capturing console output, look into how to get the `stdin` for capturing various console output. https://stackoverflow.com/a/54202970:
|
|
112
|
-
|
|
113
|
-
```js
|
|
114
|
-
const stdin = process.openStdin()
|
|
115
|
-
|
|
116
|
-
process.stdout.write('Enter name: ')
|
|
117
|
-
|
|
118
|
-
stdin.addListener('data', text => {
|
|
119
|
-
const name = text.toString().trim()
|
|
120
|
-
console.log('Your name is: ' + name)
|
|
121
|
-
|
|
122
|
-
stdin.pause() // stop reading
|
|
123
|
-
})
|
|
124
|
-
```
|
|
125
|
-
|
|
126
|
-
### Degree of constraint
|
|
127
|
-
|
|
128
|
-
I think this can basically be greenfield (designed from scratch). We do have a current system, but I think we can easily convert our existing code to a new system.
|
|
129
|
-
|
|
130
|
-
## Alternatives considered
|
|
131
|
-
|
|
132
|
-
- The `report` object saves to the running report and then sends that same information to the `log` object. Also prints the final report. This includes `stdout` logs. `log` remains otherwise the same as described above.
|
|
133
|
-
- `log` handles `stdout` logs and leaves them out of the report.
|
|
134
|
-
- For report content, `log` would call to `report`, which would do pretty formatting and then hand the format text back to `log` to be saved and/or printed to the console. I'm not sure how `log` could then know what to do with the output - leave out the codes/etc. when it logs to the console and/or saves to the final report.txt.
|
|
135
|
-
- Avoid the complexity of storing setup and takedown output in the verbose/debug file. In GitHub, they're in the GitHub console. Locally, we see the results immediately. On the other hand, storing them ourselves lets us look for information in just one place.
|
|
136
|
-
|
|
137
|
-
## Cross-cutting concerns
|
|
138
|
-
|
|
139
|
-
_Security, privacy, observability, etc._
|
|
140
|
-
|
|
141
|
-
The same ones we've already dealt with for our current logs.
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
const chai = require('chai');
|
|
2
|
-
const expect = chai.expect;
|
|
3
|
-
|
|
4
|
-
const Log = require('../../lib/utils/log.js');
|
|
5
|
-
|
|
6
|
-
const path = `_alkiln-misc_artifacts`;
|
|
7
|
-
const log = new Log({ path, context: `unit_tests log_tests` });
|
|
8
|
-
|
|
9
|
-
describe(`log`, async function () {
|
|
10
|
-
|
|
11
|
-
// Test it creates a new folder?
|
|
12
|
-
// Test the contents that it saves to various files?
|
|
13
|
-
|
|
14
|
-
it(`returns a debug message from an empty log.debug`, async function() {
|
|
15
|
-
let result = log.debug();
|
|
16
|
-
expect( result ).to.include( `🐛 ALK000v DEBUG` );
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
});
|
|
20
|
-
|