kryptheon 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/kryptheon-Shantanu.js +1672 -0
- package/bin/kryptheon.js +112 -7
- package/kryptheon-fixture.js +72 -8
- package/kryptheon-reporter.js +21 -1
- package/kryptheon-secrets.js +191 -0
- package/kryptheon-signature.js +530 -0
- package/package.json +11 -3
- package/playwright.config.js +20 -20
package/bin/kryptheon.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
// The kryptheon CLI. A thin wrapper so nobody has to remember Playwright's
|
|
3
3
|
// command line or edit a config file. Everything here shells out to the
|
|
4
4
|
// Playwright binary that ships with this package.
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
const fs = require('fs');
|
|
12
12
|
const path = require('path');
|
|
13
13
|
const { spawn, spawnSync } = require('child_process');
|
|
14
|
+
const os = require('os');
|
|
15
|
+
const secrets = require(require('path').join(__dirname, '..', 'kryptheon-secrets.js'));
|
|
14
16
|
const replay = require('../kryptheon-replay.js');
|
|
15
17
|
|
|
16
18
|
const PACKAGE_DIR = path.join(__dirname, '..');
|
|
@@ -33,6 +35,16 @@ function nodeIsTooOld(version) {
|
|
|
33
35
|
return false;
|
|
34
36
|
}
|
|
35
37
|
|
|
38
|
+
// Read from the package rather than repeated here, so it cannot go stale.
|
|
39
|
+
// Someone filing a bug report has to be able to say which version they have.
|
|
40
|
+
function packageVersion() {
|
|
41
|
+
try {
|
|
42
|
+
return require(path.join(PACKAGE_DIR, 'package.json')).version || 'unknown';
|
|
43
|
+
} catch (err) {
|
|
44
|
+
return 'unknown';
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
36
48
|
function usage() {
|
|
37
49
|
console.log('');
|
|
38
50
|
console.log(' kryptheon - record and check your app');
|
|
@@ -43,6 +55,7 @@ function usage() {
|
|
|
43
55
|
console.log(' kryptheon setup-ai tell your AI assistant to check its work');
|
|
44
56
|
console.log('');
|
|
45
57
|
console.log(' add --quiet to check for one line when everything passes');
|
|
58
|
+
console.log(' kryptheon --version print the version you have installed');
|
|
46
59
|
console.log('');
|
|
47
60
|
}
|
|
48
61
|
|
|
@@ -595,15 +608,44 @@ function takeOutSecrets(relativeFile) {
|
|
|
595
608
|
return writeSpec(relativeFile, result.source) ? result.replacements : [];
|
|
596
609
|
}
|
|
597
610
|
|
|
611
|
+
// kryptheon leaves a password file and three files of machine state in the
|
|
612
|
+
// project. If there is already a .gitignore, they belong in it.
|
|
613
|
+
function keepArtefactsOutOfGit() {
|
|
614
|
+
const added = secrets.updateGitignore(USER_DIR);
|
|
615
|
+
if (!added.length) return;
|
|
616
|
+
console.log(' Added to .gitignore: ' + added.join(', '));
|
|
617
|
+
console.log('');
|
|
618
|
+
}
|
|
619
|
+
|
|
598
620
|
function reportSecrets(replacements) {
|
|
599
621
|
if (!replacements || !replacements.length) return;
|
|
622
|
+
const names = replacements.map((item) => item.envName);
|
|
600
623
|
console.log(' What you typed into the password box was not saved in the test.');
|
|
601
624
|
console.log('');
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
625
|
+
|
|
626
|
+
// Written rather than described. Asked to make the file themselves, most
|
|
627
|
+
// people on Windows reach for Notepad or Out-File, both of which add a byte
|
|
628
|
+
// order mark that makes the first line unreadable.
|
|
629
|
+
const hasEnv = fs.existsSync(path.join(USER_DIR, '.env'));
|
|
630
|
+
const example = hasEnv ? null : secrets.writeEnvExample(USER_DIR, names);
|
|
631
|
+
if (example) {
|
|
632
|
+
console.log(' A file called .env.example has been written here:');
|
|
633
|
+
for (const name of names) console.log(' ' + name + '=yourpassword');
|
|
634
|
+
console.log('');
|
|
635
|
+
console.log(' Put your real password in it and save it as .env, in this folder.');
|
|
636
|
+
} else {
|
|
637
|
+
console.log(' Before this can be checked, put it in a file called .env next to');
|
|
638
|
+
console.log(' your tests, like this:');
|
|
639
|
+
for (const item of replacements) {
|
|
640
|
+
console.log(' ' + item.envName + '=your ' + String(item.field).toLowerCase() + ' here');
|
|
641
|
+
}
|
|
605
642
|
}
|
|
606
643
|
console.log('');
|
|
644
|
+
// Said plainly, because the alternative is signing in with a blank password
|
|
645
|
+
// and spending an afternoon looking at the wrong thing.
|
|
646
|
+
console.log(' Until that file exists, "kryptheon check" will skip this recording');
|
|
647
|
+
console.log(' rather than sign in with a blank password.');
|
|
648
|
+
console.log('');
|
|
607
649
|
console.log(' Keep .env out of version control - it holds the real value.');
|
|
608
650
|
console.log('');
|
|
609
651
|
}
|
|
@@ -699,6 +741,7 @@ async function finaliseRecording(outFile, context) {
|
|
|
699
741
|
console.log('');
|
|
700
742
|
|
|
701
743
|
reportSecrets(secrets);
|
|
744
|
+
keepArtefactsOutOfGit();
|
|
702
745
|
await offerToDropLogout(named);
|
|
703
746
|
reportReplayRisks(named);
|
|
704
747
|
|
|
@@ -898,6 +941,7 @@ function triageSpecs() {
|
|
|
898
941
|
const notRecordings = [];
|
|
899
942
|
const runnable = [];
|
|
900
943
|
const withSecrets = [];
|
|
944
|
+
const missingSecrets = [];
|
|
901
945
|
|
|
902
946
|
for (const name of listSpecFiles()) {
|
|
903
947
|
const full = path.join(TESTS_DIR, name);
|
|
@@ -930,6 +974,18 @@ function triageSpecs() {
|
|
|
930
974
|
if (replay.scrubSecrets(source).replacements.length) {
|
|
931
975
|
withSecrets.push({ name: name, fields: replay.scrubSecrets(source).replacements });
|
|
932
976
|
}
|
|
977
|
+
|
|
978
|
+
// A recording that falls back to an empty password is not run at all. It
|
|
979
|
+
// would sign in blank, fail, and send the reader looking at their app.
|
|
980
|
+
const needed = secrets.requiredSecrets(source);
|
|
981
|
+
const absent = needed.length
|
|
982
|
+
? secrets.missingSecrets(needed, { envFile: secrets.envFileFor(USER_DIR) })
|
|
983
|
+
: [];
|
|
984
|
+
if (absent.length) {
|
|
985
|
+
missingSecrets.push({ name: name, names: absent });
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
|
|
933
989
|
runnable.push(name);
|
|
934
990
|
}
|
|
935
991
|
|
|
@@ -938,6 +994,7 @@ function triageSpecs() {
|
|
|
938
994
|
notRecordings: notRecordings,
|
|
939
995
|
runnable: runnable,
|
|
940
996
|
withSecrets: withSecrets,
|
|
997
|
+
missingSecrets: missingSecrets,
|
|
941
998
|
};
|
|
942
999
|
}
|
|
943
1000
|
|
|
@@ -974,6 +1031,8 @@ function check(options) {
|
|
|
974
1031
|
console.log('');
|
|
975
1032
|
return 1;
|
|
976
1033
|
}
|
|
1034
|
+
keepArtefactsOutOfGit();
|
|
1035
|
+
|
|
977
1036
|
const triage = triageSpecs();
|
|
978
1037
|
|
|
979
1038
|
for (const name of triage.repaired) {
|
|
@@ -989,6 +1048,14 @@ function check(options) {
|
|
|
989
1048
|
console.log(' Re-record it to have the password moved into .env automatically.');
|
|
990
1049
|
}
|
|
991
1050
|
|
|
1051
|
+
for (const item of triage.missingSecrets) {
|
|
1052
|
+
console.log('');
|
|
1053
|
+
const example = fs.existsSync(path.join(USER_DIR, '.env.example')) ? '.env.example' : null;
|
|
1054
|
+
for (const line of secrets.missingSecretLines(path.join('tests', item.name), item.names, { examplePath: example })) {
|
|
1055
|
+
console.log(line);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
|
|
992
1059
|
for (const name of triage.notRecordings) {
|
|
993
1060
|
console.log('');
|
|
994
1061
|
console.log(' Skipped ' + path.join('tests', name));
|
|
@@ -1001,8 +1068,13 @@ function check(options) {
|
|
|
1001
1068
|
console.log('');
|
|
1002
1069
|
console.log(' Nothing could be checked.');
|
|
1003
1070
|
console.log('');
|
|
1004
|
-
|
|
1005
|
-
|
|
1071
|
+
if (triage.missingSecrets.length) {
|
|
1072
|
+
console.log(' Every recording here needs a password that is not set yet.');
|
|
1073
|
+
console.log(' Create the .env file above and run this again.');
|
|
1074
|
+
} else {
|
|
1075
|
+
console.log(' None of the files in tests/ is a usable recording.');
|
|
1076
|
+
console.log(' Record one with: kryptheon record <url>');
|
|
1077
|
+
}
|
|
1006
1078
|
console.log('');
|
|
1007
1079
|
return 1;
|
|
1008
1080
|
}
|
|
@@ -1013,10 +1085,34 @@ function check(options) {
|
|
|
1013
1085
|
// cannot produce diagnostics can never be counted as passing.
|
|
1014
1086
|
const only = triage.runnable.map((name) => path.join('tests', name).split(path.sep).join('/'));
|
|
1015
1087
|
if (quiet) process.env.KRYPTHEON_QUIET = '1';
|
|
1088
|
+
|
|
1089
|
+
// Debug signatures. The workers cannot print: Playwright pipes their
|
|
1090
|
+
// descriptors and hands the output to reporter stdio hooks that this
|
|
1091
|
+
// reporter does not implement, so it is dropped. They write to this file
|
|
1092
|
+
// instead and it is printed here, on stderr, where the reporter's own
|
|
1093
|
+
// output on stdout is left completely alone.
|
|
1094
|
+
const signatures = require(path.join(PACKAGE_DIR, 'kryptheon-signature.js'));
|
|
1095
|
+
let sink = null;
|
|
1096
|
+
if (signatures.signatureEnabled()) {
|
|
1097
|
+
sink = path.join(os.tmpdir(), 'kryptheon-signature-' + process.pid + '.jsonl');
|
|
1098
|
+
process.env[signatures.SINK_ENV] = sink;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1016
1101
|
const status = runPlaywright(['test', '--config', CONFIG].concat(only)).status;
|
|
1017
1102
|
|
|
1103
|
+
if (sink) signatures.drainSignatures(sink);
|
|
1104
|
+
|
|
1018
1105
|
// Without this, a green summary could still hide a recording that checked
|
|
1019
1106
|
// nothing at all.
|
|
1107
|
+
if (triage.missingSecrets.length) {
|
|
1108
|
+
const many = triage.missingSecrets.length > 1;
|
|
1109
|
+
console.log(
|
|
1110
|
+
' Note: ' + triage.missingSecrets.length + ' recording' + (many ? 's were' : ' was') +
|
|
1111
|
+
' skipped because a password is not set.'
|
|
1112
|
+
);
|
|
1113
|
+
console.log('');
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1020
1116
|
if (triage.notRecordings.length) {
|
|
1021
1117
|
const many = triage.notRecordings.length > 1;
|
|
1022
1118
|
console.log(
|
|
@@ -1025,7 +1121,9 @@ function check(options) {
|
|
|
1025
1121
|
);
|
|
1026
1122
|
console.log('');
|
|
1027
1123
|
}
|
|
1028
|
-
|
|
1124
|
+
// Skipped for a missing password is not a pass. The fix is one file away,
|
|
1125
|
+
// and reporting zero would let it be mistaken for a clean run.
|
|
1126
|
+
return status || (triage.missingSecrets.length ? 1 : 0);
|
|
1029
1127
|
}
|
|
1030
1128
|
|
|
1031
1129
|
// --- setup-ai ---------------------------------------------------------------
|
|
@@ -1291,6 +1389,11 @@ async function main() {
|
|
|
1291
1389
|
case 'accept':
|
|
1292
1390
|
return finishWith(accept(rest.join(' ').trim()));
|
|
1293
1391
|
break;
|
|
1392
|
+
case '-v':
|
|
1393
|
+
case '--version':
|
|
1394
|
+
case 'version':
|
|
1395
|
+
console.log(packageVersion());
|
|
1396
|
+
return finishWith(0);
|
|
1294
1397
|
case undefined:
|
|
1295
1398
|
case '-h':
|
|
1296
1399
|
case '--help':
|
|
@@ -1308,6 +1411,7 @@ async function main() {
|
|
|
1308
1411
|
|
|
1309
1412
|
// Exported so the naming logic can be checked without opening a browser.
|
|
1310
1413
|
module.exports = {
|
|
1414
|
+
packageVersion: packageVersion,
|
|
1311
1415
|
humanisePath: humanisePath,
|
|
1312
1416
|
nameFromTitle: nameFromTitle,
|
|
1313
1417
|
deriveTestName: deriveTestName,
|
|
@@ -1324,6 +1428,7 @@ module.exports = {
|
|
|
1324
1428
|
record: record,
|
|
1325
1429
|
finaliseRecording: finaliseRecording,
|
|
1326
1430
|
takeOutSecrets: takeOutSecrets,
|
|
1431
|
+
reportSecrets: reportSecrets,
|
|
1327
1432
|
reportReplayRisks: reportReplayRisks,
|
|
1328
1433
|
offerToDropLogout: offerToDropLogout,
|
|
1329
1434
|
noWindowLines: noWindowLines,
|
package/kryptheon-fixture.js
CHANGED
|
@@ -20,6 +20,7 @@ const base = require('@playwright/test');
|
|
|
20
20
|
// Reuse the reporter's host-stripping so a baseline stores exactly the shape
|
|
21
21
|
// the reporter prints.
|
|
22
22
|
const { requestPath } = require('./kryptheon-reporter.js');
|
|
23
|
+
const signature = require('./kryptheon-signature.js');
|
|
23
24
|
|
|
24
25
|
const MAX_ITEMS = 5; // keep the failure block readable
|
|
25
26
|
|
|
@@ -53,10 +54,14 @@ function baselineKey(specFile, title) {
|
|
|
53
54
|
function readBaselines(file) {
|
|
54
55
|
let raw;
|
|
55
56
|
try {
|
|
56
|
-
raw = fs.readFileSync(file, 'utf8');
|
|
57
|
+
raw = fs.readFileSync(file, { encoding: 'utf8' });
|
|
57
58
|
} catch (e) {
|
|
58
59
|
return {};
|
|
59
60
|
}
|
|
61
|
+
// An editor that saved the file with a byte order mark would otherwise make
|
|
62
|
+
// JSON.parse throw, and every baseline would silently vanish - which reads as
|
|
63
|
+
// "all clear" on the next run rather than as the problem it is.
|
|
64
|
+
if (raw.charCodeAt(0) === 0xfeff) raw = raw.slice(1);
|
|
60
65
|
let parsed;
|
|
61
66
|
try {
|
|
62
67
|
parsed = JSON.parse(raw);
|
|
@@ -69,7 +74,9 @@ function readBaselines(file) {
|
|
|
69
74
|
|
|
70
75
|
function saveBaselines(file, all) {
|
|
71
76
|
try {
|
|
72
|
-
|
|
77
|
+
// Explicit, and without a byte order mark: this file is read back by the
|
|
78
|
+
// same code and by people, and a mismatch would look like the app changed.
|
|
79
|
+
fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n', { encoding: 'utf8' });
|
|
73
80
|
return true;
|
|
74
81
|
} catch (e) {
|
|
75
82
|
return false; // diagnostics must never become a second failure
|
|
@@ -112,12 +119,16 @@ function toStoredErrors(list) {
|
|
|
112
119
|
}
|
|
113
120
|
|
|
114
121
|
function normalisedEntry(current) {
|
|
115
|
-
|
|
122
|
+
const stored = {
|
|
116
123
|
url: current.url,
|
|
117
124
|
title: current.title,
|
|
118
125
|
consoleErrors: toStoredErrors(current.consoleErrors),
|
|
119
126
|
failedRequests: toStoredRequests(current.failedRequests),
|
|
120
127
|
};
|
|
128
|
+
// Only when there is one. A run whose capture failed must not write an empty
|
|
129
|
+
// signature over a good one, or the next run would report everything as gone.
|
|
130
|
+
if (current.signature) stored.signature = current.signature;
|
|
131
|
+
return stored;
|
|
121
132
|
}
|
|
122
133
|
|
|
123
134
|
function writeBaseline(file, key, entry) {
|
|
@@ -170,12 +181,17 @@ function normaliseTitle(value) {
|
|
|
170
181
|
return String(value == null ? '' : value).replace(/\s+/g, ' ').trim();
|
|
171
182
|
}
|
|
172
183
|
|
|
173
|
-
//
|
|
174
|
-
// not on its own a regression.
|
|
184
|
+
// The address, the title, and what the page ended up showing decide pass or
|
|
185
|
+
// fail. Browser noise changing is not on its own a regression.
|
|
175
186
|
function compareBaselines(previous, current) {
|
|
176
187
|
const urlChanged = normaliseUrl(previous.url) !== normaliseUrl(current.url);
|
|
177
188
|
const titleChanged = normaliseTitle(previous.title) !== normaliseTitle(current.title);
|
|
178
|
-
if (!urlChanged && !titleChanged)
|
|
189
|
+
if (!urlChanged && !titleChanged) {
|
|
190
|
+
// Nothing moved in the address bar. This is the case the address and title
|
|
191
|
+
// alone cannot see: a single-page app with a catch-all route lands on the
|
|
192
|
+
// same URL under the same title whether the flow worked or not.
|
|
193
|
+
return signature.compareSignatures(previous.signature, current.signature);
|
|
194
|
+
}
|
|
179
195
|
|
|
180
196
|
const what =
|
|
181
197
|
urlChanged && titleChanged ? 'the page address and title changed'
|
|
@@ -213,6 +229,17 @@ function newObservations(current, previous) {
|
|
|
213
229
|
};
|
|
214
230
|
}
|
|
215
231
|
|
|
232
|
+
// Adds a signature to an accepted baseline that predates the feature, without
|
|
233
|
+
// touching anything else about it - including when it was recorded, which is
|
|
234
|
+
// what the failure report means by "this last worked at".
|
|
235
|
+
function upgradeBaseline(file, key, sig) {
|
|
236
|
+
const all = readBaselines(file);
|
|
237
|
+
const entry = all[key];
|
|
238
|
+
if (!entry || typeof entry !== 'object' || entry.signature || !sig) return false;
|
|
239
|
+
entry.signature = sig;
|
|
240
|
+
return saveBaselines(file, all);
|
|
241
|
+
}
|
|
242
|
+
|
|
216
243
|
// The whole decision in one call: create on first sight, otherwise compare.
|
|
217
244
|
function applyBaseline(file, key, current) {
|
|
218
245
|
const previous = readBaselines(file)[key];
|
|
@@ -221,7 +248,16 @@ function applyBaseline(file, key, current) {
|
|
|
221
248
|
return { status: 'created', message: null };
|
|
222
249
|
}
|
|
223
250
|
const message = compareBaselines(previous, current);
|
|
224
|
-
|
|
251
|
+
if (message) return { status: 'changed', message: message };
|
|
252
|
+
|
|
253
|
+
// A baseline from before signatures existed passes on address and title, and
|
|
254
|
+
// learns its signature here. Doing it on a pass rather than up front means an
|
|
255
|
+
// upgraded project never fails on its first run under the new version.
|
|
256
|
+
if (!previous.signature && current.signature) {
|
|
257
|
+
upgradeBaseline(file, key, current.signature);
|
|
258
|
+
return { status: 'learned', message: null };
|
|
259
|
+
}
|
|
260
|
+
return { status: 'match', message: null };
|
|
225
261
|
}
|
|
226
262
|
|
|
227
263
|
// ---------------------------------------------------------------------------
|
|
@@ -261,6 +297,9 @@ const test = base.test.extend({
|
|
|
261
297
|
await testInfo.attach('kryptheon-observations', {
|
|
262
298
|
body: JSON.stringify({
|
|
263
299
|
url: url,
|
|
300
|
+
// What was actually on screen when it gave up. On a flow that
|
|
301
|
+
// stopped half way this is the most useful line in the report.
|
|
302
|
+
pageShowed: signature.describeSignature(captured),
|
|
264
303
|
failedRequests: fresh.failedRequests.slice(0, MAX_ITEMS),
|
|
265
304
|
consoleErrors: fresh.consoleErrors.slice(0, MAX_ITEMS),
|
|
266
305
|
}),
|
|
@@ -273,6 +312,29 @@ const test = base.test.extend({
|
|
|
273
312
|
|
|
274
313
|
await use(page);
|
|
275
314
|
|
|
315
|
+
// Read the page before anything else looks at the outcome. A test that
|
|
316
|
+
// failed is exactly the one where knowing what was on screen matters most,
|
|
317
|
+
// so this runs whether it passed or not. It is null when the page could
|
|
318
|
+
// not be read at all, and a null is never treated as a difference - not
|
|
319
|
+
// knowing what the page showed is not evidence that it changed.
|
|
320
|
+
//
|
|
321
|
+
// Printed only when asked for, and handed to the CLI through a file:
|
|
322
|
+
// Playwright pipes this worker's descriptors and gives whatever comes out
|
|
323
|
+
// to reporter stdio hooks that this reporter does not implement, so
|
|
324
|
+
// anything printed from here is dropped before it reaches a terminal.
|
|
325
|
+
const captured = await signature.collectSignature(page, failedRequests);
|
|
326
|
+
if (signature.signatureEnabled()) signature.recordSignature(testInfo.title, captured);
|
|
327
|
+
|
|
328
|
+
// Said once per run by the reporter, not once per test: it is a fact about
|
|
329
|
+
// the app, and repeating it for every recording would be noise.
|
|
330
|
+
if (captured && captured.guessedEncoding) {
|
|
331
|
+
try {
|
|
332
|
+
await testInfo.attach('kryptheon-encoding', { body: 'guessed', contentType: 'text/plain' });
|
|
333
|
+
} catch (e) {
|
|
334
|
+
/* a note, never a second failure */
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
276
338
|
// The test's own assertions decide first. A test that already failed keeps
|
|
277
339
|
// its own error, and never contributes a baseline.
|
|
278
340
|
if (testInfo.status !== testInfo.expectedStatus) {
|
|
@@ -280,7 +342,7 @@ const test = base.test.extend({
|
|
|
280
342
|
return;
|
|
281
343
|
}
|
|
282
344
|
|
|
283
|
-
// Passed:
|
|
345
|
+
// Passed: keep what the page ended up as, noise included.
|
|
284
346
|
let current = null;
|
|
285
347
|
try {
|
|
286
348
|
current = {
|
|
@@ -288,6 +350,7 @@ const test = base.test.extend({
|
|
|
288
350
|
title: normaliseTitle(await page.title()),
|
|
289
351
|
consoleErrors: consoleErrors,
|
|
290
352
|
failedRequests: failedRequests,
|
|
353
|
+
signature: captured,
|
|
291
354
|
};
|
|
292
355
|
} catch (e) {
|
|
293
356
|
current = null; // page closed by the test - leave any baseline untouched
|
|
@@ -320,5 +383,6 @@ module.exports = {
|
|
|
320
383
|
normaliseTitle: normaliseTitle,
|
|
321
384
|
compareBaselines: compareBaselines,
|
|
322
385
|
applyBaseline: applyBaseline,
|
|
386
|
+
upgradeBaseline: upgradeBaseline,
|
|
323
387
|
BASELINE_FILE: BASELINE_FILE,
|
|
324
388
|
};
|
package/kryptheon-reporter.js
CHANGED
|
@@ -189,8 +189,11 @@ function translate(message) {
|
|
|
189
189
|
return l.indexOf('Baseline changed:') !== -1;
|
|
190
190
|
});
|
|
191
191
|
const head = lines[headIndex].replace(/^.*Baseline changed:\s*/, '');
|
|
192
|
+
// An allowlist, so a stack trace that happens to follow the message can
|
|
193
|
+
// never leak into the report. Address and title come from the original
|
|
194
|
+
// baseline; the rest name what the page stopped or started showing.
|
|
192
195
|
const details = lines.slice(headIndex + 1).filter(function (l) {
|
|
193
|
-
return /^(Address|Title) (was|now):/.test(l);
|
|
196
|
+
return /^(Address|Title) (was|now):/.test(l) || /^(Headings|Buttons and links|Form fields) that are (gone|new):/.test(l) || /^Requests that failed this time:/.test(l);
|
|
194
197
|
});
|
|
195
198
|
return {
|
|
196
199
|
reason: capitalise(head),
|
|
@@ -406,6 +409,9 @@ function whereToLookLines(obs) {
|
|
|
406
409
|
const lines = [];
|
|
407
410
|
|
|
408
411
|
if (obs.url) lines.push(' - Browser was on: ' + obs.url);
|
|
412
|
+
// Where the flow actually got to, in words. The whole signature stays behind
|
|
413
|
+
// KRYPTHEON_DEBUG_SIGNATURE for when that much detail is wanted.
|
|
414
|
+
if (obs.pageShowed) lines.push(' - Page showed: ' + obs.pageShowed);
|
|
409
415
|
|
|
410
416
|
for (const req of uniq(obs.failedRequests).slice(0, 5)) {
|
|
411
417
|
const meaning = describeStatus(req.status);
|
|
@@ -568,6 +574,8 @@ class KryptheonReporter {
|
|
|
568
574
|
this.previousRuns = [];
|
|
569
575
|
this.records = [];
|
|
570
576
|
this.startedAt = new Date();
|
|
577
|
+
// Set by any test whose page never said what encoding it was in.
|
|
578
|
+
this.sawGuessedEncoding = false;
|
|
571
579
|
}
|
|
572
580
|
|
|
573
581
|
onBegin() {
|
|
@@ -624,6 +632,9 @@ class KryptheonReporter {
|
|
|
624
632
|
}
|
|
625
633
|
|
|
626
634
|
onTestEnd(test, result) {
|
|
635
|
+
if ((result.attachments || []).some(function (a) { return a.name === 'kryptheon-encoding'; })) {
|
|
636
|
+
this.sawGuessedEncoding = true;
|
|
637
|
+
}
|
|
627
638
|
const status =
|
|
628
639
|
result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
|
|
629
640
|
const timestamp = (result.startTime instanceof Date ? result.startTime : new Date()).toISOString();
|
|
@@ -787,6 +798,15 @@ class KryptheonReporter {
|
|
|
787
798
|
'\nSummary: ' + passed + ' working, ' + failed + ' broken.\n' +
|
|
788
799
|
'History saved to ' + path.basename(HISTORY_FILE) + '\n\n'
|
|
789
800
|
);
|
|
801
|
+
|
|
802
|
+
// Once per run, not once per test: it is one fact about the app, and
|
|
803
|
+
// repeating it under every recording would bury the reports themselves.
|
|
804
|
+
if (this.sawGuessedEncoding) {
|
|
805
|
+
process.stdout.write(
|
|
806
|
+
'Note: this page does not declare a character encoding, so text may look\n' +
|
|
807
|
+
'garbled here and in the browser. Add <meta charset="utf-8"> to the page.\n\n'
|
|
808
|
+
);
|
|
809
|
+
}
|
|
790
810
|
}
|
|
791
811
|
}
|
|
792
812
|
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Which environment variables a recording needs, and whether they are there.
|
|
2
|
+
//
|
|
3
|
+
// Why this exists. When a password is typed during recording it is taken out
|
|
4
|
+
// of the spec and replaced with `process.env.KRYPTHEON_PASSWORD || ''`. If
|
|
5
|
+
// nobody ever creates the .env file, that expression quietly evaluates to an
|
|
6
|
+
// empty string: the run submits a blank password, the login fails for the most
|
|
7
|
+
// boring reason imaginable, and the report blames the app. Two hours can go
|
|
8
|
+
// into that before anyone thinks to check the environment.
|
|
9
|
+
//
|
|
10
|
+
// So the names are read out of the spec before anything runs, and a recording
|
|
11
|
+
// whose secret is missing is not run at all - a blank password is never sent.
|
|
12
|
+
|
|
13
|
+
const fs = require('fs');
|
|
14
|
+
const path = require('path');
|
|
15
|
+
|
|
16
|
+
// The exact shape the scrubber writes. Anything else in the file that happens
|
|
17
|
+
// to mention process.env is the author's own business and left alone.
|
|
18
|
+
const SECRET_USE = /process\.env\.([A-Z][A-Z0-9_]*)\s*\|\|\s*(['"])\2/g;
|
|
19
|
+
|
|
20
|
+
/** Every environment variable this recording falls back to an empty string for. */
|
|
21
|
+
function requiredSecrets(source) {
|
|
22
|
+
const out = [];
|
|
23
|
+
const seen = Object.create(null);
|
|
24
|
+
let match;
|
|
25
|
+
SECRET_USE.lastIndex = 0;
|
|
26
|
+
while ((match = SECRET_USE.exec(String(source || '')))) {
|
|
27
|
+
const name = match[1];
|
|
28
|
+
if (seen[name]) continue;
|
|
29
|
+
seen[name] = true;
|
|
30
|
+
out.push(name);
|
|
31
|
+
}
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Reads a .env file into a plain object. Never throws: a missing file simply
|
|
37
|
+
* means nothing was defined there.
|
|
38
|
+
*
|
|
39
|
+
* Values are not expanded or unescaped beyond stripping one layer of quotes -
|
|
40
|
+
* this only has to answer "is it set", and guessing at more would be a second
|
|
41
|
+
* dialect of .env for people to trip over.
|
|
42
|
+
*/
|
|
43
|
+
function readEnvFile(file) {
|
|
44
|
+
const out = Object.create(null);
|
|
45
|
+
let raw;
|
|
46
|
+
try {
|
|
47
|
+
raw = fs.readFileSync(file, 'utf8');
|
|
48
|
+
} catch (e) {
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
// A byte order mark would otherwise become part of the first name.
|
|
52
|
+
if (raw.charCodeAt(0) === 0xfeff) raw = raw.slice(1);
|
|
53
|
+
for (const line of raw.split('\n')) {
|
|
54
|
+
const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
|
55
|
+
if (!match) continue;
|
|
56
|
+
out[match[1]] = match[2].replace(/^(['"])([\s\S]*)\1$/, '$2');
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A value that is present and not blank. An empty variable is not set. */
|
|
62
|
+
function isSet(value) {
|
|
63
|
+
return typeof value === 'string' && value.trim() !== '';
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Which of the names a recording needs are missing, looking at both the real
|
|
68
|
+
* environment and the project's .env - the same two places the test itself
|
|
69
|
+
* will look when it runs.
|
|
70
|
+
*/
|
|
71
|
+
function missingSecrets(names, options) {
|
|
72
|
+
const opts = options || {};
|
|
73
|
+
const env = opts.env || process.env;
|
|
74
|
+
const fromFile = opts.envFile ? readEnvFile(opts.envFile) : {};
|
|
75
|
+
return (names || []).filter((name) => !isSet(env[name]) && !isSet(fromFile[name]));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* How to create the file without a byte order mark.
|
|
80
|
+
*
|
|
81
|
+
* PowerShell's own `Out-File -Encoding utf8` writes one, and so does Notepad,
|
|
82
|
+
* which is why this spells out a command that does not rather than leaving it
|
|
83
|
+
* to whatever the person reaches for first.
|
|
84
|
+
*/
|
|
85
|
+
function howToWriteEnv(names, platform) {
|
|
86
|
+
const line = names.map((name) => name + '=yourpassword').join('\\n');
|
|
87
|
+
if ((platform || process.platform) === 'win32') {
|
|
88
|
+
return [
|
|
89
|
+
' In PowerShell, this writes it without a byte order mark:',
|
|
90
|
+
' [IO.File]::WriteAllText("$PWD\\.env", "' + line + '`n")',
|
|
91
|
+
'',
|
|
92
|
+
' Avoid Out-File and Notepad here: both add a mark that makes the first',
|
|
93
|
+
' line unreadable, and the password would silently come out blank.',
|
|
94
|
+
];
|
|
95
|
+
}
|
|
96
|
+
return [' For example:', " printf '" + line.replace(/\\n/g, "\\n") + "\\n' > .env"];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** What to tell somebody who has not created the file yet. */
|
|
100
|
+
function missingSecretLines(specName, names, options) {
|
|
101
|
+
const opts = options || {};
|
|
102
|
+
const many = names.length > 1;
|
|
103
|
+
const lines = [
|
|
104
|
+
' Skipped ' + specName,
|
|
105
|
+
' This recording needs a password' + (many ? ' and other secrets' : '') + ', and there is none set.',
|
|
106
|
+
'',
|
|
107
|
+
];
|
|
108
|
+
if (opts.examplePath) {
|
|
109
|
+
lines.push(' There is a ' + opts.examplePath + ' here already. Put the real value in it');
|
|
110
|
+
lines.push(' and save it as .env, in the same folder.');
|
|
111
|
+
} else {
|
|
112
|
+
lines.push(' Create a .env file in this folder containing:');
|
|
113
|
+
for (const name of names) lines.push(' ' + name + '=yourpassword');
|
|
114
|
+
}
|
|
115
|
+
lines.push('');
|
|
116
|
+
for (const line of howToWriteEnv(names, opts.platform)) lines.push(line);
|
|
117
|
+
lines.push('');
|
|
118
|
+
lines.push(' Without it the recording would sign in with a blank password and');
|
|
119
|
+
lines.push(' fail for a reason that has nothing to do with your app, so it was');
|
|
120
|
+
lines.push(' not run. Keep .env out of version control.');
|
|
121
|
+
return lines;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Writes .env.example so nobody has to get the encoding right by hand.
|
|
126
|
+
*
|
|
127
|
+
* Plain UTF-8, no byte order mark. Never overwrites: the file may already hold
|
|
128
|
+
* names from an earlier recording, and clobbering it would lose them.
|
|
129
|
+
*/
|
|
130
|
+
function writeEnvExample(dir, names) {
|
|
131
|
+
const file = path.join(dir, '.env.example');
|
|
132
|
+
if (fs.existsSync(file)) return null;
|
|
133
|
+
const body = (names || []).map((name) => name + '=yourpassword').join('\n') + '\n';
|
|
134
|
+
try {
|
|
135
|
+
fs.writeFileSync(file, body, { encoding: 'utf8' });
|
|
136
|
+
return file;
|
|
137
|
+
} catch (e) {
|
|
138
|
+
return null; // a convenience, never a reason to fail a recording
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Everything kryptheon leaves in a project that should not be committed: the
|
|
143
|
+
// real password, and three files that are machine state rather than source.
|
|
144
|
+
const GITIGNORE_ENTRIES = ['.env', 'kryptheon-baselines.json', 'kryptheon-history.jsonl', 'test-results/'];
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Adds the missing entries to an existing .gitignore. Returns what it added.
|
|
148
|
+
*
|
|
149
|
+
* Only touches a file that is already there - creating one in a project that
|
|
150
|
+
* has deliberately gone without is not this tool's decision to make.
|
|
151
|
+
*/
|
|
152
|
+
function updateGitignore(dir, entries) {
|
|
153
|
+
const file = path.join(dir, '.gitignore');
|
|
154
|
+
let raw;
|
|
155
|
+
try {
|
|
156
|
+
raw = fs.readFileSync(file, { encoding: 'utf8' });
|
|
157
|
+
} catch (e) {
|
|
158
|
+
return [];
|
|
159
|
+
}
|
|
160
|
+
if (raw.charCodeAt(0) === 0xfeff) raw = raw.slice(1);
|
|
161
|
+
const present = new Set(
|
|
162
|
+
raw
|
|
163
|
+
.split('\n')
|
|
164
|
+
.map((line) => line.trim().replace(/^\/+/, '').replace(/\/+$/, ''))
|
|
165
|
+
.filter(Boolean),
|
|
166
|
+
);
|
|
167
|
+
const wanted = entries || GITIGNORE_ENTRIES;
|
|
168
|
+
const missing = wanted.filter((entry) => !present.has(entry.replace(/\/+$/, '')));
|
|
169
|
+
if (!missing.length) return [];
|
|
170
|
+
const needsNewline = raw.length && !raw.endsWith('\n');
|
|
171
|
+
const block = (needsNewline ? '\n' : '') + '\n# kryptheon\n' + missing.join('\n') + '\n';
|
|
172
|
+
try {
|
|
173
|
+
fs.appendFileSync(file, block, { encoding: 'utf8' });
|
|
174
|
+
return missing;
|
|
175
|
+
} catch (e) {
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
module.exports = {
|
|
181
|
+
requiredSecrets: requiredSecrets,
|
|
182
|
+
readEnvFile: readEnvFile,
|
|
183
|
+
missingSecrets: missingSecrets,
|
|
184
|
+
missingSecretLines: missingSecretLines,
|
|
185
|
+
howToWriteEnv: howToWriteEnv,
|
|
186
|
+
writeEnvExample: writeEnvExample,
|
|
187
|
+
updateGitignore: updateGitignore,
|
|
188
|
+
GITIGNORE_ENTRIES: GITIGNORE_ENTRIES,
|
|
189
|
+
isSet: isSet,
|
|
190
|
+
envFileFor: (dir) => path.join(dir, '.env'),
|
|
191
|
+
};
|