kryptheon 0.1.44 → 0.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -0
- package/bin/kryptheon.js +3462 -3242
- package/kryptheon-dashboard-page.js +116 -11
- package/kryptheon-dashboard.html +8 -2
- package/kryptheon-dashboard.js +1132 -1009
- package/kryptheon-edge.js +792 -0
- package/kryptheon-look.js +1 -1
- package/kryptheon-probe-fixture.js +303 -0
- package/kryptheon-probe-reporter.js +29 -0
- package/kryptheon-probe.config.js +37 -0
- package/kryptheon-probe.js +317 -0
- package/kryptheon-reliability.js +599 -0
- package/kryptheon-replay.js +0 -0
- package/package.json +12 -4
package/kryptheon-look.js
CHANGED
|
@@ -44,7 +44,7 @@ function look(root, runners, options) {
|
|
|
44
44
|
const wanted = (diff.first ? ['frontend', 'regression', 'data'] : diff.checks).slice();
|
|
45
45
|
for (const id of opts.force || []) if (!wanted.includes(id)) wanted.push(id);
|
|
46
46
|
const plan = wanted.map((id) => {
|
|
47
|
-
if (!available.has(id)) return { id: id, state: '
|
|
47
|
+
if (!available.has(id)) return { id: id, state: 'coming soon' };
|
|
48
48
|
if (!enabled[id]) return { id: id, state: 'switched off in the dashboard' };
|
|
49
49
|
if (id === 'data') return { id: id, state: 'needs your database - run npx kryptheon-night' };
|
|
50
50
|
if (id === 'regression' && !before.recordings) return { id: id, state: 'no recordings yet - npx kryptheon record <url>' };
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
// The fixture the probe copies run under - never the recordings themselves.
|
|
2
|
+
//
|
|
3
|
+
// A probe is a copy of a recording, changed on purpose: one field given an
|
|
4
|
+
// unusual value (kryptheon-edge.js), or one step clicked twice. What matters
|
|
5
|
+
// in such a run is not whether it passed - a form that refuses a bad value is
|
|
6
|
+
// doing its job - but what the app did: did the page throw, did the server
|
|
7
|
+
// answer 5xx, what did the app write, and did a tag typed into a field come
|
|
8
|
+
// back as real markup. This writes all of that to one JSON file per run.
|
|
9
|
+
//
|
|
10
|
+
// It never touches the baselines or the history: a probe is not a check of
|
|
11
|
+
// the recorded flow, and must never change what "the last time it worked"
|
|
12
|
+
// means for one.
|
|
13
|
+
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
16
|
+
const base = require('@playwright/test');
|
|
17
|
+
|
|
18
|
+
const OUT = process.env.KRYPTHEON_PROBE_OUT || '';
|
|
19
|
+
const GAP_MS = Number(process.env.KRYPTHEON_PROBE_GAP_MS || 1000);
|
|
20
|
+
// After a "too many requests", the app is asked to wait this long before the
|
|
21
|
+
// next run, so one 429 does not turn every run after it into another.
|
|
22
|
+
const BACKOFF_MS = Number(process.env.KRYPTHEON_PROBE_BACKOFF_MS || 10000);
|
|
23
|
+
const MAX_BODY = 20000;
|
|
24
|
+
const WRITE = /^(POST|PUT|PATCH|DELETE)$/;
|
|
25
|
+
// Values under these names are never kept, whatever request they were in.
|
|
26
|
+
const SECRET_KEY = /(pass(word|code)?|secret|token|otp|card|cvv|cvc|\bpin\b|api[_-]?key|authorization)/i;
|
|
27
|
+
const MARKER = require('./kryptheon-probe.js').MARKER;
|
|
28
|
+
|
|
29
|
+
function sleep(ms) {
|
|
30
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Every value the recording reads from .env. They are what a sign-in sends,
|
|
34
|
+
// and they must not reach a results file in any form.
|
|
35
|
+
function secretValues() {
|
|
36
|
+
return String(process.env.KRYPTHEON_PROBE_SECRETS || '')
|
|
37
|
+
.split(',')
|
|
38
|
+
.map((name) => name.trim())
|
|
39
|
+
.filter(Boolean)
|
|
40
|
+
.map((name) => process.env[name])
|
|
41
|
+
.filter((v) => typeof v === 'string' && v.length >= 3);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function hideValues(text, secrets) {
|
|
45
|
+
let s = String(text == null ? '' : text);
|
|
46
|
+
for (const v of secrets) s = s.split(v).join('[hidden]');
|
|
47
|
+
// encodeURIComponent: a form post carries the password encoded.
|
|
48
|
+
for (const v of secrets) s = s.split(encodeURIComponent(v)).join('[hidden]');
|
|
49
|
+
return s;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function hideKeys(value) {
|
|
53
|
+
if (Array.isArray(value)) return value.map(hideKeys);
|
|
54
|
+
if (value && typeof value === 'object') {
|
|
55
|
+
const out = {};
|
|
56
|
+
for (const k of Object.keys(value)) out[k] = SECRET_KEY.test(k) ? '[hidden]' : hideKeys(value[k]);
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A request body as it may be kept: secrets taken out, by value and by name.
|
|
64
|
+
* JSON and form posts are kept as data, so a later run can be compared field
|
|
65
|
+
* by field; anything else only as text.
|
|
66
|
+
*/
|
|
67
|
+
function keptBody(raw, contentType, secrets) {
|
|
68
|
+
if (raw == null || raw === '') return { text: '' };
|
|
69
|
+
const text = String(raw);
|
|
70
|
+
if (/json/i.test(contentType || '') || /^\s*[[{]/.test(text)) {
|
|
71
|
+
try {
|
|
72
|
+
const data = hideKeys(JSON.parse(text));
|
|
73
|
+
return { data: JSON.parse(hideValues(JSON.stringify(data), secrets)) };
|
|
74
|
+
} catch (err) {
|
|
75
|
+
/* not JSON after all */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (/x-www-form-urlencoded/i.test(contentType || '')) {
|
|
79
|
+
const data = {};
|
|
80
|
+
for (const [k, v] of new URLSearchParams(text)) data[k] = SECRET_KEY.test(k) ? '[hidden]' : v;
|
|
81
|
+
return { data: JSON.parse(hideValues(JSON.stringify(data), secrets)) };
|
|
82
|
+
}
|
|
83
|
+
return { text: hideValues(text, secrets).slice(0, MAX_BODY) };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function where(url) {
|
|
87
|
+
try {
|
|
88
|
+
const u = new URL(url);
|
|
89
|
+
return { host: u.host, origin: u.origin, path: u.pathname };
|
|
90
|
+
} catch (err) {
|
|
91
|
+
return { host: '', origin: '', path: String(url || '').split('?')[0] };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function firstLine(text) {
|
|
96
|
+
return String(text || '').replace(/\x1b\[[0-9;]*[A-Za-z]/g, '').split(/\r?\n/).map((l) => l.trim()).filter(Boolean)[0] || '';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Every element the app made out of a probe tag. An attribute or a text node
|
|
100
|
+
// holding the same characters is not one: only markup the browser parsed is.
|
|
101
|
+
async function findMarkers(page) {
|
|
102
|
+
try {
|
|
103
|
+
return await page.evaluate((attr) => Array.prototype.slice.call(document.querySelectorAll('[' + attr + ']')).slice(0, 10).map((el) => ({
|
|
104
|
+
id: el.getAttribute(attr),
|
|
105
|
+
tag: el.tagName.toLowerCase(),
|
|
106
|
+
html: el.outerHTML.slice(0, 200),
|
|
107
|
+
path: location.pathname,
|
|
108
|
+
})), MARKER);
|
|
109
|
+
} catch (err) {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const test = base.test.extend({
|
|
115
|
+
page: async ({ page }, use, testInfo) => {
|
|
116
|
+
const secrets = secretValues();
|
|
117
|
+
const events = [];
|
|
118
|
+
const fields = {};
|
|
119
|
+
let seq = 0;
|
|
120
|
+
// Returns the event as kept, so what arrives later (a write's answer) is
|
|
121
|
+
// added to the very object that is saved - not to a copy of it.
|
|
122
|
+
const add = (e) => {
|
|
123
|
+
const kept = Object.assign({ seq: ++seq }, e);
|
|
124
|
+
events.push(kept);
|
|
125
|
+
return kept;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// A gap before every run, and a longer one after the app said "too many
|
|
129
|
+
// requests": the app is being asked many times in a row, and a rate limit
|
|
130
|
+
// is the app protecting itself, not a problem in it.
|
|
131
|
+
let backoff = false;
|
|
132
|
+
try {
|
|
133
|
+
backoff = Boolean(OUT) && fs.existsSync(path.join(OUT, 'rate-limited'));
|
|
134
|
+
} catch (err) {
|
|
135
|
+
backoff = false;
|
|
136
|
+
}
|
|
137
|
+
await sleep(backoff ? BACKOFF_MS : GAP_MS);
|
|
138
|
+
|
|
139
|
+
page.on('pageerror', (err) => add({ kind: 'error', text: hideValues(firstLine((err && err.message) || String(err)), secrets).slice(0, 300) }));
|
|
140
|
+
page.on('response', (res) => {
|
|
141
|
+
const req = res.request();
|
|
142
|
+
const method = req.method();
|
|
143
|
+
const status = res.status();
|
|
144
|
+
const at = where(res.url());
|
|
145
|
+
if (status === 429) {
|
|
146
|
+
add({ kind: 'limited', method: method, host: at.host, path: at.path, status: status });
|
|
147
|
+
try {
|
|
148
|
+
if (OUT) fs.writeFileSync(path.join(OUT, 'rate-limited'), '1');
|
|
149
|
+
} catch (err) {
|
|
150
|
+
/* a note, never a failure */
|
|
151
|
+
}
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
if (status >= 500) add({ kind: 'server', method: method, host: at.host, path: at.path, status: status });
|
|
155
|
+
if (WRITE.test(method)) {
|
|
156
|
+
let raw = null;
|
|
157
|
+
try {
|
|
158
|
+
raw = req.postData();
|
|
159
|
+
} catch (err) {
|
|
160
|
+
raw = null;
|
|
161
|
+
}
|
|
162
|
+
const type = (req.headers() || {})['content-type'] || '';
|
|
163
|
+
const kept = keptBody(raw, type, secrets);
|
|
164
|
+
// A write that carried a secret is the sign-in, not data the app
|
|
165
|
+
// keeps - it is never listed as test data left behind.
|
|
166
|
+
const signIn = JSON.stringify(kept).indexOf('[hidden]') !== -1;
|
|
167
|
+
const event = Object.assign({ kind: 'write', method: method, host: at.host, origin: at.origin, path: at.path, status: status, signIn: signIn }, kept);
|
|
168
|
+
const placed = add(event);
|
|
169
|
+
// What the server answered - often the record it saved, with its id.
|
|
170
|
+
// That is the proof that two saves made two records, and of what was
|
|
171
|
+
// really stored. Never for the sign-in, whose answer is a session.
|
|
172
|
+
// Read after the event is placed, so its order among the marks is
|
|
173
|
+
// the order the answer arrived in.
|
|
174
|
+
if (!signIn && /json/i.test((res.headers() || {})['content-type'] || '')) {
|
|
175
|
+
const reading = res.text().then((text) => {
|
|
176
|
+
if (text && text.length <= MAX_BODY * 5) {
|
|
177
|
+
try {
|
|
178
|
+
placed.answer = JSON.parse(hideValues(JSON.stringify(hideKeys(JSON.parse(text))), secrets));
|
|
179
|
+
} catch (err) {
|
|
180
|
+
/* not JSON after all: no answer kept */
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}, () => {}).then(() => answers.delete(reading));
|
|
184
|
+
answers.add(reading);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
// Answers still being read. Counted before a mark and before the end.
|
|
189
|
+
const answers = new Set();
|
|
190
|
+
const answersRead = async () => {
|
|
191
|
+
for (let waited = 0; answers.size && waited < 5000; waited += 100) await sleep(100);
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
// Called from inside a probe copy, around the steps that matter.
|
|
195
|
+
page.__kxMark = async (name) => {
|
|
196
|
+
// Let what the last step started arrive before it is counted: a moment
|
|
197
|
+
// for a second click's request to leave, then every write still on its
|
|
198
|
+
// way. A load state alone is not enough - once a page has gone quiet it
|
|
199
|
+
// says so at once, even while a new save is in flight.
|
|
200
|
+
await sleep(300);
|
|
201
|
+
for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
|
|
202
|
+
await answersRead();
|
|
203
|
+
try {
|
|
204
|
+
await page.waitForLoadState('networkidle', { timeout: 3000 });
|
|
205
|
+
} catch (err) {
|
|
206
|
+
/* a page that never goes quiet is still marked */
|
|
207
|
+
}
|
|
208
|
+
add({ kind: 'mark', name: String(name) });
|
|
209
|
+
};
|
|
210
|
+
// What kind of field a recorded fill typed into, read from the page
|
|
211
|
+
// itself: whether it is required, a number, a date, how long it may be.
|
|
212
|
+
page.__kxField = async (line, locator) => {
|
|
213
|
+
try {
|
|
214
|
+
fields[line] = await locator.first().evaluate((el) => {
|
|
215
|
+
const label = (el.labels && el.labels[0] && el.labels[0].textContent) || '';
|
|
216
|
+
return {
|
|
217
|
+
tag: el.tagName.toLowerCase(),
|
|
218
|
+
type: (el.getAttribute('type') || '').toLowerCase(),
|
|
219
|
+
required: el.required === true || el.getAttribute('aria-required') === 'true' || /\*\s*$/.test(label.trim()),
|
|
220
|
+
min: el.getAttribute('min'),
|
|
221
|
+
maxLength: el.maxLength > 0 ? el.maxLength : null,
|
|
222
|
+
pattern: el.getAttribute('pattern'),
|
|
223
|
+
label: label.replace(/\s+/g, ' ').trim().slice(0, 80),
|
|
224
|
+
placeholder: (el.getAttribute('placeholder') || '').slice(0, 80),
|
|
225
|
+
name: (el.getAttribute('name') || el.id || '').slice(0, 80),
|
|
226
|
+
};
|
|
227
|
+
}, null, { timeout: 5000 });
|
|
228
|
+
} catch (err) {
|
|
229
|
+
fields[line] = null; // not knowing is never read as "not required"
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
// Writes still on their way. A recording often ends right after the step
|
|
234
|
+
// that saves, and the answer to that save is the proof - so the run waits
|
|
235
|
+
// for it before it looks at anything or opens the page again, which would
|
|
236
|
+
// cut the answer off.
|
|
237
|
+
const inFlight = new Set();
|
|
238
|
+
page.on('request', (req) => {
|
|
239
|
+
if (WRITE.test(req.method())) inFlight.add(req);
|
|
240
|
+
});
|
|
241
|
+
const settled = (req) => inFlight.delete(req);
|
|
242
|
+
page.on('requestfinished', settled);
|
|
243
|
+
page.on('requestfailed', settled);
|
|
244
|
+
|
|
245
|
+
const started = Date.now();
|
|
246
|
+
await use(page);
|
|
247
|
+
|
|
248
|
+
for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
|
|
249
|
+
await answersRead();
|
|
250
|
+
try {
|
|
251
|
+
await page.waitForLoadState('networkidle', { timeout: 3000 });
|
|
252
|
+
} catch (err) {
|
|
253
|
+
/* a page that never goes quiet is still read */
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Markup made from a probe tag: on the page the flow ended on, and on the
|
|
257
|
+
// same address opened again - which is what proves it was saved and is
|
|
258
|
+
// shown to whoever opens it next. A GET, never a reload: reloading a page
|
|
259
|
+
// that came from a form post would send the form again.
|
|
260
|
+
const markers = { live: await findMarkers(page), reopened: [] };
|
|
261
|
+
try {
|
|
262
|
+
const url = page.url();
|
|
263
|
+
if (/^https?:/.test(url)) {
|
|
264
|
+
await page.goto(url, { waitUntil: 'load', timeout: 15000 });
|
|
265
|
+
try {
|
|
266
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 });
|
|
267
|
+
} catch (err) {
|
|
268
|
+
/* the markers are read either way */
|
|
269
|
+
}
|
|
270
|
+
markers.reopened = await findMarkers(page);
|
|
271
|
+
}
|
|
272
|
+
} catch (err) {
|
|
273
|
+
/* the page could not be opened again; what was live still counts */
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const error = testInfo.error || (testInfo.errors && testInfo.errors[0]) || null;
|
|
277
|
+
const file = path.basename(testInfo.file);
|
|
278
|
+
let failedLine = null;
|
|
279
|
+
if (error && error.stack) {
|
|
280
|
+
const m = new RegExp(file.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ':(\\d+)').exec(error.stack);
|
|
281
|
+
if (m) failedLine = Number(m[1]);
|
|
282
|
+
}
|
|
283
|
+
const result = {
|
|
284
|
+
id: file.replace(/\.kxprobe\.js$/, ''),
|
|
285
|
+
status: testInfo.status,
|
|
286
|
+
error: error ? hideValues(firstLine(error.message), secrets).slice(0, 300) : null,
|
|
287
|
+
failedLine: failedLine,
|
|
288
|
+
events: events,
|
|
289
|
+
fields: fields,
|
|
290
|
+
markers: markers,
|
|
291
|
+
ms: Date.now() - started,
|
|
292
|
+
};
|
|
293
|
+
if (OUT) {
|
|
294
|
+
try {
|
|
295
|
+
fs.writeFileSync(path.join(OUT, result.id + '.json'), JSON.stringify(result), 'utf8');
|
|
296
|
+
} catch (err) {
|
|
297
|
+
/* the run is then reported as not finished, never as clean */
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
module.exports = { test: test, expect: base.expect, keptBody: keptBody, hideValues: hideValues, MARKER: MARKER };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Says one line per probe run as it ends, in a form kryptheon-probe.js reads
|
|
2
|
+
// back and turns into words: "KXPROBE <done> <total> <id> <status>". Nothing
|
|
3
|
+
// else - a probe that fails is usually the app refusing a bad value, and a
|
|
4
|
+
// failure report for it would say the opposite of what happened.
|
|
5
|
+
|
|
6
|
+
class ProbeReporter {
|
|
7
|
+
onBegin(config, suite) {
|
|
8
|
+
this.total = suite.allTests().length;
|
|
9
|
+
this.done = 0;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
onTestEnd(test, result) {
|
|
13
|
+
this.done += 1;
|
|
14
|
+
const id = String(test.location.file).replace(/^.*[\\/]/, '').replace(/\.kxprobe\.js$/, '');
|
|
15
|
+
console.log('KXPROBE ' + this.done + ' ' + this.total + ' ' + id + ' ' + result.status);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
onError(error) {
|
|
19
|
+
// The run itself could not start - a broken copy, a missing browser.
|
|
20
|
+
const text = String((error && error.message) || error).split(/\r?\n/)[0];
|
|
21
|
+
console.log('KXPROBE-ERROR ' + text.slice(0, 300));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
printsToStdio() {
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
module.exports = ProbeReporter;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
// The Playwright config for probe copies (kryptheon-probe.js). Kept apart from
|
|
3
|
+
// playwright.config.js on purpose: probes live in the store, not in tests/;
|
|
4
|
+
// they report through their own reporter; and their screenshots and traces
|
|
5
|
+
// go to a folder of their own - Playwright empties its output folder on every
|
|
6
|
+
// run, and the last check's evidence must not be wiped by a probe.
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const { defineConfig } = require('@playwright/test');
|
|
9
|
+
|
|
10
|
+
const PACKAGE_DIR = __dirname;
|
|
11
|
+
const USER_DIR = process.cwd();
|
|
12
|
+
const PROBE_DIR = process.env.KRYPTHEON_PROBE_DIR || path.join(USER_DIR, 'tests');
|
|
13
|
+
|
|
14
|
+
// The same .env reading as playwright.config.js, for the same reasons.
|
|
15
|
+
const secrets = require(path.join(PACKAGE_DIR, 'kryptheon-secrets.js'));
|
|
16
|
+
const values = secrets.readEnvFile(path.join(USER_DIR, '.env'));
|
|
17
|
+
for (const name of Object.keys(values)) {
|
|
18
|
+
if (!secrets.isSet(process.env[name])) process.env[name] = values[name];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
module.exports = defineConfig({
|
|
22
|
+
testDir: PROBE_DIR,
|
|
23
|
+
testMatch: ['**/*.kxprobe.js'],
|
|
24
|
+
outputDir: path.join(PROBE_DIR, 'output'),
|
|
25
|
+
fullyParallel: false,
|
|
26
|
+
workers: 1,
|
|
27
|
+
retries: 0,
|
|
28
|
+
// Room for the gap before each run and for opening the page again after it.
|
|
29
|
+
timeout: 60000,
|
|
30
|
+
reporter: [[path.join(PACKAGE_DIR, 'kryptheon-probe-reporter.js')]],
|
|
31
|
+
use: {
|
|
32
|
+
headless: true,
|
|
33
|
+
channel: 'chromium',
|
|
34
|
+
screenshot: 'off',
|
|
35
|
+
trace: 'off',
|
|
36
|
+
},
|
|
37
|
+
});
|