kryptheon 0.1.46 → 0.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/bin/kryptheon.js +63 -37
- package/kryptheon-dashboard-page.js +20 -6
- package/kryptheon-dashboard.html +1 -1
- package/kryptheon-dashboard.js +44 -33
- package/kryptheon-edge.js +28 -13
- package/kryptheon-probe-fixture.js +39 -3
- package/kryptheon-probe.js +45 -2
- package/kryptheon-reliability.js +599 -0
- package/kryptheon-replay.js +0 -0
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -265,6 +265,38 @@ it notices on the first replay - you are warned and must answer yourself.
|
|
|
265
265
|
`npx kryptheon recheck <id>` tries a finding's value again: FIXED, or STILL
|
|
266
266
|
OPEN with the new proof.
|
|
267
267
|
|
|
268
|
+
## Reliability
|
|
269
|
+
|
|
270
|
+
```
|
|
271
|
+
npx kryptheon reliability
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
Asks two questions of your recorded flows:
|
|
275
|
+
|
|
276
|
+
- **Does it work every time?** Each flow is replayed three times. One that
|
|
277
|
+
fails and then works again, with nothing changed, is reported - with the
|
|
278
|
+
failed run as proof. It is called a problem only when your app answered
|
|
279
|
+
differently in the failed run (a 5xx or a page error the good runs did not
|
|
280
|
+
get); without that, the recording's own timing could be the cause, and it
|
|
281
|
+
is said as a thing to check. A flow that fails all three times is broken,
|
|
282
|
+
which `npx kryptheon check` reports. One that works once and then stops is
|
|
283
|
+
a thing to check: the first run's own data may be in the way.
|
|
284
|
+
- **What happens on a double-click?** Each step that saves (Save, Add,
|
|
285
|
+
Create, Submit...) is clicked twice, quickly. When that saves twice - more
|
|
286
|
+
successful POSTs to the same place than one click makes - it is reported
|
|
287
|
+
with both answers, and the two record ids the server named when it names
|
|
288
|
+
them. A server that answers both clicks with the same record saved it once,
|
|
289
|
+
and that is not reported. A double save is tried a second time before it is
|
|
290
|
+
called a problem.
|
|
291
|
+
|
|
292
|
+
Slower networks are not tried yet.
|
|
293
|
+
|
|
294
|
+
Like the input check, **it adds data to your app** - your flows' own saves,
|
|
295
|
+
replayed - and leaves it there. It runs only when you say yes, with the same
|
|
296
|
+
rules: `--yes` only for an app on this computer, at most 40 runs, Stop in the
|
|
297
|
+
dashboard. `npx kryptheon recheck <id>` replays a flaky flow five times (FIXED
|
|
298
|
+
only if all five work), or double-clicks the step again.
|
|
299
|
+
|
|
268
300
|
## What goes where
|
|
269
301
|
|
|
270
302
|
Only your recordings go in your project, in `tests/`. Everything Kryptheon
|
package/bin/kryptheon.js
CHANGED
|
@@ -101,6 +101,7 @@ function usage() {
|
|
|
101
101
|
console.log(' npx kryptheon recheck [id] prove a fix: run the check again, and say FIXED or not');
|
|
102
102
|
console.log(' npx kryptheon code read your frontend code for text inserted as HTML');
|
|
103
103
|
console.log(' npx kryptheon edge try unusual values in your recorded forms (adds test data; asks first)');
|
|
104
|
+
console.log(' npx kryptheon reliability replay each flow 3 times, double-click what saves (adds test data; asks first)');
|
|
104
105
|
console.log(' npx kryptheon dashboard the page above (also: npx kryptheon, in a terminal)');
|
|
105
106
|
console.log('');
|
|
106
107
|
console.log(' add --quiet to check for one line when everything passes');
|
|
@@ -1938,10 +1939,10 @@ function probeFlows() {
|
|
|
1938
1939
|
* that are not this computer - none for --yes, which is only ever for this
|
|
1939
1940
|
* computer - or null when the answer is no.
|
|
1940
1941
|
*/
|
|
1941
|
-
async function probeConsent(what, addresses, yes) {
|
|
1942
|
+
async function probeConsent(what, addresses, yes, words) {
|
|
1942
1943
|
const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
|
|
1943
1944
|
console.log('');
|
|
1944
|
-
for (const line of probe.consentLines(what, addresses)) console.log(' ' + line);
|
|
1945
|
+
for (const line of probe.consentLines(what, addresses, words)) console.log(' ' + line);
|
|
1945
1946
|
console.log('');
|
|
1946
1947
|
const live = probe.liveLooking(addresses);
|
|
1947
1948
|
if (yes && !live.length) return [];
|
|
@@ -1962,27 +1963,47 @@ async function probeConsent(what, addresses, yes) {
|
|
|
1962
1963
|
return said ? live : null;
|
|
1963
1964
|
}
|
|
1964
1965
|
|
|
1965
|
-
|
|
1966
|
+
// The checks that run changed copies of the recordings, and what is said
|
|
1967
|
+
// about each before it starts. Both add data to the app; both ask first.
|
|
1968
|
+
const PROBE_CHECKS = {
|
|
1969
|
+
edge: {
|
|
1970
|
+
module: 'kryptheon-edge.js',
|
|
1971
|
+
what: 'Input and edge cases: replays your recorded forms with one field at a time given an unusual value ' +
|
|
1972
|
+
'(empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most 40 runs.',
|
|
1973
|
+
none: 'There is no recording to try values in. Record a flow that fills a form:',
|
|
1974
|
+
},
|
|
1975
|
+
reliability: {
|
|
1976
|
+
module: 'kryptheon-reliability.js',
|
|
1977
|
+
what: 'Reliability: replays each recorded flow three times, and clicks each step that saves (Save, Add, Create...) twice, quickly. At most 40 runs.',
|
|
1978
|
+
none: 'There is no recording to replay. Record one of your flows:',
|
|
1979
|
+
// It types nothing of its own, so nothing it saves carries the marker.
|
|
1980
|
+
words: { marker: false },
|
|
1981
|
+
},
|
|
1982
|
+
};
|
|
1983
|
+
|
|
1984
|
+
function probeModule(check) {
|
|
1985
|
+
return require(path.join(PACKAGE_DIR, PROBE_CHECKS[check].module));
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
async function probeCommand(check, args) {
|
|
1966
1989
|
const here = project.inspectProject(USER_DIR);
|
|
1967
1990
|
if (!here.ok) {
|
|
1968
1991
|
for (const line of project.noProjectLines(here)) console.error(line);
|
|
1969
1992
|
return 1;
|
|
1970
1993
|
}
|
|
1971
1994
|
if (!openStore()) return 1;
|
|
1972
|
-
const
|
|
1995
|
+
const runner = probeModule(check);
|
|
1973
1996
|
const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
|
|
1974
1997
|
const { flows } = probeFlows();
|
|
1975
1998
|
if (!flows.length) {
|
|
1976
1999
|
console.log('');
|
|
1977
|
-
console.log('
|
|
2000
|
+
console.log(' ' + PROBE_CHECKS[check].none);
|
|
1978
2001
|
console.log(' npx kryptheon record <url>');
|
|
1979
2002
|
console.log('');
|
|
1980
2003
|
return 2;
|
|
1981
2004
|
}
|
|
1982
|
-
const addresses =
|
|
1983
|
-
const
|
|
1984
|
-
'(empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most ' + probe.MAX_RUNS + ' runs.';
|
|
1985
|
-
const confirmed = await probeConsent(what, addresses, args.includes('--yes'));
|
|
2005
|
+
const addresses = probe.knownAddresses(USER_DIR, flows.map((f) => f.source));
|
|
2006
|
+
const confirmed = await probeConsent(PROBE_CHECKS[check].what, addresses, args.includes('--yes'), PROBE_CHECKS[check].words);
|
|
1986
2007
|
if (!confirmed) return 2;
|
|
1987
2008
|
if (!ensureBrowser()) return 2;
|
|
1988
2009
|
|
|
@@ -1991,14 +2012,14 @@ async function edgeCommand(args) {
|
|
|
1991
2012
|
if (control) control.stop();
|
|
1992
2013
|
};
|
|
1993
2014
|
process.once('SIGINT', onSigint);
|
|
1994
|
-
const report = await
|
|
2015
|
+
const report = await runner.run(USER_DIR, flows, {
|
|
1995
2016
|
say: (line) => console.log(' ' + line),
|
|
1996
2017
|
onControl: (c) => (control = c),
|
|
1997
2018
|
confirmed: confirmed,
|
|
1998
2019
|
});
|
|
1999
2020
|
process.removeListener('SIGINT', onSigint);
|
|
2000
|
-
|
|
2001
|
-
for (const line of
|
|
2021
|
+
runner.save(USER_DIR, report);
|
|
2022
|
+
for (const line of runner.summaryLines(report)) console.log(' ' + line);
|
|
2002
2023
|
if (report.findings.some((f) => f.status === 'confirmed')) return 1;
|
|
2003
2024
|
return report.flows.some((f) => f.state === 'tested') ? 0 : 2;
|
|
2004
2025
|
}
|
|
@@ -2385,19 +2406,23 @@ const VERDICT_WORDS = {
|
|
|
2385
2406
|
'could not confirm': 'COULD NOT CONFIRM',
|
|
2386
2407
|
};
|
|
2387
2408
|
|
|
2388
|
-
async function
|
|
2409
|
+
async function probeRecheckCommand(target, options) {
|
|
2389
2410
|
const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
|
|
2390
2411
|
const { flows } = probeFlows();
|
|
2391
2412
|
const file = String(target.where).replace(/:\d+$/, '');
|
|
2392
2413
|
const flow = flows.find((f) => f.file === file);
|
|
2393
|
-
const addresses =
|
|
2394
|
-
const
|
|
2395
|
-
const
|
|
2414
|
+
const addresses = probe.knownAddresses(USER_DIR, flow ? [flow.source] : []);
|
|
2415
|
+
const name = '"' + (flow ? flow.title : file) + '"';
|
|
2416
|
+
const what = 'Re-check ' + target.id + ': ' + (
|
|
2417
|
+
target.check === 'edge' ? 'replays ' + name + ' as recorded, then once more with the value that found this. Two runs.'
|
|
2418
|
+
: target.kind === 'flaky' ? 'replays ' + name + ' five times. It is fixed only if all five work.'
|
|
2419
|
+
: 'replays ' + name + ' once, then once more with this step double-clicked. Two runs.');
|
|
2420
|
+
const confirmed = await probeConsent(what, addresses, Boolean(options && options.yes), PROBE_CHECKS[target.check].words);
|
|
2396
2421
|
if (!confirmed) return 2;
|
|
2397
2422
|
let control = null;
|
|
2398
2423
|
const onSigint = () => control && control.stop();
|
|
2399
2424
|
process.once('SIGINT', onSigint);
|
|
2400
|
-
const r = await
|
|
2425
|
+
const r = await probeRecheck(target.id, (line) => console.log(' ' + line), { onStop: (fn) => (control = { stop: fn }) }, confirmed);
|
|
2401
2426
|
process.removeListener('SIGINT', onSigint);
|
|
2402
2427
|
console.log('');
|
|
2403
2428
|
console.log(' RE-CHECK ' + target.id + ' ' + target.where);
|
|
@@ -2445,8 +2470,8 @@ function recheckCommand(id, options) {
|
|
|
2445
2470
|
|
|
2446
2471
|
// An input finding is re-checked by trying its value again, which adds test
|
|
2447
2472
|
// data to the app - so it asks first, the way `edge` does.
|
|
2448
|
-
const
|
|
2449
|
-
if (
|
|
2473
|
+
const probeTarget = dash.buildState(USER_DIR).findings.find((f) => f.id === String(id).trim() && PROBE_CHECKS[f.check]);
|
|
2474
|
+
if (probeTarget) return probeRecheckCommand(probeTarget, options);
|
|
2450
2475
|
|
|
2451
2476
|
const r = looker.recheck(USER_DIR, { frontend: () => readCodeAndSave(opened), regression: replayForLoop }, String(id).trim());
|
|
2452
2477
|
console.log('');
|
|
@@ -2608,18 +2633,18 @@ function runFromDashboard(id, opened, say) {
|
|
|
2608
2633
|
* this process, where its Stop can reach the browser run.
|
|
2609
2634
|
*/
|
|
2610
2635
|
async function probeFromDashboard(id, say, control, confirmed) {
|
|
2611
|
-
if (id
|
|
2612
|
-
const
|
|
2636
|
+
if (!PROBE_CHECKS[id]) return { ok: false, why: '"' + id + '" cannot be run from the dashboard' };
|
|
2637
|
+
const runner = probeModule(id);
|
|
2613
2638
|
const { flows } = probeFlows();
|
|
2614
|
-
if (!flows.length) return { ok: false, why: 'there is no recording that can be replayed - record
|
|
2639
|
+
if (!flows.length) return { ok: false, why: 'there is no recording that can be replayed - record one of your flows' };
|
|
2615
2640
|
if (!ensureBrowser()) return { ok: false, why: 'the browser could not be downloaded' };
|
|
2616
|
-
const report = await
|
|
2641
|
+
const report = await runner.run(USER_DIR, flows, {
|
|
2617
2642
|
say: say,
|
|
2618
2643
|
onControl: (c) => control.onStop(() => c.stop()),
|
|
2619
2644
|
confirmed: confirmed || [],
|
|
2620
2645
|
});
|
|
2621
|
-
|
|
2622
|
-
for (const line of
|
|
2646
|
+
runner.save(USER_DIR, report);
|
|
2647
|
+
for (const line of runner.summaryLines(report)) say(line);
|
|
2623
2648
|
const problems = report.findings.filter((f) => f.status === 'confirmed').length;
|
|
2624
2649
|
const tested = report.flows.some((f) => f.state === 'tested');
|
|
2625
2650
|
if (!tested) return { ok: false, why: 'nothing was tested - see what it printed' };
|
|
@@ -2631,23 +2656,23 @@ async function probeFromDashboard(id, say, control, confirmed) {
|
|
|
2631
2656
|
};
|
|
2632
2657
|
}
|
|
2633
2658
|
|
|
2634
|
-
/** One input finding, tried again:
|
|
2635
|
-
async function
|
|
2659
|
+
/** One input or reliability finding, tried again: only what found it, never the whole check. */
|
|
2660
|
+
async function probeRecheck(id, say, control, confirmed) {
|
|
2636
2661
|
const dash = require(path.join(PACKAGE_DIR, 'kryptheon-dashboard.js'));
|
|
2637
|
-
const
|
|
2638
|
-
const target = dash.buildState(USER_DIR).findings.find((f) => f.id === id && f.check === 'edge');
|
|
2662
|
+
const target = dash.buildState(USER_DIR).findings.find((f) => f.id === id && PROBE_CHECKS[f.check]);
|
|
2639
2663
|
if (!target) return null;
|
|
2664
|
+
const runner = probeModule(target.check);
|
|
2640
2665
|
const { flows } = probeFlows();
|
|
2641
2666
|
if (!ensureBrowser()) return { target: target, verdict: 'could not confirm', why: 'the browser could not be downloaded', newProblems: [] };
|
|
2642
|
-
const r = await
|
|
2667
|
+
const r = await runner.recheck(USER_DIR, flows, target, {
|
|
2643
2668
|
say: say,
|
|
2644
2669
|
onControl: (c) => control && control.onStop(() => c.stop()),
|
|
2645
2670
|
confirmed: confirmed || [],
|
|
2646
2671
|
});
|
|
2647
2672
|
const attempt = {
|
|
2648
|
-
id: id, at: new Date().toISOString(), check:
|
|
2673
|
+
id: id, at: new Date().toISOString(), check: target.check, where: target.where, headline: target.headline,
|
|
2649
2674
|
verdict: r.verdict, why: r.why || '',
|
|
2650
|
-
newProblems: (r.newProblems || []).map((f) => ({ check:
|
|
2675
|
+
newProblems: (r.newProblems || []).map((f) => ({ check: target.check, severity: f.severity, status: f.status, where: f.where, headline: f.headline })),
|
|
2651
2676
|
};
|
|
2652
2677
|
try {
|
|
2653
2678
|
fs.appendFileSync(store.pathsFor(USER_DIR).fixes, JSON.stringify(attempt) + '\n', 'utf8');
|
|
@@ -2657,9 +2682,9 @@ async function edgeRecheck(id, say, control, confirmed) {
|
|
|
2657
2682
|
return Object.assign({ target: target, testData: r.testData || [] }, attempt);
|
|
2658
2683
|
}
|
|
2659
2684
|
|
|
2660
|
-
async function
|
|
2661
|
-
const r = await
|
|
2662
|
-
if (!r) return { ok: false, why: 'no open
|
|
2685
|
+
async function probeRecheckFromDashboard(id, say, control, confirmed) {
|
|
2686
|
+
const r = await probeRecheck(id, say, control, confirmed);
|
|
2687
|
+
if (!r) return { ok: false, why: 'no open finding with that id that can be re-checked this way' };
|
|
2663
2688
|
say('VERDICT ' + VERDICT_WORDS[r.verdict] + (r.why ? ' - ' + r.why : ''));
|
|
2664
2689
|
return Object.assign({ ok: true, summary: r.verdict }, r, { target: undefined });
|
|
2665
2690
|
}
|
|
@@ -3022,7 +3047,7 @@ async function dashboardCommand() {
|
|
|
3022
3047
|
run: (id, say) => runFromDashboard(id, opened, say),
|
|
3023
3048
|
recheck: (id, say) => recheckFromDashboard(id, opened, say),
|
|
3024
3049
|
probe: (id, say, control, confirmed) => probeFromDashboard(id, say, control, confirmed),
|
|
3025
|
-
probeRecheck: (id, say, control, confirmed) =>
|
|
3050
|
+
probeRecheck: (id, say, control, confirmed) => probeRecheckFromDashboard(id, say, control, confirmed),
|
|
3026
3051
|
record: (address, say, options) => recordFromDashboard(address, say, options),
|
|
3027
3052
|
db: (action, connection, say) => databaseFromDashboard(action, connection, say),
|
|
3028
3053
|
words: (connection) => nightWords(connection),
|
|
@@ -3349,7 +3374,8 @@ async function main() {
|
|
|
3349
3374
|
case 'recheck':
|
|
3350
3375
|
return finishWith(await recheckCommand(rest.find((a) => !/^--/.test(a)), { yes: rest.includes('--yes') }));
|
|
3351
3376
|
case 'edge':
|
|
3352
|
-
|
|
3377
|
+
case 'reliability':
|
|
3378
|
+
return finishWith(await probeCommand(command, rest));
|
|
3353
3379
|
case '-v':
|
|
3354
3380
|
case '--version':
|
|
3355
3381
|
case 'version':
|
|
@@ -302,8 +302,8 @@
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
|
|
305
|
-
var CHECK_NAME = { regression: 'Recorded flows', frontend: 'Frontend code', data: 'Database', edge: 'Input and edge cases' };
|
|
306
|
-
var CHECK_TONE = { regression: 'sun', frontend: 'amber', data: 'dusk', edge: 'ember' };
|
|
305
|
+
var CHECK_NAME = { regression: 'Recorded flows', frontend: 'Frontend code', data: 'Database', edge: 'Input and edge cases', reliability: 'Reliability' };
|
|
306
|
+
var CHECK_TONE = { regression: 'sun', frontend: 'amber', data: 'dusk', edge: 'ember', reliability: 'ember' };
|
|
307
307
|
|
|
308
308
|
function counts() {
|
|
309
309
|
var confirmed = state.findings.filter(function (f) { return f.status === 'confirmed'; }).length;
|
|
@@ -871,7 +871,7 @@
|
|
|
871
871
|
var askingProbe = el('div', 'controls f-quick');
|
|
872
872
|
askingProbe.appendChild(probeConsentBox(probeAsking.check, 'Yes, try it again', function (body) {
|
|
873
873
|
runJob('/api/recheck', Object.assign({ id: f.id }, body)).then(function (done) { heard(f, (done && done.result) || {}); }, function () {});
|
|
874
|
-
},
|
|
874
|
+
}, RECHECK_WORDS[f.check === 'reliability' ? f.kind : 'edge']));
|
|
875
875
|
card.appendChild(askingProbe);
|
|
876
876
|
} else if (pending && pending.step === 'finding' && pending.finding && pending.finding.id === f.id) {
|
|
877
877
|
var asking = el('div', 'controls f-quick');
|
|
@@ -949,7 +949,13 @@
|
|
|
949
949
|
}
|
|
950
950
|
|
|
951
951
|
var STATE_TONE = { 'problems found': 'ember', 'things to check': 'amber', 'last run held': 'leaf', 'nothing found': 'leaf', 'stopped part way': 'amber', 'nothing was tested': 'amber' };
|
|
952
|
-
var PROBE_LABEL = { edge: 'Try inputs - adds test data' };
|
|
952
|
+
var PROBE_LABEL = { edge: 'Try inputs - adds test data', reliability: 'Replay 3x - adds test data' };
|
|
953
|
+
// What one re-check press does, said before the yes.
|
|
954
|
+
var RECHECK_WORDS = {
|
|
955
|
+
edge: 'Re-check: replays this flow as recorded, then once more with the value that found this. Two runs.',
|
|
956
|
+
flaky: 'Re-check: replays this flow five times. It is fixed only if all five work.',
|
|
957
|
+
double: 'Re-check: replays this flow once, then once more with this step double-clicked. Two runs.',
|
|
958
|
+
};
|
|
953
959
|
|
|
954
960
|
// Which probe check is asking for a yes right now, and for which finding
|
|
955
961
|
// when it is a re-check. Nothing that adds data to the app runs without it.
|
|
@@ -1120,7 +1126,7 @@
|
|
|
1120
1126
|
if (f.check === 'data') { prepare('recheck', 'finding', f); return; }
|
|
1121
1127
|
// An input finding is re-checked by typing its value again, which adds
|
|
1122
1128
|
// test data to the app: the same yes as the check's own button, first.
|
|
1123
|
-
if (f.check === 'edge') { probeAsking = { check:
|
|
1129
|
+
if (f.check === 'edge' || f.check === 'reliability') { probeAsking = { check: f.check, finding: f }; openRow = null; renderFindings(); return; }
|
|
1124
1130
|
runJob('/api/recheck', { id: f.id }).then(function (done) { heard(f, (done && done.result) || {}); }, function () {});
|
|
1125
1131
|
}
|
|
1126
1132
|
function heard(f, r) {
|
|
@@ -1281,7 +1287,15 @@
|
|
|
1281
1287
|
function load() {
|
|
1282
1288
|
return fetch('/api/state', { cache: 'no-store' })
|
|
1283
1289
|
.then(function (res) { if (!res.ok) throw new Error('the dashboard answered ' + res.status); return res.json(); })
|
|
1284
|
-
.then(
|
|
1290
|
+
.then(function (next) {
|
|
1291
|
+
// Drawn again only when something changed: a redraw every few
|
|
1292
|
+
// seconds replaced every button under the pointer, and a press that
|
|
1293
|
+
// landed on two drawings of one button was lost.
|
|
1294
|
+
var text = JSON.stringify(next);
|
|
1295
|
+
if (text === load.last) return;
|
|
1296
|
+
load.last = text;
|
|
1297
|
+
render(next);
|
|
1298
|
+
})
|
|
1285
1299
|
.catch(function (err) { byId('subline').textContent = 'Could not load: ' + err.message; });
|
|
1286
1300
|
}
|
|
1287
1301
|
|
package/kryptheon-dashboard.html
CHANGED
|
@@ -212,7 +212,7 @@
|
|
|
212
212
|
.detail-box { margin: 0 8px 22px 20px; padding: 18px 0 0; border-top: 1px dashed var(--line); }
|
|
213
213
|
.d-step { margin-top: 18px; font-size: 10px; color: var(--orange-deep); }
|
|
214
214
|
.d-step:first-child { margin-top: 0; }
|
|
215
|
-
.d-text { margin-top: 6px; font-size: 14px; line-height: 1.6; color: var(--muted); overflow-wrap: anywhere; }
|
|
215
|
+
.d-text { margin-top: 6px; font-size: 14px; line-height: 1.6; color: var(--muted); overflow-wrap: anywhere; white-space: pre-line; }
|
|
216
216
|
.detail-box pre { max-height: 360px; margin: 10px 0 0; padding: 18px; overflow: auto; font: 12px/1.7 var(--mono); white-space: pre-wrap; overflow-wrap: anywhere; color: var(--on-dark); background: var(--ink); }
|
|
217
217
|
.d-actions { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 14px; }
|
|
218
218
|
.attempts { display: grid; gap: 8px; margin-top: 14px; font-size: 13px; color: var(--muted); }
|
package/kryptheon-dashboard.js
CHANGED
|
@@ -48,7 +48,8 @@ const CHECKS = [
|
|
|
48
48
|
{ id: 'performance', label: 'Performance', available: false },
|
|
49
49
|
{ id: 'edge', label: 'Input and edge cases', available: true, asks: true,
|
|
50
50
|
what: 'Replays your recorded forms with one field at a time given an unusual value - empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist - and reports only what it can prove: a page error, a 5xx, a bad value saved, a tag shown as HTML. It adds test data to your app, so it runs only when you press its button.' },
|
|
51
|
-
{ id: 'reliability', label: 'Reliability', available:
|
|
51
|
+
{ id: 'reliability', label: 'Reliability', available: true, asks: true,
|
|
52
|
+
what: 'Replays each recorded flow three times and reports one that sometimes works and sometimes fails, with the failed run as proof; and clicks each step that saves twice, quickly, reporting when that saves twice, with both answers as proof. It adds test data to your app, so it runs only when you press its button.' },
|
|
52
53
|
{ id: 'api', label: 'API', available: false },
|
|
53
54
|
{ id: 'backend', label: 'Backend', available: false },
|
|
54
55
|
{ id: 'synthetic', label: 'Synthetic test users', available: false },
|
|
@@ -194,7 +195,7 @@ function countRecordings(root) {
|
|
|
194
195
|
* and the words said about them (kryptheon-probe.js). Read from the
|
|
195
196
|
* recordings every time, so what the page shows is what would really run.
|
|
196
197
|
*/
|
|
197
|
-
function probeAsk(root, what, env) {
|
|
198
|
+
function probeAsk(root, what, env, options) {
|
|
198
199
|
const probe = require('./kryptheon-probe.js');
|
|
199
200
|
const sources = [];
|
|
200
201
|
try {
|
|
@@ -203,14 +204,19 @@ function probeAsk(root, what, env) {
|
|
|
203
204
|
} catch (err) {
|
|
204
205
|
/* no recordings: nothing to ask about */
|
|
205
206
|
}
|
|
206
|
-
// The pages, and where the app was last seen saving (kryptheon-
|
|
207
|
-
const addresses =
|
|
208
|
-
return { addresses: addresses, live: probe.liveLooking(addresses), lines: probe.consentLines(what, addresses) };
|
|
207
|
+
// The pages, and where the app was last seen saving (kryptheon-probe.js).
|
|
208
|
+
const addresses = probe.knownAddresses(root, sources, env);
|
|
209
|
+
return { addresses: addresses, live: probe.liveLooking(addresses), lines: probe.consentLines(what, addresses, options) };
|
|
209
210
|
}
|
|
210
211
|
|
|
211
212
|
const PROBE_WHAT = {
|
|
212
213
|
edge: 'Input and edge cases: replays your recorded forms with one field at a time given an unusual value (empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most 40 runs.',
|
|
214
|
+
reliability: 'Reliability: replays each recorded flow three times, and clicks each step that saves (Save, Add, Create...) twice, quickly. At most 40 runs.',
|
|
213
215
|
};
|
|
216
|
+
// The reliability check types nothing of its own, so nothing it saves can
|
|
217
|
+
// carry the kryptheon-test marker - and its words do not promise one.
|
|
218
|
+
const PROBE_WORDS = { reliability: { marker: false } };
|
|
219
|
+
const PROBE_CHECKS = ['edge', 'reliability'];
|
|
214
220
|
|
|
215
221
|
/**
|
|
216
222
|
* The order someone new should go in, one step at a time. A step is "done"
|
|
@@ -448,14 +454,17 @@ function buildState(root, env) {
|
|
|
448
454
|
}
|
|
449
455
|
}
|
|
450
456
|
|
|
451
|
-
// Input and edge cases: the last press of
|
|
452
|
-
// its proof, and what
|
|
453
|
-
// can try exactly that
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
+
// Input and edge cases, and reliability: the last press of each one's
|
|
458
|
+
// button. Each finding carries its proof, and what it was about - the
|
|
459
|
+
// field and value, the flow, the step - so a re-check can try exactly that
|
|
460
|
+
// again.
|
|
461
|
+
const probeLast = {};
|
|
462
|
+
for (const id of PROBE_CHECKS) {
|
|
463
|
+
const last = (probeLast[id] = readJson(path.join(paths.dir, require('./kryptheon-probe.js').REPORT_FILES[id])));
|
|
464
|
+
if (!last || !config.enabled[id]) continue;
|
|
465
|
+
for (const f of last.findings || []) {
|
|
457
466
|
findings.push({
|
|
458
|
-
check:
|
|
467
|
+
check: id,
|
|
459
468
|
severity: f.severity,
|
|
460
469
|
status: f.status,
|
|
461
470
|
evidence: f.status === 'confirmed' ? 'runtime confirmed' : 'seen once',
|
|
@@ -464,7 +473,7 @@ function buildState(root, env) {
|
|
|
464
473
|
headline: f.headline,
|
|
465
474
|
detail: (f.proof || []).join('\n'),
|
|
466
475
|
fixPrompt: f.fixPrompt || '',
|
|
467
|
-
when:
|
|
476
|
+
when: last.endedAt || last.startedAt,
|
|
468
477
|
kind: f.kind,
|
|
469
478
|
type: f.type,
|
|
470
479
|
});
|
|
@@ -534,12 +543,12 @@ function buildState(root, env) {
|
|
|
534
543
|
f.id = changes.findingId(f);
|
|
535
544
|
f.attempts = attempts.filter((a) => a.id === f.id).map((a) => ({ at: a.at, verdict: a.verdict, why: a.why || '', newProblems: (a.newProblems || []).length }));
|
|
536
545
|
forAll.push({ id: f.id, severity: f.severity, headline: f.headline, where: f.where, prompt: f.fixPrompt || '' });
|
|
537
|
-
if (f.fixPrompt && f.check
|
|
538
|
-
// Re-checking
|
|
539
|
-
//
|
|
540
|
-
//
|
|
546
|
+
if (f.fixPrompt && PROBE_CHECKS.includes(f.check)) {
|
|
547
|
+
// Re-checking runs the flow in the app again, so it adds a little test
|
|
548
|
+
// data: --yes says yes for an app on this computer, and is refused for
|
|
549
|
+
// any other address - there the person must answer.
|
|
541
550
|
f.fixPrompt += '\n\nWhen you have fixed it, prove it: run npx kryptheon recheck ' + f.id + ' --yes' +
|
|
542
|
-
' (it
|
|
551
|
+
' (it runs the flow in the app again, adding a little test data; --yes works only for an app running on this computer). ' +
|
|
543
552
|
'It is fixed only when that says FIXED.';
|
|
544
553
|
} else if (f.fixPrompt) {
|
|
545
554
|
f.fixPrompt += '\n\nWhen you have fixed it, prove it: run npx kryptheon recheck ' + f.id +
|
|
@@ -557,12 +566,13 @@ function buildState(root, env) {
|
|
|
557
566
|
if (!c.available) state = 'coming soon';
|
|
558
567
|
else if (!config.enabled[c.id]) state = 'switched off';
|
|
559
568
|
else if (c.id === 'regression') state = !recordings ? 'no recordings yet' : !lastRun ? 'never run' : lastRun.failed ? 'problems found' : 'last run held';
|
|
560
|
-
else if (c.id
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
: findings.some((f) => f.check === '
|
|
564
|
-
:
|
|
565
|
-
:
|
|
569
|
+
else if (PROBE_CHECKS.includes(c.id)) {
|
|
570
|
+
const last = probeLast[c.id];
|
|
571
|
+
state = !last ? 'never run'
|
|
572
|
+
: findings.some((f) => f.check === c.id && f.status === 'confirmed') ? 'problems found'
|
|
573
|
+
: findings.some((f) => f.check === c.id) ? 'things to check'
|
|
574
|
+
: last.stopped ? 'stopped part way'
|
|
575
|
+
: !(last.flows || []).some((x) => x.state === 'tested') ? 'nothing was tested'
|
|
566
576
|
: 'last run held';
|
|
567
577
|
}
|
|
568
578
|
else if (c.id === 'frontend') state = !code ? 'never run' : (code.findings || []).length ? 'things to check' : 'nothing found';
|
|
@@ -576,11 +586,12 @@ function buildState(root, env) {
|
|
|
576
586
|
nightly: c.id === 'data' ? nightlyLine(nightly) : null,
|
|
577
587
|
// A check that adds data to the app: never in "Run all", never on its
|
|
578
588
|
// own - only its own button, after the page has shown these words.
|
|
579
|
-
asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env) : null,
|
|
580
|
-
lastRun: c.id
|
|
581
|
-
at:
|
|
582
|
-
held: (
|
|
583
|
-
|
|
589
|
+
asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env, PROBE_WORDS[c.id]) : null,
|
|
590
|
+
lastRun: probeLast[c.id] ? {
|
|
591
|
+
at: probeLast[c.id].endedAt, stopped: !!probeLast[c.id].stopped, runsDone: probeLast[c.id].runsDone, runsPlanned: probeLast[c.id].runsPlanned,
|
|
592
|
+
held: (probeLast[c.id].held || []).length,
|
|
593
|
+
notTested: (probeLast[c.id].notTested || []).length + (probeLast[c.id].flows || []).filter((x) => x.state !== 'tested').length,
|
|
594
|
+
testData: (probeLast[c.id].testData || []).length,
|
|
584
595
|
} : null,
|
|
585
596
|
// Runnable from the page: what needs nothing the page cannot supply. The
|
|
586
597
|
// database check needs a connection string, which stays in the terminal.
|
|
@@ -653,8 +664,8 @@ function buildState(root, env) {
|
|
|
653
664
|
|
|
654
665
|
// What each run is called on the page, and the command that does the same in
|
|
655
666
|
// a terminal - shown under every button, for when the page cannot be used.
|
|
656
|
-
const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms' };
|
|
657
|
-
const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge' };
|
|
667
|
+
const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking' };
|
|
668
|
+
const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge', reliability: 'npx kryptheon reliability' };
|
|
658
669
|
const DB_ACTIONS = {
|
|
659
670
|
scan: { title: 'Checking your database', command: 'npx kryptheon-night' },
|
|
660
671
|
install: { title: 'Setting up the nightly check', command: 'npx kryptheon-night install' },
|
|
@@ -991,8 +1002,8 @@ function createServer(root, options) {
|
|
|
991
1002
|
if (!/^[0-9a-f]{8}$/.test(id)) return json(res, 400, { ok: false, why: 'not a finding id' });
|
|
992
1003
|
const open = buildState(root, env).findings.find((f) => f.id === id);
|
|
993
1004
|
if (!open) return json(res, 404, { ok: false, why: 'no open finding with that id' });
|
|
994
|
-
if (open.check
|
|
995
|
-
const refused = probeRefusal(
|
|
1005
|
+
if (PROBE_CHECKS.includes(open.check)) {
|
|
1006
|
+
const refused = probeRefusal(open.check);
|
|
996
1007
|
if (refused) return json(res, 400, { ok: false, why: refused });
|
|
997
1008
|
if (typeof opts.probeRecheck !== 'function') return json(res, 400, { ok: false, why: 'this finding cannot be re-checked from here' });
|
|
998
1009
|
return begin('recheck', 'Re-checking: ' + open.headline, 'npx kryptheon recheck ' + id, (say, control) => opts.probeRecheck(id, say, control, confirmed));
|
package/kryptheon-edge.js
CHANGED
|
@@ -271,11 +271,32 @@ function diffLine(s) {
|
|
|
271
271
|
return (s.key ? s.key + ': ' + show(s.was) + ' in your recorded run, ' + show(s.now) + ' in this one' : 'the request body holds the typed value');
|
|
272
272
|
}
|
|
273
273
|
|
|
274
|
+
/**
|
|
275
|
+
* What the server said it stored for this field, from its answer - the row it
|
|
276
|
+
* saved, when it sends one back. The request shows what was sent; this shows
|
|
277
|
+
* what was kept, which is the stronger proof.
|
|
278
|
+
*/
|
|
279
|
+
function stored(s) {
|
|
280
|
+
if (!s.key || s.write.answer == null) return null;
|
|
281
|
+
const last = s.key.split('.').pop();
|
|
282
|
+
const answer = flatten(s.write.answer);
|
|
283
|
+
const key = Object.prototype.hasOwnProperty.call(answer, s.key) ? s.key
|
|
284
|
+
: Object.keys(answer).find((k) => k === last || k.endsWith('.' + last));
|
|
285
|
+
return key ? { key: key, value: answer[key] } : null;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function storedLine(s) {
|
|
289
|
+
const kept = stored(s);
|
|
290
|
+
return kept ? 'The server\'s answer: ' + kept.key + ' is ' + show(kept.value) + ' - that is what it stored.' : null;
|
|
291
|
+
}
|
|
292
|
+
|
|
274
293
|
function proofLines(saved) {
|
|
275
294
|
const lines = [];
|
|
276
295
|
for (const s of saved.slice(0, 3)) {
|
|
277
296
|
lines.push('Request: ' + probe.writeWords(s.write));
|
|
278
297
|
lines.push('Difference from your recorded run: ' + diffLine(s));
|
|
298
|
+
const stored = storedLine(s);
|
|
299
|
+
if (stored) lines.push(stored);
|
|
279
300
|
}
|
|
280
301
|
return lines;
|
|
281
302
|
}
|
|
@@ -327,7 +348,10 @@ function judgeVariant(clean, variant, plan) {
|
|
|
327
348
|
|
|
328
349
|
const saved = savedValues(clean, variant, field, plan.value);
|
|
329
350
|
if (saved.length) {
|
|
330
|
-
|
|
351
|
+
// What the server says it kept decides, when it says: a -5 it turned
|
|
352
|
+
// into 0 was not saved as -5. Without an answer, the request decides.
|
|
353
|
+
const kept = stored(saved[0]);
|
|
354
|
+
const as = savedAs(kind, plan.value, kept ? kept.value : saved[0].now);
|
|
331
355
|
const info = field.info || {};
|
|
332
356
|
const problem =
|
|
333
357
|
(kind === 'empty' && info.required && as !== 'changed') ||
|
|
@@ -681,18 +705,9 @@ function carryOver(previous, report, root) {
|
|
|
681
705
|
return report;
|
|
682
706
|
}
|
|
683
707
|
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
* is asked about, so a database somewhere else is named before it is written.
|
|
688
|
-
*/
|
|
689
|
-
function knownAddresses(root, sources, env) {
|
|
690
|
-
const out = [];
|
|
691
|
-
for (const source of sources) for (const a of probe.addressesOf(source)) if (!out.includes(a)) out.push(a);
|
|
692
|
-
const last = load(root, env);
|
|
693
|
-
for (const h of (last && last.writeHosts) || []) if (!out.includes(h)) out.push(h);
|
|
694
|
-
return out;
|
|
695
|
-
}
|
|
708
|
+
// Kept here for callers of this module; the list itself is shared by every
|
|
709
|
+
// probe check (kryptheon-probe.js).
|
|
710
|
+
const knownAddresses = probe.knownAddresses;
|
|
696
711
|
|
|
697
712
|
/** Keeps the report where the dashboard reads it - with what it did not try still said. */
|
|
698
713
|
function save(root, report, env, options) {
|
|
@@ -117,7 +117,13 @@ const test = base.test.extend({
|
|
|
117
117
|
const events = [];
|
|
118
118
|
const fields = {};
|
|
119
119
|
let seq = 0;
|
|
120
|
-
|
|
120
|
+
// Returns the event as kept, so what arrives later (a write's answer) is
|
|
121
|
+
// added to the very object that is saved - not to a copy of it.
|
|
122
|
+
const add = (e) => {
|
|
123
|
+
const kept = Object.assign({ seq: ++seq }, e);
|
|
124
|
+
events.push(kept);
|
|
125
|
+
return kept;
|
|
126
|
+
};
|
|
121
127
|
|
|
122
128
|
// A gap before every run, and a longer one after the app said "too many
|
|
123
129
|
// requests": the app is being asked many times in a row, and a rate limit
|
|
@@ -158,13 +164,42 @@ const test = base.test.extend({
|
|
|
158
164
|
// A write that carried a secret is the sign-in, not data the app
|
|
159
165
|
// keeps - it is never listed as test data left behind.
|
|
160
166
|
const signIn = JSON.stringify(kept).indexOf('[hidden]') !== -1;
|
|
161
|
-
|
|
167
|
+
const event = Object.assign({ kind: 'write', method: method, host: at.host, origin: at.origin, path: at.path, status: status, signIn: signIn }, kept);
|
|
168
|
+
const placed = add(event);
|
|
169
|
+
// What the server answered - often the record it saved, with its id.
|
|
170
|
+
// That is the proof that two saves made two records, and of what was
|
|
171
|
+
// really stored. Never for the sign-in, whose answer is a session.
|
|
172
|
+
// Read after the event is placed, so its order among the marks is
|
|
173
|
+
// the order the answer arrived in.
|
|
174
|
+
if (!signIn && /json/i.test((res.headers() || {})['content-type'] || '')) {
|
|
175
|
+
const reading = res.text().then((text) => {
|
|
176
|
+
if (text && text.length <= MAX_BODY * 5) {
|
|
177
|
+
try {
|
|
178
|
+
placed.answer = JSON.parse(hideValues(JSON.stringify(hideKeys(JSON.parse(text))), secrets));
|
|
179
|
+
} catch (err) {
|
|
180
|
+
/* not JSON after all: no answer kept */
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}, () => {}).then(() => answers.delete(reading));
|
|
184
|
+
answers.add(reading);
|
|
185
|
+
}
|
|
162
186
|
}
|
|
163
187
|
});
|
|
188
|
+
// Answers still being read. Counted before a mark and before the end.
|
|
189
|
+
const answers = new Set();
|
|
190
|
+
const answersRead = async () => {
|
|
191
|
+
for (let waited = 0; answers.size && waited < 5000; waited += 100) await sleep(100);
|
|
192
|
+
};
|
|
164
193
|
|
|
165
194
|
// Called from inside a probe copy, around the steps that matter.
|
|
166
195
|
page.__kxMark = async (name) => {
|
|
167
|
-
// Let what the last step started arrive before it is counted
|
|
196
|
+
// Let what the last step started arrive before it is counted: a moment
|
|
197
|
+
// for a second click's request to leave, then every write still on its
|
|
198
|
+
// way. A load state alone is not enough - once a page has gone quiet it
|
|
199
|
+
// says so at once, even while a new save is in flight.
|
|
200
|
+
await sleep(300);
|
|
201
|
+
for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
|
|
202
|
+
await answersRead();
|
|
168
203
|
try {
|
|
169
204
|
await page.waitForLoadState('networkidle', { timeout: 3000 });
|
|
170
205
|
} catch (err) {
|
|
@@ -211,6 +246,7 @@ const test = base.test.extend({
|
|
|
211
246
|
await use(page);
|
|
212
247
|
|
|
213
248
|
for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
|
|
249
|
+
await answersRead();
|
|
214
250
|
try {
|
|
215
251
|
await page.waitForLoadState('networkidle', { timeout: 3000 });
|
|
216
252
|
} catch (err) {
|