kryptheon 0.1.49 → 0.1.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +435 -333
- package/bin/kryptheon.js +90 -4
- package/kryptheon-code.js +158 -2
- package/kryptheon-dashboard-page.js +78 -6
- package/kryptheon-dashboard.html +9 -0
- package/kryptheon-dashboard.js +240 -12
- package/kryptheon-edge.js +83 -14
- package/kryptheon-heal.js +57 -0
- package/kryptheon-payments-run.js +501 -0
- package/kryptheon-payments.js +626 -0
- package/kryptheon-probe-fixture.js +175 -8
- package/kryptheon-probe.js +368 -317
- package/kryptheon-reliability.js +430 -40
- package/kryptheon-replay.js +0 -0
- package/kryptheon-secrets.js +14 -3
- package/package.json +12 -5
package/kryptheon-dashboard.js
CHANGED
|
@@ -25,6 +25,8 @@ const { spawnSync } = require('child_process');
|
|
|
25
25
|
const store = require('./kryptheon-store.js');
|
|
26
26
|
const changes = require('./kryptheon-changes.js');
|
|
27
27
|
const autostart = require('./kryptheon-autostart.js');
|
|
28
|
+
const selectors = require('./kryptheon-selectors.js');
|
|
29
|
+
const replay = require('./kryptheon-replay.js');
|
|
28
30
|
const VERSION = require('./package.json').version;
|
|
29
31
|
|
|
30
32
|
/**
|
|
@@ -49,7 +51,9 @@ const CHECKS = [
|
|
|
49
51
|
{ id: 'edge', label: 'Input and edge cases', available: true, asks: true,
|
|
50
52
|
what: 'Replays your recorded forms with one field at a time given an unusual value - empty, 5000 characters, an HTML tag, emoji, a negative number, a number just past the lowest or highest the page allows, a date that does not exist - and reports only what it can prove: a page error, a 5xx, a bad value saved, a tag shown as HTML. It adds test data to your app, so it runs only when you press its button.' },
|
|
51
53
|
{ id: 'reliability', label: 'Reliability', available: true, asks: true,
|
|
52
|
-
what: 'Replays each recorded flow
|
|
54
|
+
what: 'Replays each recorded flow and reports one that sometimes works and sometimes fails, with the failed run as proof; and clicks each step that saves twice, quickly, reporting when that saves twice, with both answers as proof; on an app on this computer, sends each save twice at the same moment, reporting a server that makes two records or fails; and looks for what each save added on the page right after it, reporting a page that shows it only when opened again; after an edit, opens the form again and reports a field that kept its old value; and at phone size clicks every button with saving blocked, reporting a click - or a page as it loads - that throws an error. It adds test data to your app, so it runs only when you press its button.' },
|
|
55
|
+
{ id: 'payments', label: 'Payments: a webhook sent twice', available: true,
|
|
56
|
+
what: 'Real payments in Razorpay\'s or Stripe\'s TEST mode, through a proxy on this computer, each checked as it comes until you press Stop. Its signed webhook is sent to your app again, then once with a wrong signature, and your database is read before and after each - reporting the plan, credits or order given a second time, a repeat answered with an error, or a forged webhook accepted. Live mode is never sent again. It runs only when you press its button.' },
|
|
53
57
|
{ id: 'api', label: 'API', available: false },
|
|
54
58
|
{ id: 'backend', label: 'Backend', available: false },
|
|
55
59
|
{ id: 'synthetic', label: 'Synthetic test users', available: false },
|
|
@@ -57,6 +61,24 @@ const CHECKS = [
|
|
|
57
61
|
|
|
58
62
|
const CHECK_IDS = new Set(CHECKS.map((c) => c.id));
|
|
59
63
|
|
|
64
|
+
// What no check here looks at, said plainly so a clean report is never read
|
|
65
|
+
// as "all of it is fine". Each blind round's misses that no check covers are
|
|
66
|
+
// here (rounds 19 and 20: roles, payment rules, counts and searches).
|
|
67
|
+
const NOT_CHECKED = [
|
|
68
|
+
{ what: 'Who can do what',
|
|
69
|
+
why: 'Whether a lower role - a member, a customer - can open, change or delete what only a higher role should, or reach another account\'s record by changing an id in the address. Kryptheon replays each recording as the person who made it; it never tries one account\'s actions as another.',
|
|
70
|
+
yourself: 'Sign in as each role and open the higher role\'s pages and actions directly, by their address.' },
|
|
71
|
+
{ what: 'Payment rules beyond a repeat',
|
|
72
|
+
why: 'Only the same webhook sent again, and a copy with a wrong signature, are tried. Not tried: a webhook with no signature at all, an amount or currency that does not match the order, buying again while a plan is active, or when a refund takes access away.',
|
|
73
|
+
yourself: 'Read your webhook handler for each of these, and test them in your provider\'s test mode.' },
|
|
74
|
+
{ what: 'Whether numbers and lists are right',
|
|
75
|
+
why: 'A count or total that leaves something out, a search or filter that drops results. Kryptheon sees what a page shows, not what it should show.',
|
|
76
|
+
yourself: 'Compare each count and search result with what you know is in your data.' },
|
|
77
|
+
{ what: 'Permissions decided in your server code',
|
|
78
|
+
why: 'The database check tests rules the database itself enforces (row level security, grants). An app that decides who sees what in its own server code - with plain Postgres and no such rules - is not tested there.',
|
|
79
|
+
yourself: 'Same as "Who can do what": try each role\'s requests against each other role\'s records.' },
|
|
80
|
+
];
|
|
81
|
+
|
|
60
82
|
function readJson(file) {
|
|
61
83
|
try {
|
|
62
84
|
let raw = fs.readFileSync(file, 'utf8');
|
|
@@ -167,9 +189,12 @@ function riskyPartsLine(f) {
|
|
|
167
189
|
}
|
|
168
190
|
|
|
169
191
|
/** When this flow last passed, from the run history, or null. */
|
|
170
|
-
function lastPassOf(history, title) {
|
|
192
|
+
function lastPassOf(history, title, specFile) {
|
|
193
|
+
// By file too when both say one: round 18 had four recordings all titled
|
|
194
|
+
// "Round 18 shop", and one's pass was read as another's.
|
|
195
|
+
const same = (x) => x.title === title && (!specFile || !x.specFile || x.specFile === specFile);
|
|
171
196
|
for (let i = history.length - 1; i >= 0; i--) {
|
|
172
|
-
const t = (history[i].tests || []).find(
|
|
197
|
+
const t = (history[i].tests || []).find(same);
|
|
173
198
|
if (t && t.status === 'passed') return history[i].runAt;
|
|
174
199
|
}
|
|
175
200
|
return null;
|
|
@@ -245,6 +270,70 @@ const PROBE_WHAT = {
|
|
|
245
270
|
const PROBE_WORDS = { reliability: { marker: false } };
|
|
246
271
|
const PROBE_CHECKS = ['edge', 'reliability'];
|
|
247
272
|
|
|
273
|
+
// Loaded when it is first needed, so a page that never shows payments never
|
|
274
|
+
// reads it (and pg is only ever loaded by a run).
|
|
275
|
+
function paymentsRun() {
|
|
276
|
+
return require('./kryptheon-payments-run.js');
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Every payment run kept in the store's payments folder, oldest first. A
|
|
281
|
+
* store from before the traces has only the last report.
|
|
282
|
+
*/
|
|
283
|
+
function readPaymentRuns(paths, last) {
|
|
284
|
+
const dir = path.join(paths.dir, paymentsRun().TRACE_DIR);
|
|
285
|
+
let names = [];
|
|
286
|
+
try {
|
|
287
|
+
names = fs.readdirSync(dir).filter((n) => /\.json$/.test(n)).sort();
|
|
288
|
+
} catch (err) {
|
|
289
|
+
names = [];
|
|
290
|
+
}
|
|
291
|
+
const runs = names.map((n) => readJson(path.join(dir, n))).filter((r) => r && Array.isArray(r.results));
|
|
292
|
+
return runs.length ? runs : last ? [last] : [];
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** The words before the payment check's yes, and the app it would start from. */
|
|
296
|
+
function paymentsAsk(saved, last) {
|
|
297
|
+
const run = paymentsRun();
|
|
298
|
+
const local = saved && require('./kryptheon-payments.js').isLocalAddress(saved) ? saved : '';
|
|
299
|
+
const app = local || (last && last.app) || 'http://localhost:3000';
|
|
300
|
+
return { app: app, schema: (last && last.schema) || 'public', port: run.DEFAULT_PORT, lines: run.consentLines(app, (last && last.schema) || 'public', run.DEFAULT_PORT) };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* What a fresh payment check says about one payment finding. Fixed only when
|
|
305
|
+
* the run tested a webhook of the same kind and the finding is gone; a run
|
|
306
|
+
* that tested nothing, or tested another event, proves nothing about it.
|
|
307
|
+
*/
|
|
308
|
+
function paymentsVerdict(target, before, after, ran, report) {
|
|
309
|
+
const tested = ((report && report.results) || []).filter((r) => !(r.notTested || []).length);
|
|
310
|
+
// Judged by the re-check's own run: older runs stay in the store, so the
|
|
311
|
+
// list after it still holds the finding until this verdict is kept.
|
|
312
|
+
const foundAgain = tested.some((r) => target.where === r.path + ' (' + r.event.type + ')' &&
|
|
313
|
+
(r.findings || []).some((f) => paymentsRun().findingLines(r, f)[0].replace(/^(CRITICAL|HIGH|MEDIUM)\s+/, '') === target.headline));
|
|
314
|
+
let verdict;
|
|
315
|
+
let why = '';
|
|
316
|
+
if (!ran || !ran.ok) {
|
|
317
|
+
verdict = 'could not confirm';
|
|
318
|
+
why = (ran && ran.why) || 'the payment check could not run';
|
|
319
|
+
} else if (foundAgain) {
|
|
320
|
+
verdict = 'still open';
|
|
321
|
+
} else if (!tested.some((r) => target.where === r.path + ' (' + r.event.type + ')')) {
|
|
322
|
+
verdict = 'could not confirm';
|
|
323
|
+
why = tested.length ? 'that run tested a different webhook, not ' + target.where : 'no webhook was tested in that run';
|
|
324
|
+
} else {
|
|
325
|
+
verdict = 'fixed';
|
|
326
|
+
}
|
|
327
|
+
const had = new Set(before.map((f) => f.id));
|
|
328
|
+
return {
|
|
329
|
+
id: target.id, at: new Date().toISOString(), check: 'payments', where: target.where, headline: target.headline,
|
|
330
|
+
verdict: verdict, why: why,
|
|
331
|
+
newProblems: after.filter((f) => !had.has(f.id) && f.check === 'payments').map((f) => ({
|
|
332
|
+
id: f.id, check: f.check, severity: f.severity, status: f.status, where: f.where, headline: f.headline,
|
|
333
|
+
})),
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
|
|
248
337
|
/**
|
|
249
338
|
* The order someone new should go in, one step at a time. A step is "done"
|
|
250
339
|
* only when it has actually run and left a record - never because nothing was
|
|
@@ -426,6 +515,33 @@ function nightlyLine(nightly) {
|
|
|
426
515
|
* evidence, confidence, where, what, and the fix - so the page can show them
|
|
427
516
|
* the same way whichever check found them.
|
|
428
517
|
*/
|
|
518
|
+
/**
|
|
519
|
+
* The step a failed run stopped at, when its selector rests on data that
|
|
520
|
+
* moves (kryptheon-selectors.js): { line, selector, why }, or null.
|
|
521
|
+
*/
|
|
522
|
+
function movingStep(root, t) {
|
|
523
|
+
const line = t.failure && t.failure.line;
|
|
524
|
+
if (!line || !t.specFile) return null;
|
|
525
|
+
// More than one of it on the page is a different reason, already said.
|
|
526
|
+
if (/has \d+ of them/.test(String(t.failure.plainLanguage || ''))) return null;
|
|
527
|
+
let source;
|
|
528
|
+
try {
|
|
529
|
+
source = fs.readFileSync(path.resolve(root, String(t.specFile)), 'utf8');
|
|
530
|
+
} catch (err) {
|
|
531
|
+
return null;
|
|
532
|
+
}
|
|
533
|
+
const hit = selectors.findFragileSelectors(source).find((f) => f.line === line && f.reasons.some((r) => r.kind === 'number' || r.kind === 'data'));
|
|
534
|
+
// A row of a list picked by what it says: the words are a record's, and
|
|
535
|
+
// records change (blind round 20: "J Juniper Alex Parker Brush").
|
|
536
|
+
const text = String(source.split('\n')[line - 1] || '');
|
|
537
|
+
if (!hit && replay.ROW_PICK.test(text)) {
|
|
538
|
+
return { line: line, selector: selectors.selectorChain(text.trim()), why: 'it picks a row by what that record says, which changes when the record does' };
|
|
539
|
+
}
|
|
540
|
+
if (!hit) return null;
|
|
541
|
+
const reason = hit.reasons.find((r) => r.kind === 'number' || r.kind === 'data');
|
|
542
|
+
return { line: line, selector: hit.selector, why: reason.why + (reason.kind === 'number' ? ' that can change as data is added' : '') };
|
|
543
|
+
}
|
|
544
|
+
|
|
429
545
|
function buildState(root, env) {
|
|
430
546
|
const paths = store.pathsFor(root, env);
|
|
431
547
|
const config = readConfig(paths);
|
|
@@ -446,17 +562,32 @@ function buildState(root, env) {
|
|
|
446
562
|
if (lastRun && config.enabled.regression) {
|
|
447
563
|
for (const t of lastRun.tests || []) {
|
|
448
564
|
if (t.status !== 'failed' && t.status !== 'timedOut') continue;
|
|
565
|
+
// A recording that has never passed has not broken: nothing is known
|
|
566
|
+
// to have worked. Said as a thing to check - the recording or the app -
|
|
567
|
+
// as the replay's own summary says ("0 broken, 1 has no baseline").
|
|
568
|
+
// Found on a blind test (round 18): it was listed as "HIGH confirmed -
|
|
569
|
+
// broke" while the summary said nothing was broken.
|
|
570
|
+
const passedBefore = lastPassOf(history, t.title, t.specFile);
|
|
571
|
+
// Never passed, and stopped on a step whose selector holds a number or
|
|
572
|
+
// a date: the recording caught data that moves. Found on blind round
|
|
573
|
+
// 19: three recordings clicked "Projects3" and "fgff gffgddg 0" - counts
|
|
574
|
+
// that grew with every replay - and each was only "the recording or
|
|
575
|
+
// the app?".
|
|
576
|
+
const moved = !passedBefore && movingStep(root, t);
|
|
449
577
|
findings.push({
|
|
450
578
|
check: 'regression',
|
|
451
|
-
severity: 'HIGH',
|
|
452
|
-
status: 'confirmed',
|
|
453
|
-
evidence: 'runtime confirmed',
|
|
454
|
-
confidence: 'high',
|
|
579
|
+
severity: passedBefore ? 'HIGH' : moved ? 'LOW' : 'MEDIUM',
|
|
580
|
+
status: passedBefore ? 'confirmed' : 'verification required',
|
|
581
|
+
evidence: passedBefore ? 'runtime confirmed' : 'seen once',
|
|
582
|
+
confidence: passedBefore ? 'high' : 'medium',
|
|
455
583
|
where: String((t.failure && t.failure.file) || t.specFile || 'tests').split('\\').join('/') +
|
|
456
584
|
(t.failure && t.failure.line ? ':' + t.failure.line : ''),
|
|
457
|
-
headline: 'The recorded flow "' + t.title + '" broke.'
|
|
585
|
+
headline: passedBefore ? 'The recorded flow "' + t.title + '" broke.'
|
|
586
|
+
: moved ? 'The recorded flow "' + t.title + '" has never passed: it stops at line ' + moved.line + ', ' + moved.selector + ', and ' + moved.why +
|
|
587
|
+
' - most likely the recording, not the app. Record that step again on something that does not change.'
|
|
588
|
+
: 'The recorded flow "' + t.title + '" has never passed - the recording or the app?',
|
|
458
589
|
detail: (t.failure && t.failure.plainLanguage) || '',
|
|
459
|
-
fixPrompt: flowPrompt(t,
|
|
590
|
+
fixPrompt: flowPrompt(t, passedBefore),
|
|
460
591
|
when: lastRun.runAt,
|
|
461
592
|
});
|
|
462
593
|
}
|
|
@@ -507,6 +638,48 @@ function buildState(root, env) {
|
|
|
507
638
|
}
|
|
508
639
|
}
|
|
509
640
|
|
|
641
|
+
// The payment check: every run kept in the store, not only the last. A
|
|
642
|
+
// finding stays until a re-check says FIXED. Found on a blind test (round
|
|
643
|
+
// 18): credits given twice was found by one payment, the next payment (a
|
|
644
|
+
// different product on the same webhook) held, and the finding vanished.
|
|
645
|
+
const paymentsLast = readJson(path.join(paths.dir, paymentsRun().REPORT_FILE));
|
|
646
|
+
const paymentRuns = readPaymentRuns(paths, paymentsLast);
|
|
647
|
+
const paymentsFixedAt = {};
|
|
648
|
+
for (const a of readHistory(paths.fixes, 500)) {
|
|
649
|
+
if (a.check === 'payments' && a.verdict === 'fixed' && (!paymentsFixedAt[a.id] || a.at > paymentsFixedAt[a.id])) paymentsFixedAt[a.id] = a.at;
|
|
650
|
+
}
|
|
651
|
+
const paymentFindings = new Map();
|
|
652
|
+
if (config.enabled.payments) for (const run of paymentRuns) {
|
|
653
|
+
for (const r of run.results || []) {
|
|
654
|
+
for (const f of r.findings || []) {
|
|
655
|
+
const words = paymentsRun().findingLines(r, f);
|
|
656
|
+
const item = {
|
|
657
|
+
check: 'payments',
|
|
658
|
+
severity: f.severity,
|
|
659
|
+
// A refund that took nothing back is seen, not proven wrong.
|
|
660
|
+
status: f.toCheck ? 'verification required' : 'confirmed',
|
|
661
|
+
evidence: f.toCheck ? 'seen once' : 'runtime confirmed',
|
|
662
|
+
confidence: f.toCheck ? 'medium' : 'high',
|
|
663
|
+
where: r.path + ' (' + r.event.type + ')',
|
|
664
|
+
headline: words[0].replace(/^(CRITICAL|HIGH|MEDIUM)\s+/, ''),
|
|
665
|
+
// What changed, then the whole trace: each delivery, when, its
|
|
666
|
+
// answer, and the rows it wrote - so the proof is on the finding.
|
|
667
|
+
detail: words.slice(2).map((l) => l.trim()).join('\n') +
|
|
668
|
+
(r.trace ? '\n\n' + paymentsRun().traceLines(r).join('\n') : '') +
|
|
669
|
+
(run.traceFile ? '\n\nKept on this computer: ' + run.traceFile : ''),
|
|
670
|
+
fixPrompt: paymentsRun().fixPrompt(r, f),
|
|
671
|
+
when: run.at,
|
|
672
|
+
kind: f.kind,
|
|
673
|
+
};
|
|
674
|
+
// The newest sighting of each finding is the one shown.
|
|
675
|
+
paymentFindings.set(changes.findingId(item), item);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
for (const [id, item] of paymentFindings) {
|
|
680
|
+
if (!paymentsFixedAt[id] || paymentsFixedAt[id] < item.when) findings.push(item);
|
|
681
|
+
}
|
|
682
|
+
|
|
510
683
|
let nightWhen = null;
|
|
511
684
|
try {
|
|
512
685
|
nightWhen = fs.statSync(paths.nightLast).mtime.toISOString();
|
|
@@ -570,7 +743,11 @@ function buildState(root, env) {
|
|
|
570
743
|
f.id = changes.findingId(f);
|
|
571
744
|
f.attempts = attempts.filter((a) => a.id === f.id).map((a) => ({ at: a.at, verdict: a.verdict, why: a.why || '', newProblems: (a.newProblems || []).length }));
|
|
572
745
|
forAll.push({ id: f.id, severity: f.severity, headline: f.headline, where: f.where, prompt: f.fixPrompt || '' });
|
|
573
|
-
if (f.fixPrompt &&
|
|
746
|
+
if (f.fixPrompt && f.check === 'payments') {
|
|
747
|
+
// Proving it needs a new payment: the old webhook has been handled.
|
|
748
|
+
f.fixPrompt += '\n\nWhen you have fixed it, prove it: press Re-check on this finding in the Kryptheon dashboard ' +
|
|
749
|
+
'(or run npx kryptheon payments) and make one more test payment. It is fixed only when that says FIXED.';
|
|
750
|
+
} else if (f.fixPrompt && PROBE_CHECKS.includes(f.check)) {
|
|
574
751
|
// Re-checking runs the flow in the app again, so it adds a little test
|
|
575
752
|
// data: --yes says yes for an app on this computer, and is refused for
|
|
576
753
|
// any other address - there the person must answer.
|
|
@@ -603,6 +780,12 @@ function buildState(root, env) {
|
|
|
603
780
|
: 'last run held';
|
|
604
781
|
}
|
|
605
782
|
else if (c.id === 'frontend') state = !code ? 'never run' : (code.findings || []).length ? 'things to check' : 'nothing found';
|
|
783
|
+
else if (c.id === 'payments') {
|
|
784
|
+
state = !paymentsLast ? 'never run'
|
|
785
|
+
: findings.some((f) => f.check === 'payments') ? 'problems found'
|
|
786
|
+
: !(paymentsLast.results || []).some((r) => !(r.notTested || []).length) ? 'nothing was tested'
|
|
787
|
+
: 'last run held';
|
|
788
|
+
}
|
|
606
789
|
else if (c.id === 'data') {
|
|
607
790
|
const ranNightly = nightly && nightly.ranAt && !nightly.stopped;
|
|
608
791
|
state = !night && !ranNightly ? 'never run here'
|
|
@@ -614,6 +797,9 @@ function buildState(root, env) {
|
|
|
614
797
|
// A check that adds data to the app: never in "Run all", never on its
|
|
615
798
|
// own - only its own button, after the page has shown these words.
|
|
616
799
|
asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env, PROBE_WORDS[c.id]) : null,
|
|
800
|
+
// What the payment check says before its yes, and the app it starts
|
|
801
|
+
// from: the saved address when it is this computer's, else the usual one.
|
|
802
|
+
payments: c.id === 'payments' && config.enabled.payments ? paymentsAsk(readAppAddress(paths), paymentsLast) : null,
|
|
617
803
|
lastRun: probeLast[c.id] ? {
|
|
618
804
|
at: probeLast[c.id].endedAt, stopped: !!probeLast[c.id].stopped, runsDone: probeLast[c.id].runsDone, runsPlanned: probeLast[c.id].runsPlanned,
|
|
619
805
|
held: (probeLast[c.id].held || []).length,
|
|
@@ -660,6 +846,7 @@ function buildState(root, env) {
|
|
|
660
846
|
looks: looks,
|
|
661
847
|
proven: proven,
|
|
662
848
|
checks: checks,
|
|
849
|
+
notChecked: NOT_CHECKED,
|
|
663
850
|
findings: findings,
|
|
664
851
|
allFixPrompt: combinedPrompt(forAll),
|
|
665
852
|
runs: history.slice().reverse().map((r) => ({ runAt: r.runAt, status: r.status, passed: r.passed, failed: r.failed, durationMs: r.durationMs })),
|
|
@@ -691,8 +878,10 @@ function buildState(root, env) {
|
|
|
691
878
|
|
|
692
879
|
// What each run is called on the page, and the command that does the same in
|
|
693
880
|
// a terminal - shown under every button, for when the page cannot be used.
|
|
694
|
-
const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking'
|
|
695
|
-
|
|
881
|
+
const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking',
|
|
882
|
+
payments: 'Waiting for test payments, and sending each webhook again' };
|
|
883
|
+
const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge', reliability: 'npx kryptheon reliability',
|
|
884
|
+
payments: 'npx kryptheon payments' };
|
|
696
885
|
const DB_ACTIONS = {
|
|
697
886
|
scan: { title: 'Checking your database', command: 'npx kryptheon-night' },
|
|
698
887
|
install: { title: 'Setting up the nightly check', command: 'npx kryptheon-night install' },
|
|
@@ -1087,6 +1276,43 @@ function createServer(root, options) {
|
|
|
1087
1276
|
return begin('db', DB_ACTIONS[action].title, DB_ACTIONS[action].command,
|
|
1088
1277
|
(say) => opts.db(action, connection, say), secretsOf(connection));
|
|
1089
1278
|
}
|
|
1279
|
+
if (url.pathname === '/api/payments' && typeof opts.payments === 'function') {
|
|
1280
|
+
// One test payment, its webhook sent again. Its copies are processed
|
|
1281
|
+
// by the app, so only with the yes from the page, only for an app on
|
|
1282
|
+
// this computer, and the connection string is handled like the
|
|
1283
|
+
// database check's: given to this job only, scrubbed from its lines.
|
|
1284
|
+
if (!readConfig(store.pathsFor(root, env)).enabled.payments) return json(res, 400, { ok: false, why: 'the payment check is switched off' });
|
|
1285
|
+
if (input.consent !== true) return json(res, 400, { ok: false, why: 'its copies are processed by your app, so it runs only after you say yes' });
|
|
1286
|
+
const connection = connectionFrom(input.connection);
|
|
1287
|
+
if (!connection) return json(res, 400, { ok: false, why: 'paste your development database\'s connection string in "Connect your database" first' });
|
|
1288
|
+
const app = appAddress(typeof input.app === 'string' ? input.app : '');
|
|
1289
|
+
const full = app && !/^https?:\/\//i.test(app) ? 'http://' + app : app;
|
|
1290
|
+
if (!full || !require('./kryptheon-payments.js').isLocalAddress(full)) {
|
|
1291
|
+
return json(res, 400, { ok: false, why: 'the payment check only runs against an app on this computer - start it locally and give its address, like http://localhost:3000' });
|
|
1292
|
+
}
|
|
1293
|
+
const schema = typeof input.schema === 'string' && /^[A-Za-z_][A-Za-z0-9_]{0,62}$/.test(input.schema) ? input.schema : 'public';
|
|
1294
|
+
const run = { app: full, schema: schema, connection: connection, testMode: input.testMode === true };
|
|
1295
|
+
const id = input.id === undefined ? null : String(input.id);
|
|
1296
|
+
if (id !== null) {
|
|
1297
|
+
if (!/^[0-9a-f]{8}$/.test(id)) return json(res, 400, { ok: false, why: 'not a finding id' });
|
|
1298
|
+
const target = buildState(root, env).findings.find((f) => f.id === id && f.check === 'payments');
|
|
1299
|
+
if (!target) return json(res, 404, { ok: false, why: 'no open payment finding with that id' });
|
|
1300
|
+
return begin('recheck', 'Re-checking: ' + target.headline, RUN_COMMAND.payments, (say, control) => {
|
|
1301
|
+
const before = buildState(root, env).findings;
|
|
1302
|
+
return Promise.resolve(opts.payments(run, say, control)).then((ran) => {
|
|
1303
|
+
const report = readJson(path.join(store.pathsFor(root, env).dir, paymentsRun().REPORT_FILE));
|
|
1304
|
+
const attempt = paymentsVerdict(target, before, buildState(root, env).findings, ran, report);
|
|
1305
|
+
try {
|
|
1306
|
+
fs.appendFileSync(store.pathsFor(root, env).fixes, JSON.stringify(attempt) + '\n', 'utf8');
|
|
1307
|
+
} catch (err) {
|
|
1308
|
+
attempt.saveError = err.message;
|
|
1309
|
+
}
|
|
1310
|
+
return Object.assign({ ok: true, summary: attempt.verdict }, attempt);
|
|
1311
|
+
});
|
|
1312
|
+
}, secretsOf(connection));
|
|
1313
|
+
}
|
|
1314
|
+
return begin('payments', RUN_TITLE.payments, RUN_COMMAND.payments, (say, control) => opts.payments(run, say, control), secretsOf(connection));
|
|
1315
|
+
}
|
|
1090
1316
|
if (url.pathname === '/api/app') {
|
|
1091
1317
|
// The app's web address: saved for this project, so Record and the
|
|
1092
1318
|
// outside check start from it. Empty clears it.
|
|
@@ -1157,6 +1383,8 @@ module.exports = {
|
|
|
1157
1383
|
createJobs: createJobs,
|
|
1158
1384
|
DB_ACTIONS: DB_ACTIONS,
|
|
1159
1385
|
databaseVerdict: databaseVerdict,
|
|
1386
|
+
paymentsVerdict: paymentsVerdict,
|
|
1387
|
+
paymentsAsk: paymentsAsk,
|
|
1160
1388
|
startSteps: startSteps,
|
|
1161
1389
|
nextAction: nextAction,
|
|
1162
1390
|
combinedPrompt: combinedPrompt,
|
package/kryptheon-edge.js
CHANGED
|
@@ -152,14 +152,21 @@ function candidateFields(source) {
|
|
|
152
152
|
lines.forEach((line, i) => {
|
|
153
153
|
if (/\.fill\(/.test(line) && (replay.isSecretField(replay.nameInLine(line)) || /process\.env\./.test(line))) signInEnd = i + 1;
|
|
154
154
|
});
|
|
155
|
-
|
|
155
|
+
// Each field once, by its last fill: that is the value the form sent. Found
|
|
156
|
+
// on blind round 20: "Duration" typed 6, then 62, then 60 - the first was
|
|
157
|
+
// taken as what was recorded, no request carried it, and no value past
|
|
158
|
+
// the form was ever sent.
|
|
159
|
+
const fills = fillSteps(text).filter((step) => step.line > signInEnd && !replay.isSecretField(step.name));
|
|
160
|
+
const last = new Map();
|
|
161
|
+
for (const step of fills) last.set(step.locator.replace(/\s+/g, ' '), step);
|
|
162
|
+
// In the order the fields were first typed into; each its last fill.
|
|
156
163
|
const out = [];
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const key = step.locator.replace(/\s+/g, ' ');
|
|
164
|
+
const seen = new Set();
|
|
165
|
+
for (const first of fills) {
|
|
166
|
+
const key = first.locator.replace(/\s+/g, ' ');
|
|
161
167
|
if (seen.has(key)) continue;
|
|
162
168
|
seen.add(key);
|
|
169
|
+
const step = last.get(key);
|
|
163
170
|
// Saved by a click on Save/Add/Create... or an Enter, before the flow
|
|
164
171
|
// goes anywhere else.
|
|
165
172
|
let saves = false;
|
|
@@ -342,12 +349,26 @@ function flatten(value, prefix, out) {
|
|
|
342
349
|
return into;
|
|
343
350
|
}
|
|
344
351
|
|
|
345
|
-
|
|
352
|
+
// Taken by the app: a 2xx, or a form's redirect after it saved (probe.accepted).
|
|
353
|
+
const ok = (w) => probe.accepted(w);
|
|
346
354
|
const writesOf = (r) => ((r && r.events) || []).filter((e) => e.kind === 'write');
|
|
347
355
|
const routeOf = (w) => w.method + ' ' + w.host + w.path;
|
|
348
356
|
// A write the app accepted and keeps: not the sign-in, not a delete.
|
|
349
357
|
const leftBehind = (w) => ok(w) && w.method !== 'DELETE' && !w.signIn;
|
|
350
358
|
|
|
359
|
+
/**
|
|
360
|
+
* The kind of record an address saves: its ids, and a trailing edit or new,
|
|
361
|
+
* taken off - POST /pets and POST /pets/7/edit both save a pet, and POST
|
|
362
|
+
* /settings saves something else. Found on blind round 20: a pet's "name"
|
|
363
|
+
* and the studio's "name" were said as one problem.
|
|
364
|
+
*/
|
|
365
|
+
function recordStem(p) {
|
|
366
|
+
return String(p || '')
|
|
367
|
+
.replace(/\/(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24})(?=\/|$)/gi, '/:id')
|
|
368
|
+
.replace(/\/(edit|new|create|update)$/i, '')
|
|
369
|
+
.replace(/\/:id$/, '');
|
|
370
|
+
}
|
|
371
|
+
|
|
351
372
|
/** Writes of the plain replay and of a variant, paired by where they went, in order. */
|
|
352
373
|
function pairWrites(clean, variant) {
|
|
353
374
|
const left = {};
|
|
@@ -490,7 +511,21 @@ function judgeVariant(clean, variant, plan) {
|
|
|
490
511
|
}));
|
|
491
512
|
}
|
|
492
513
|
|
|
493
|
-
|
|
514
|
+
// Sent back to the form, where the recorded save went on to another page:
|
|
515
|
+
// the form was sent back to be filled in again - the value was refused.
|
|
516
|
+
// Back to it by a redirect, or the form drawn again in a 200 with no record
|
|
517
|
+
// in it - the way a server-rendered form shows its errors.
|
|
518
|
+
const movedOn = (w) => w && ok(w) && w.status >= 300 && w.status < 400 && !probe.backToForm(w);
|
|
519
|
+
// Or both went back to the form, but with another message than the
|
|
520
|
+
// recorded save's (blind round 21: /settings?notice=Settings saved for the
|
|
521
|
+
// save, /settings?notice=Check the settings and try again for the value).
|
|
522
|
+
const answeredOtherwise = (s) => s.cleanWrite && ok(s.cleanWrite) && probe.backToForm(s.cleanWrite) && probe.backToForm(s.write) &&
|
|
523
|
+
probe.answerShape(s.cleanWrite) !== probe.answerShape(s.write);
|
|
524
|
+
const sentBack = (s) => s.write.shownAfter !== true && (answeredOtherwise(s) || (movedOn(s.cleanWrite) &&
|
|
525
|
+
(probe.backToForm(s.write) || (s.write.status >= 200 && s.write.status < 300 && s.write.answer == null))));
|
|
526
|
+
const allSaved = savedValues(clean, variant, field, plan.value);
|
|
527
|
+
const saved = allSaved.filter((s) => !sentBack(s));
|
|
528
|
+
const sentBackWrites = allSaved.filter(sentBack).map((s) => s.write);
|
|
494
529
|
if (saved.length) {
|
|
495
530
|
// What the server says it kept decides, when it says: a -5 it turned
|
|
496
531
|
// into 0 was not saved as -5. Without an answer, the request decides.
|
|
@@ -521,19 +556,40 @@ function judgeVariant(clean, variant, plan) {
|
|
|
521
556
|
: kind === 'long' && info.maxLength != null ? 'The field allows ' + info.maxLength + ' characters, but the server stored ' + LONG_LENGTH + '.'
|
|
522
557
|
: kind === 'over' ? 'The page allows at most ' + info.maxLength + ' characters, but the server stored ' + String(plan.value).length + '.'
|
|
523
558
|
: null;
|
|
559
|
+
// A form that redirects sends no record back. A redirect on to another
|
|
560
|
+
// page is the save going through (Post, Redirect, Get); one straight back
|
|
561
|
+
// to the form may be the form sent back to be filled in again - unless
|
|
562
|
+
// the page it went to shows the value.
|
|
563
|
+
const w0 = saved[0].write;
|
|
564
|
+
const redirected = w0.status >= 300 && w0.status < 400;
|
|
565
|
+
// Back to the form with the very answer the recorded save got - "Settings
|
|
566
|
+
// saved" for both - is the save going through.
|
|
567
|
+
const c0 = saved[0].cleanWrite;
|
|
568
|
+
const asSaved = redirected && c0 && ok(c0) && probe.backToForm(c0) && probe.backToForm(w0) && probe.answerShape(c0) === probe.answerShape(w0);
|
|
569
|
+
const weak = redirected && probe.backToForm(w0) && w0.shownAfter !== true && !asSaved;
|
|
570
|
+
const pageLine = redirected && w0.shownAfter === true ? ['The page it went to after saving, ' + w0.to + ', shows ' + show(plan.value) + '.']
|
|
571
|
+
: asSaved ? ['It answered as your recorded save did (' + w0.to + ') - the value was taken.']
|
|
572
|
+
: weak ? ['It answered with a redirect back to the form (' + w0.to + '), which does not say whether it kept the value.'] : [];
|
|
524
573
|
if (problem || toCheck) {
|
|
574
|
+
const sure = problem && !weak;
|
|
525
575
|
findings.push(Object.assign({}, base, {
|
|
526
|
-
|
|
527
|
-
|
|
576
|
+
// The same value taken into the same field by the same app, whichever
|
|
577
|
+
// flow sent it - a create and an edit of the same record.
|
|
578
|
+
same: saved[0].key ? kind + '|' + (w0.origin || w0.host || '') + recordStem(w0.path) + '|' + saved[0].key : null,
|
|
579
|
+
sameRequest: probe.writeWords(w0),
|
|
580
|
+
type: 'accepted', severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
|
|
581
|
+
headline: sure ? '"' + name + '" accepts ' + words + ', and the app saved it.'
|
|
582
|
+
: problem ? '"' + name + '" was sent ' + words + ' and the app did not refuse it - check whether it kept it.'
|
|
528
583
|
: '"' + name + '" took ' + words + ' and the app ' + savedWords + ' - check that this is allowed.',
|
|
529
584
|
proof: ['Flow: ' + plan.flow.title + ' (' + where + ')', 'Typed into "' + name + '": ' + show(plan.value)]
|
|
530
|
-
.concat(proofLines(saved), why ? [why] : []),
|
|
585
|
+
.concat(proofLines(saved), pageLine, why && !weak ? [why] : []),
|
|
531
586
|
}));
|
|
532
587
|
}
|
|
533
588
|
}
|
|
534
589
|
|
|
535
590
|
if (findings.length) return { outcome: 'found', findings: findings };
|
|
536
|
-
|
|
591
|
+
// Refused: a 4xx, or a form's redirect to the sign-in page or with an error.
|
|
592
|
+
const refused = writesOf(variant).filter((w) => (w.status >= 400 && w.status < 500) || (w.status >= 300 && w.status < 400 && !ok(w)) || sentBackWrites.includes(w));
|
|
537
593
|
// The form's own checks (required, min, max, type) stopped the submit:
|
|
538
594
|
// nothing reached the server, so the server was not tested - which is not
|
|
539
595
|
// the same as the app handling it. Found on blind round 17: an empty
|
|
@@ -545,7 +601,7 @@ function judgeVariant(clean, variant, plan) {
|
|
|
545
601
|
}
|
|
546
602
|
return {
|
|
547
603
|
outcome: 'held',
|
|
548
|
-
why: refused.length ? 'the app refused it (' + probe.writeWords(refused[0]) + ')'
|
|
604
|
+
why: refused.length ? (sentBackWrites.includes(refused[0]) ? 'the app sent the form back instead of saving it (' : 'the app refused it (') + probe.writeWords(refused[0]) + ')'
|
|
549
605
|
: saved.length ? 'saved, and nothing broke' : 'nothing was saved and nothing broke',
|
|
550
606
|
findings: [],
|
|
551
607
|
};
|
|
@@ -677,7 +733,7 @@ async function run(root, flows, options) {
|
|
|
677
733
|
|
|
678
734
|
const report = {
|
|
679
735
|
check: 'edge', runId: runId, startedAt: new Date().toISOString(), endedAt: null,
|
|
680
|
-
stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [],
|
|
736
|
+
stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [], writeHostsFrom: 'page+vendor',
|
|
681
737
|
};
|
|
682
738
|
|
|
683
739
|
// 1. The plain replay of every flow that has a field to try.
|
|
@@ -925,7 +981,19 @@ async function run(root, flows, options) {
|
|
|
925
981
|
report.notTested.push(Object.assign(label, { why: !results[p.id] && stopped ? 'stopped before it finished' : v.verdict.why }));
|
|
926
982
|
}
|
|
927
983
|
else if (v.verdict.outcome === 'held') report.held.push(Object.assign(label, { why: v.verdict.why }));
|
|
928
|
-
for (const f of v.verdict.findings)
|
|
984
|
+
for (const f of v.verdict.findings) {
|
|
985
|
+
// A create flow and an edit flow posting to the same address with the
|
|
986
|
+
// same field are one problem in the server, said once (blind round 17:
|
|
987
|
+
// each value was reported twice).
|
|
988
|
+
const first = f.same && report.findings.find((g) => g.same === f.same && g.status === f.status);
|
|
989
|
+
if (first) {
|
|
990
|
+
first.alsoIn = (first.alsoIn || 0) + 1;
|
|
991
|
+
if (first.alsoIn <= 3) first.proof.push('The same in "' + f.flow + '" (' + f.where + '): ' + f.sameRequest + '.');
|
|
992
|
+
report.held.push(Object.assign({}, label, { why: 'the same as in "' + first.flow + '" (' + first.where + ') - said once, there' }));
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
report.findings.push(f);
|
|
996
|
+
}
|
|
929
997
|
for (const w of writesOf(results[p.id])) if (leftBehind(w)) report.testData.push({ flow: p.flow.title, value: KINDS[p.kind].words, request: probe.writeWords(w) });
|
|
930
998
|
}
|
|
931
999
|
// Markup, judged across every run of this press.
|
|
@@ -1102,6 +1170,7 @@ module.exports = { formStops: formStops, straightManySource: straightManySource,
|
|
|
1102
1170
|
straightSource: straightSource,
|
|
1103
1171
|
judgeMarkers: judgeMarkers,
|
|
1104
1172
|
fixPrompt: fixPrompt,
|
|
1173
|
+
recordStem: recordStem,
|
|
1105
1174
|
run: run,
|
|
1106
1175
|
recheck: recheck,
|
|
1107
1176
|
save: save,
|
package/kryptheon-heal.js
CHANGED
|
@@ -203,6 +203,46 @@ function watch(page, options) {
|
|
|
203
203
|
return real.apply(locator, args);
|
|
204
204
|
}
|
|
205
205
|
|
|
206
|
+
// A text step whose words hold a count: "Projects3", "Shared Launch room 0".
|
|
207
|
+
// Found on blind round 19: three recordings clicked such text, each replay
|
|
208
|
+
// added a record, the count moved, and they never passed again - so none of
|
|
209
|
+
// the checks that replay them ever ran. Followed only when the same words
|
|
210
|
+
// with other numbers are on the page exactly once.
|
|
211
|
+
const byText = page.getByText.bind(page);
|
|
212
|
+
async function textAct(locator, real, method, args, text) {
|
|
213
|
+
const there = await locator
|
|
214
|
+
.first()
|
|
215
|
+
.waitFor({ state: 'attached', timeout: probeMs })
|
|
216
|
+
.then(() => true, () => false);
|
|
217
|
+
if (!there) {
|
|
218
|
+
const pattern = countPattern(text);
|
|
219
|
+
if (pattern) {
|
|
220
|
+
const replacement = byText(pattern);
|
|
221
|
+
const count = await replacement.count().catch(() => 0);
|
|
222
|
+
if (count === 1) {
|
|
223
|
+
const now = String(await replacement.innerText({ timeout: 2000 }).catch(() => '')).replace(/\s+/g, ' ').trim().slice(0, 120);
|
|
224
|
+
if (now) {
|
|
225
|
+
renamed.push({ role: 'text', was: text, now: now, counted: true });
|
|
226
|
+
return replacement[method].apply(replacement, args);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return real.apply(locator, args);
|
|
232
|
+
}
|
|
233
|
+
page.getByText = function (text, textOptions) {
|
|
234
|
+
const locator = byText(text, textOptions);
|
|
235
|
+
if (typeof text !== 'string' || !countPattern(text)) return locator;
|
|
236
|
+
for (const method of ACTIONS) {
|
|
237
|
+
const real = locator[method];
|
|
238
|
+
if (typeof real !== 'function') continue;
|
|
239
|
+
locator[method] = function () {
|
|
240
|
+
return textAct(locator, real, method, Array.prototype.slice.call(arguments), text);
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
return locator;
|
|
244
|
+
};
|
|
245
|
+
|
|
206
246
|
page.getByRole = function (role, roleOptions) {
|
|
207
247
|
const locator = byRole(role, roleOptions);
|
|
208
248
|
const name = roleOptions && typeof roleOptions.name === 'string' ? roleOptions.name : null;
|
|
@@ -236,8 +276,24 @@ function mergeMemory(previous, learned) {
|
|
|
236
276
|
return out;
|
|
237
277
|
}
|
|
238
278
|
|
|
279
|
+
/**
|
|
280
|
+
* The same words with any number in them: a regular expression for text whose
|
|
281
|
+
* numbers are a count, or null when the words without their numbers are too
|
|
282
|
+
* few to say which text it is (at least four letters must be left).
|
|
283
|
+
*/
|
|
284
|
+
function countPattern(text) {
|
|
285
|
+
const value = String(text == null ? '' : text);
|
|
286
|
+
if (!/\d/.test(value)) return null;
|
|
287
|
+
if (value.replace(/[^\p{L}]/gu, '').length < 4) return null;
|
|
288
|
+
const parts = value.split(/\d+/).map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '\\s*'));
|
|
289
|
+
return new RegExp(parts.join('\\s*\\d+\\s*'), 'i');
|
|
290
|
+
}
|
|
291
|
+
|
|
239
292
|
/** The report line, shared by the terminal and the history. */
|
|
240
293
|
function describeRename(entry) {
|
|
294
|
+
if (entry.role === 'text' && entry.counted) {
|
|
295
|
+
return 'The text "' + entry.was + '" now reads "' + entry.now + '" - only its number changed, so I clicked it and carried on.';
|
|
296
|
+
}
|
|
241
297
|
const what = entry.role === 'link' ? 'link' : entry.role === 'button' ? 'button' : entry.role;
|
|
242
298
|
return 'The ' + what + ' "' + entry.was + '" is now called "' + entry.now + '" - I used "' + entry.now + '" and carried on.';
|
|
243
299
|
}
|
|
@@ -246,6 +302,7 @@ module.exports = {
|
|
|
246
302
|
PROBE_MS: PROBE_MS,
|
|
247
303
|
namesByRole: namesByRole,
|
|
248
304
|
decideRename: decideRename,
|
|
305
|
+
countPattern: countPattern,
|
|
249
306
|
samePage: samePage,
|
|
250
307
|
pageNames: pageNames,
|
|
251
308
|
watch: watch,
|