kryptheon 0.1.49 → 0.1.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,8 @@ const { spawnSync } = require('child_process');
25
25
  const store = require('./kryptheon-store.js');
26
26
  const changes = require('./kryptheon-changes.js');
27
27
  const autostart = require('./kryptheon-autostart.js');
28
+ const selectors = require('./kryptheon-selectors.js');
29
+ const replay = require('./kryptheon-replay.js');
28
30
  const VERSION = require('./package.json').version;
29
31
 
30
32
  /**
@@ -49,7 +51,9 @@ const CHECKS = [
49
51
  { id: 'edge', label: 'Input and edge cases', available: true, asks: true,
50
52
  what: 'Replays your recorded forms with one field at a time given an unusual value - empty, 5000 characters, an HTML tag, emoji, a negative number, a number just past the lowest or highest the page allows, a date that does not exist - and reports only what it can prove: a page error, a 5xx, a bad value saved, a tag shown as HTML. It adds test data to your app, so it runs only when you press its button.' },
51
53
  { id: 'reliability', label: 'Reliability', available: true, asks: true,
52
- what: 'Replays each recorded flow three times and reports one that sometimes works and sometimes fails, with the failed run as proof; and clicks each step that saves twice, quickly, reporting when that saves twice, with both answers as proof; on an app on this computer, sends each save twice at the same moment, reporting a server that makes two records; and looks for what each save added on the page right after it, reporting a page that shows it only when opened again; and at phone size clicks every button with saving blocked, reporting a click that makes the page throw an error. It adds test data to your app, so it runs only when you press its button.' },
54
+ what: 'Replays each recorded flow and reports one that sometimes works and sometimes fails, with the failed run as proof; and clicks each step that saves twice, quickly, reporting when that saves twice, with both answers as proof; on an app on this computer, sends each save twice at the same moment, reporting a server that makes two records or fails; and looks for what each save added on the page right after it, reporting a page that shows it only when opened again; after an edit, opens the form again and reports a field that kept its old value; and at phone size clicks every button with saving blocked, reporting a click - or a page as it loads - that throws an error. It adds test data to your app, so it runs only when you press its button.' },
55
+ { id: 'payments', label: 'Payments: a webhook sent twice', available: true,
56
+ what: 'Real payments in Razorpay\'s or Stripe\'s TEST mode, through a proxy on this computer, each checked as it comes until you press Stop. Its signed webhook is sent to your app again, then once with a wrong signature, and your database is read before and after each - reporting the plan, credits or order given a second time, a repeat answered with an error, or a forged webhook accepted. Live mode is never sent again. It runs only when you press its button.' },
53
57
  { id: 'api', label: 'API', available: false },
54
58
  { id: 'backend', label: 'Backend', available: false },
55
59
  { id: 'synthetic', label: 'Synthetic test users', available: false },
@@ -57,6 +61,24 @@ const CHECKS = [
57
61
 
58
62
  const CHECK_IDS = new Set(CHECKS.map((c) => c.id));
59
63
 
64
+ // What no check here looks at, said plainly so a clean report is never read
65
+ // as "all of it is fine". Each blind round's misses that no check covers are
66
+ // here (rounds 19 and 20: roles, payment rules, counts and searches).
67
+ const NOT_CHECKED = [
68
+ { what: 'Who can do what',
69
+ why: 'Whether a lower role - a member, a customer - can open, change or delete what only a higher role should, or reach another account\'s record by changing an id in the address. Kryptheon replays each recording as the person who made it; it never tries one account\'s actions as another.',
70
+ yourself: 'Sign in as each role and open the higher role\'s pages and actions directly, by their address.' },
71
+ { what: 'Payment rules beyond a repeat',
72
+ why: 'Only the same webhook sent again, and a copy with a wrong signature, are tried. Not tried: a webhook with no signature at all, an amount or currency that does not match the order, buying again while a plan is active, or when a refund takes access away.',
73
+ yourself: 'Read your webhook handler for each of these, and test them in your provider\'s test mode.' },
74
+ { what: 'Whether numbers and lists are right',
75
+ why: 'A count or total that leaves something out, a search or filter that drops results. Kryptheon sees what a page shows, not what it should show.',
76
+ yourself: 'Compare each count and search result with what you know is in your data.' },
77
+ { what: 'Permissions decided in your server code',
78
+ why: 'The database check tests rules the database itself enforces (row level security, grants). An app that decides who sees what in its own server code - with plain Postgres and no such rules - is not tested there.',
79
+ yourself: 'Same as "Who can do what": try each role\'s requests against each other role\'s records.' },
80
+ ];
81
+
60
82
  function readJson(file) {
61
83
  try {
62
84
  let raw = fs.readFileSync(file, 'utf8');
@@ -167,9 +189,12 @@ function riskyPartsLine(f) {
167
189
  }
168
190
 
169
191
  /** When this flow last passed, from the run history, or null. */
170
- function lastPassOf(history, title) {
192
+ function lastPassOf(history, title, specFile) {
193
+ // By file too when both say one: round 18 had four recordings all titled
194
+ // "Round 18 shop", and one's pass was read as another's.
195
+ const same = (x) => x.title === title && (!specFile || !x.specFile || x.specFile === specFile);
171
196
  for (let i = history.length - 1; i >= 0; i--) {
172
- const t = (history[i].tests || []).find((x) => x.title === title);
197
+ const t = (history[i].tests || []).find(same);
173
198
  if (t && t.status === 'passed') return history[i].runAt;
174
199
  }
175
200
  return null;
@@ -245,6 +270,70 @@ const PROBE_WHAT = {
245
270
  const PROBE_WORDS = { reliability: { marker: false } };
246
271
  const PROBE_CHECKS = ['edge', 'reliability'];
247
272
 
273
+ // Loaded when it is first needed, so a page that never shows payments never
274
+ // reads it (and pg is only ever loaded by a run).
275
+ function paymentsRun() {
276
+ return require('./kryptheon-payments-run.js');
277
+ }
278
+
279
+ /**
280
+ * Every payment run kept in the store's payments folder, oldest first. A
281
+ * store from before the traces has only the last report.
282
+ */
283
+ function readPaymentRuns(paths, last) {
284
+ const dir = path.join(paths.dir, paymentsRun().TRACE_DIR);
285
+ let names = [];
286
+ try {
287
+ names = fs.readdirSync(dir).filter((n) => /\.json$/.test(n)).sort();
288
+ } catch (err) {
289
+ names = [];
290
+ }
291
+ const runs = names.map((n) => readJson(path.join(dir, n))).filter((r) => r && Array.isArray(r.results));
292
+ return runs.length ? runs : last ? [last] : [];
293
+ }
294
+
295
+ /** The words before the payment check's yes, and the app it would start from. */
296
+ function paymentsAsk(saved, last) {
297
+ const run = paymentsRun();
298
+ const local = saved && require('./kryptheon-payments.js').isLocalAddress(saved) ? saved : '';
299
+ const app = local || (last && last.app) || 'http://localhost:3000';
300
+ return { app: app, schema: (last && last.schema) || 'public', port: run.DEFAULT_PORT, lines: run.consentLines(app, (last && last.schema) || 'public', run.DEFAULT_PORT) };
301
+ }
302
+
303
+ /**
304
+ * What a fresh payment check says about one payment finding. Fixed only when
305
+ * the run tested a webhook of the same kind and the finding is gone; a run
306
+ * that tested nothing, or tested another event, proves nothing about it.
307
+ */
308
+ function paymentsVerdict(target, before, after, ran, report) {
309
+ const tested = ((report && report.results) || []).filter((r) => !(r.notTested || []).length);
310
+ // Judged by the re-check's own run: older runs stay in the store, so the
311
+ // list after it still holds the finding until this verdict is kept.
312
+ const foundAgain = tested.some((r) => target.where === r.path + ' (' + r.event.type + ')' &&
313
+ (r.findings || []).some((f) => paymentsRun().findingLines(r, f)[0].replace(/^(CRITICAL|HIGH|MEDIUM)\s+/, '') === target.headline));
314
+ let verdict;
315
+ let why = '';
316
+ if (!ran || !ran.ok) {
317
+ verdict = 'could not confirm';
318
+ why = (ran && ran.why) || 'the payment check could not run';
319
+ } else if (foundAgain) {
320
+ verdict = 'still open';
321
+ } else if (!tested.some((r) => target.where === r.path + ' (' + r.event.type + ')')) {
322
+ verdict = 'could not confirm';
323
+ why = tested.length ? 'that run tested a different webhook, not ' + target.where : 'no webhook was tested in that run';
324
+ } else {
325
+ verdict = 'fixed';
326
+ }
327
+ const had = new Set(before.map((f) => f.id));
328
+ return {
329
+ id: target.id, at: new Date().toISOString(), check: 'payments', where: target.where, headline: target.headline,
330
+ verdict: verdict, why: why,
331
+ newProblems: after.filter((f) => !had.has(f.id) && f.check === 'payments').map((f) => ({
332
+ id: f.id, check: f.check, severity: f.severity, status: f.status, where: f.where, headline: f.headline,
333
+ })),
334
+ };
335
+ }
336
+
248
337
  /**
249
338
  * The order someone new should go in, one step at a time. A step is "done"
250
339
  * only when it has actually run and left a record - never because nothing was
@@ -426,6 +515,33 @@ function nightlyLine(nightly) {
426
515
  * evidence, confidence, where, what, and the fix - so the page can show them
427
516
  * the same way whichever check found them.
428
517
  */
518
+ /**
519
+ * The step a failed run stopped at, when its selector rests on data that
520
+ * moves (kryptheon-selectors.js): { line, selector, why }, or null.
521
+ */
522
+ function movingStep(root, t) {
523
+ const line = t.failure && t.failure.line;
524
+ if (!line || !t.specFile) return null;
525
+ // More than one of it on the page is a different reason, already said.
526
+ if (/has \d+ of them/.test(String(t.failure.plainLanguage || ''))) return null;
527
+ let source;
528
+ try {
529
+ source = fs.readFileSync(path.resolve(root, String(t.specFile)), 'utf8');
530
+ } catch (err) {
531
+ return null;
532
+ }
533
+ const hit = selectors.findFragileSelectors(source).find((f) => f.line === line && f.reasons.some((r) => r.kind === 'number' || r.kind === 'data'));
534
+ // A row of a list picked by what it says: the words are a record's, and
535
+ // records change (blind round 20: "J Juniper Alex Parker Brush").
536
+ const text = String(source.split('\n')[line - 1] || '');
537
+ if (!hit && replay.ROW_PICK.test(text)) {
538
+ return { line: line, selector: selectors.selectorChain(text.trim()), why: 'it picks a row by what that record says, which changes when the record does' };
539
+ }
540
+ if (!hit) return null;
541
+ const reason = hit.reasons.find((r) => r.kind === 'number' || r.kind === 'data');
542
+ return { line: line, selector: hit.selector, why: reason.why + (reason.kind === 'number' ? ' that can change as data is added' : '') };
543
+ }
544
+
429
545
  function buildState(root, env) {
430
546
  const paths = store.pathsFor(root, env);
431
547
  const config = readConfig(paths);
@@ -446,17 +562,32 @@ function buildState(root, env) {
446
562
  if (lastRun && config.enabled.regression) {
447
563
  for (const t of lastRun.tests || []) {
448
564
  if (t.status !== 'failed' && t.status !== 'timedOut') continue;
565
+ // A recording that has never passed has not broken: nothing is known
566
+ // to have worked. Said as a thing to check - the recording or the app -
567
+ // as the replay's own summary says ("0 broken, 1 has no baseline").
568
+ // Found on a blind test (round 18): it was listed as "HIGH confirmed -
569
+ // broke" while the summary said nothing was broken.
570
+ const passedBefore = lastPassOf(history, t.title, t.specFile);
571
+ // Never passed, and stopped on a step whose selector holds a number or
572
+ // a date: the recording caught data that moves. Found on blind round
573
+ // 19: three recordings clicked "Projects3" and "fgff gffgddg 0" - counts
574
+ // that grew with every replay - and each was only "the recording or
575
+ // the app?".
576
+ const moved = !passedBefore && movingStep(root, t);
449
577
  findings.push({
450
578
  check: 'regression',
451
- severity: 'HIGH',
452
- status: 'confirmed',
453
- evidence: 'runtime confirmed',
454
- confidence: 'high',
579
+ severity: passedBefore ? 'HIGH' : moved ? 'LOW' : 'MEDIUM',
580
+ status: passedBefore ? 'confirmed' : 'verification required',
581
+ evidence: passedBefore ? 'runtime confirmed' : 'seen once',
582
+ confidence: passedBefore ? 'high' : 'medium',
455
583
  where: String((t.failure && t.failure.file) || t.specFile || 'tests').split('\\').join('/') +
456
584
  (t.failure && t.failure.line ? ':' + t.failure.line : ''),
457
- headline: 'The recorded flow "' + t.title + '" broke.',
585
+ headline: passedBefore ? 'The recorded flow "' + t.title + '" broke.'
586
+ : moved ? 'The recorded flow "' + t.title + '" has never passed: it stops at line ' + moved.line + ', ' + moved.selector + ', and ' + moved.why +
587
+ ' - most likely the recording, not the app. Record that step again on something that does not change.'
588
+ : 'The recorded flow "' + t.title + '" has never passed - the recording or the app?',
458
589
  detail: (t.failure && t.failure.plainLanguage) || '',
459
- fixPrompt: flowPrompt(t, lastPassOf(history, t.title)),
590
+ fixPrompt: flowPrompt(t, passedBefore),
460
591
  when: lastRun.runAt,
461
592
  });
462
593
  }
@@ -507,6 +638,48 @@ function buildState(root, env) {
507
638
  }
508
639
  }
509
640
 
641
+ // The payment check: every run kept in the store, not only the last. A
642
+ // finding stays until a re-check says FIXED. Found on a blind test (round
643
+ // 18): credits given twice was found by one payment, the next payment (a
644
+ // different product on the same webhook) held, and the finding vanished.
645
+ const paymentsLast = readJson(path.join(paths.dir, paymentsRun().REPORT_FILE));
646
+ const paymentRuns = readPaymentRuns(paths, paymentsLast);
647
+ const paymentsFixedAt = {};
648
+ for (const a of readHistory(paths.fixes, 500)) {
649
+ if (a.check === 'payments' && a.verdict === 'fixed' && (!paymentsFixedAt[a.id] || a.at > paymentsFixedAt[a.id])) paymentsFixedAt[a.id] = a.at;
650
+ }
651
+ const paymentFindings = new Map();
652
+ if (config.enabled.payments) for (const run of paymentRuns) {
653
+ for (const r of run.results || []) {
654
+ for (const f of r.findings || []) {
655
+ const words = paymentsRun().findingLines(r, f);
656
+ const item = {
657
+ check: 'payments',
658
+ severity: f.severity,
659
+ // A refund that took nothing back is seen, not proven wrong.
660
+ status: f.toCheck ? 'verification required' : 'confirmed',
661
+ evidence: f.toCheck ? 'seen once' : 'runtime confirmed',
662
+ confidence: f.toCheck ? 'medium' : 'high',
663
+ where: r.path + ' (' + r.event.type + ')',
664
+ headline: words[0].replace(/^(CRITICAL|HIGH|MEDIUM)\s+/, ''),
665
+ // What changed, then the whole trace: each delivery, when, its
666
+ // answer, and the rows it wrote - so the proof is on the finding.
667
+ detail: words.slice(2).map((l) => l.trim()).join('\n') +
668
+ (r.trace ? '\n\n' + paymentsRun().traceLines(r).join('\n') : '') +
669
+ (run.traceFile ? '\n\nKept on this computer: ' + run.traceFile : ''),
670
+ fixPrompt: paymentsRun().fixPrompt(r, f),
671
+ when: run.at,
672
+ kind: f.kind,
673
+ };
674
+ // The newest sighting of each finding is the one shown.
675
+ paymentFindings.set(changes.findingId(item), item);
676
+ }
677
+ }
678
+ }
679
+ for (const [id, item] of paymentFindings) {
680
+ if (!paymentsFixedAt[id] || paymentsFixedAt[id] < item.when) findings.push(item);
681
+ }
682
+
510
683
  let nightWhen = null;
511
684
  try {
512
685
  nightWhen = fs.statSync(paths.nightLast).mtime.toISOString();
@@ -570,7 +743,11 @@ function buildState(root, env) {
570
743
  f.id = changes.findingId(f);
571
744
  f.attempts = attempts.filter((a) => a.id === f.id).map((a) => ({ at: a.at, verdict: a.verdict, why: a.why || '', newProblems: (a.newProblems || []).length }));
572
745
  forAll.push({ id: f.id, severity: f.severity, headline: f.headline, where: f.where, prompt: f.fixPrompt || '' });
573
- if (f.fixPrompt && PROBE_CHECKS.includes(f.check)) {
746
+ if (f.fixPrompt && f.check === 'payments') {
747
+ // Proving it needs a new payment: the old webhook has been handled.
748
+ f.fixPrompt += '\n\nWhen you have fixed it, prove it: press Re-check on this finding in the Kryptheon dashboard ' +
749
+ '(or run npx kryptheon payments) and make one more test payment. It is fixed only when that says FIXED.';
750
+ } else if (f.fixPrompt && PROBE_CHECKS.includes(f.check)) {
574
751
  // Re-checking runs the flow in the app again, so it adds a little test
575
752
  // data: --yes says yes for an app on this computer, and is refused for
576
753
  // any other address - there the person must answer.
@@ -603,6 +780,12 @@ function buildState(root, env) {
603
780
  : 'last run held';
604
781
  }
605
782
  else if (c.id === 'frontend') state = !code ? 'never run' : (code.findings || []).length ? 'things to check' : 'nothing found';
783
+ else if (c.id === 'payments') {
784
+ state = !paymentsLast ? 'never run'
785
+ : findings.some((f) => f.check === 'payments') ? 'problems found'
786
+ : !(paymentsLast.results || []).some((r) => !(r.notTested || []).length) ? 'nothing was tested'
787
+ : 'last run held';
788
+ }
606
789
  else if (c.id === 'data') {
607
790
  const ranNightly = nightly && nightly.ranAt && !nightly.stopped;
608
791
  state = !night && !ranNightly ? 'never run here'
@@ -614,6 +797,9 @@ function buildState(root, env) {
614
797
  // A check that adds data to the app: never in "Run all", never on its
615
798
  // own - only its own button, after the page has shown these words.
616
799
  asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env, PROBE_WORDS[c.id]) : null,
800
+ // What the payment check says before its yes, and the app it starts
801
+ // from: the saved address when it is this computer's, else the usual one.
802
+ payments: c.id === 'payments' && config.enabled.payments ? paymentsAsk(readAppAddress(paths), paymentsLast) : null,
617
803
  lastRun: probeLast[c.id] ? {
618
804
  at: probeLast[c.id].endedAt, stopped: !!probeLast[c.id].stopped, runsDone: probeLast[c.id].runsDone, runsPlanned: probeLast[c.id].runsPlanned,
619
805
  held: (probeLast[c.id].held || []).length,
@@ -660,6 +846,7 @@ function buildState(root, env) {
660
846
  looks: looks,
661
847
  proven: proven,
662
848
  checks: checks,
849
+ notChecked: NOT_CHECKED,
663
850
  findings: findings,
664
851
  allFixPrompt: combinedPrompt(forAll),
665
852
  runs: history.slice().reverse().map((r) => ({ runAt: r.runAt, status: r.status, passed: r.passed, failed: r.failed, durationMs: r.durationMs })),
@@ -691,8 +878,10 @@ function buildState(root, env) {
691
878
 
692
879
  // What each run is called on the page, and the command that does the same in
693
880
  // a terminal - shown under every button, for when the page cannot be used.
694
- const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking' };
695
- const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge', reliability: 'npx kryptheon reliability' };
881
+ const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking',
882
+ payments: 'Waiting for test payments, and sending each webhook again' };
883
+ const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge', reliability: 'npx kryptheon reliability',
884
+ payments: 'npx kryptheon payments' };
696
885
  const DB_ACTIONS = {
697
886
  scan: { title: 'Checking your database', command: 'npx kryptheon-night' },
698
887
  install: { title: 'Setting up the nightly check', command: 'npx kryptheon-night install' },
@@ -1087,6 +1276,43 @@ function createServer(root, options) {
1087
1276
  return begin('db', DB_ACTIONS[action].title, DB_ACTIONS[action].command,
1088
1277
  (say) => opts.db(action, connection, say), secretsOf(connection));
1089
1278
  }
1279
+ if (url.pathname === '/api/payments' && typeof opts.payments === 'function') {
1280
+ // One test payment, its webhook sent again. Its copies are processed
1281
+ // by the app, so only with the yes from the page, only for an app on
1282
+ // this computer, and the connection string is handled like the
1283
+ // database check's: given to this job only, scrubbed from its lines.
1284
+ if (!readConfig(store.pathsFor(root, env)).enabled.payments) return json(res, 400, { ok: false, why: 'the payment check is switched off' });
1285
+ if (input.consent !== true) return json(res, 400, { ok: false, why: 'its copies are processed by your app, so it runs only after you say yes' });
1286
+ const connection = connectionFrom(input.connection);
1287
+ if (!connection) return json(res, 400, { ok: false, why: 'paste your development database\'s connection string in "Connect your database" first' });
1288
+ const app = appAddress(typeof input.app === 'string' ? input.app : '');
1289
+ const full = app && !/^https?:\/\//i.test(app) ? 'http://' + app : app;
1290
+ if (!full || !require('./kryptheon-payments.js').isLocalAddress(full)) {
1291
+ return json(res, 400, { ok: false, why: 'the payment check only runs against an app on this computer - start it locally and give its address, like http://localhost:3000' });
1292
+ }
1293
+ const schema = typeof input.schema === 'string' && /^[A-Za-z_][A-Za-z0-9_]{0,62}$/.test(input.schema) ? input.schema : 'public';
1294
+ const run = { app: full, schema: schema, connection: connection, testMode: input.testMode === true };
1295
+ const id = input.id === undefined ? null : String(input.id);
1296
+ if (id !== null) {
1297
+ if (!/^[0-9a-f]{8}$/.test(id)) return json(res, 400, { ok: false, why: 'not a finding id' });
1298
+ const target = buildState(root, env).findings.find((f) => f.id === id && f.check === 'payments');
1299
+ if (!target) return json(res, 404, { ok: false, why: 'no open payment finding with that id' });
1300
+ return begin('recheck', 'Re-checking: ' + target.headline, RUN_COMMAND.payments, (say, control) => {
1301
+ const before = buildState(root, env).findings;
1302
+ return Promise.resolve(opts.payments(run, say, control)).then((ran) => {
1303
+ const report = readJson(path.join(store.pathsFor(root, env).dir, paymentsRun().REPORT_FILE));
1304
+ const attempt = paymentsVerdict(target, before, buildState(root, env).findings, ran, report);
1305
+ try {
1306
+ fs.appendFileSync(store.pathsFor(root, env).fixes, JSON.stringify(attempt) + '\n', 'utf8');
1307
+ } catch (err) {
1308
+ attempt.saveError = err.message;
1309
+ }
1310
+ return Object.assign({ ok: true, summary: attempt.verdict }, attempt);
1311
+ });
1312
+ }, secretsOf(connection));
1313
+ }
1314
+ return begin('payments', RUN_TITLE.payments, RUN_COMMAND.payments, (say, control) => opts.payments(run, say, control), secretsOf(connection));
1315
+ }
1090
1316
  if (url.pathname === '/api/app') {
1091
1317
  // The app's web address: saved for this project, so Record and the
1092
1318
  // outside check start from it. Empty clears it.
@@ -1157,6 +1383,8 @@ module.exports = {
1157
1383
  createJobs: createJobs,
1158
1384
  DB_ACTIONS: DB_ACTIONS,
1159
1385
  databaseVerdict: databaseVerdict,
1386
+ paymentsVerdict: paymentsVerdict,
1387
+ paymentsAsk: paymentsAsk,
1160
1388
  startSteps: startSteps,
1161
1389
  nextAction: nextAction,
1162
1390
  combinedPrompt: combinedPrompt,
package/kryptheon-edge.js CHANGED
@@ -152,14 +152,21 @@ function candidateFields(source) {
152
152
  lines.forEach((line, i) => {
153
153
  if (/\.fill\(/.test(line) && (replay.isSecretField(replay.nameInLine(line)) || /process\.env\./.test(line))) signInEnd = i + 1;
154
154
  });
155
- const seen = new Set();
155
+ // Each field once, by its last fill: that is the value the form sent. Found
156
+ // on blind round 20: "Duration" typed 6, then 62, then 60 - the first was
157
+ // taken as what was recorded, no request carried it, and no value past
158
+ // the form was ever sent.
159
+ const fills = fillSteps(text).filter((step) => step.line > signInEnd && !replay.isSecretField(step.name));
160
+ const last = new Map();
161
+ for (const step of fills) last.set(step.locator.replace(/\s+/g, ' '), step);
162
+ // In the order the fields were first typed into; each its last fill.
156
163
  const out = [];
157
- for (const step of fillSteps(text)) {
158
- if (step.line <= signInEnd) continue;
159
- if (replay.isSecretField(step.name)) continue;
160
- const key = step.locator.replace(/\s+/g, ' ');
164
+ const seen = new Set();
165
+ for (const first of fills) {
166
+ const key = first.locator.replace(/\s+/g, ' ');
161
167
  if (seen.has(key)) continue;
162
168
  seen.add(key);
169
+ const step = last.get(key);
163
170
  // Saved by a click on Save/Add/Create... or an Enter, before the flow
164
171
  // goes anywhere else.
165
172
  let saves = false;
@@ -342,12 +349,26 @@ function flatten(value, prefix, out) {
342
349
  return into;
343
350
  }
344
351
 
345
- const ok = (w) => w.status >= 200 && w.status < 300;
352
+ // Taken by the app: a 2xx, or a form's redirect after it saved (probe.accepted).
353
+ const ok = (w) => probe.accepted(w);
346
354
  const writesOf = (r) => ((r && r.events) || []).filter((e) => e.kind === 'write');
347
355
  const routeOf = (w) => w.method + ' ' + w.host + w.path;
348
356
  // A write the app accepted and keeps: not the sign-in, not a delete.
349
357
  const leftBehind = (w) => ok(w) && w.method !== 'DELETE' && !w.signIn;
350
358
 
359
+ /**
360
+ * The kind of record an address saves: its ids, and a trailing edit or new,
361
+ * taken off - POST /pets and POST /pets/7/edit both save a pet, and POST
362
+ * /settings saves something else. Found on blind round 20: a pet's "name"
363
+ * and the studio's "name" were said as one problem.
364
+ */
365
+ function recordStem(p) {
366
+ return String(p || '')
367
+ .replace(/\/(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24})(?=\/|$)/gi, '/:id')
368
+ .replace(/\/(edit|new|create|update)$/i, '')
369
+ .replace(/\/:id$/, '');
370
+ }
371
+
351
372
  /** Writes of the plain replay and of a variant, paired by where they went, in order. */
352
373
  function pairWrites(clean, variant) {
353
374
  const left = {};
@@ -490,7 +511,21 @@ function judgeVariant(clean, variant, plan) {
490
511
  }));
491
512
  }
492
513
 
493
- const saved = savedValues(clean, variant, field, plan.value);
514
+ // Sent back to the form, where the recorded save went on to another page:
515
+ // the form was sent back to be filled in again - the value was refused.
516
+ // Back to it by a redirect, or the form drawn again in a 200 with no record
517
+ // in it - the way a server-rendered form shows its errors.
518
+ const movedOn = (w) => w && ok(w) && w.status >= 300 && w.status < 400 && !probe.backToForm(w);
519
+ // Or both went back to the form, but with another message than the
520
+ // recorded save's (blind round 21: /settings?notice=Settings saved for the
521
+ // save, /settings?notice=Check the settings and try again for the value).
522
+ const answeredOtherwise = (s) => s.cleanWrite && ok(s.cleanWrite) && probe.backToForm(s.cleanWrite) && probe.backToForm(s.write) &&
523
+ probe.answerShape(s.cleanWrite) !== probe.answerShape(s.write);
524
+ const sentBack = (s) => s.write.shownAfter !== true && (answeredOtherwise(s) || (movedOn(s.cleanWrite) &&
525
+ (probe.backToForm(s.write) || (s.write.status >= 200 && s.write.status < 300 && s.write.answer == null))));
526
+ const allSaved = savedValues(clean, variant, field, plan.value);
527
+ const saved = allSaved.filter((s) => !sentBack(s));
528
+ const sentBackWrites = allSaved.filter(sentBack).map((s) => s.write);
494
529
  if (saved.length) {
495
530
  // What the server says it kept decides, when it says: a -5 it turned
496
531
  // into 0 was not saved as -5. Without an answer, the request decides.
@@ -521,19 +556,40 @@ function judgeVariant(clean, variant, plan) {
521
556
  : kind === 'long' && info.maxLength != null ? 'The field allows ' + info.maxLength + ' characters, but the server stored ' + LONG_LENGTH + '.'
522
557
  : kind === 'over' ? 'The page allows at most ' + info.maxLength + ' characters, but the server stored ' + String(plan.value).length + '.'
523
558
  : null;
559
+ // A form that redirects sends no record back. A redirect on to another
560
+ // page is the save going through (Post, Redirect, Get); one straight back
561
+ // to the form may be the form sent back to be filled in again - unless
562
+ // the page it went to shows the value.
563
+ const w0 = saved[0].write;
564
+ const redirected = w0.status >= 300 && w0.status < 400;
565
+ // Back to the form with the very answer the recorded save got - "Settings
566
+ // saved" for both - is the save going through.
567
+ const c0 = saved[0].cleanWrite;
568
+ const asSaved = redirected && c0 && ok(c0) && probe.backToForm(c0) && probe.backToForm(w0) && probe.answerShape(c0) === probe.answerShape(w0);
569
+ const weak = redirected && probe.backToForm(w0) && w0.shownAfter !== true && !asSaved;
570
+ const pageLine = redirected && w0.shownAfter === true ? ['The page it went to after saving, ' + w0.to + ', shows ' + show(plan.value) + '.']
571
+ : asSaved ? ['It answered as your recorded save did (' + w0.to + ') - the value was taken.']
572
+ : weak ? ['It answered with a redirect back to the form (' + w0.to + '), which does not say whether it kept the value.'] : [];
524
573
  if (problem || toCheck) {
574
+ const sure = problem && !weak;
525
575
  findings.push(Object.assign({}, base, {
526
- type: 'accepted', severity: problem ? 'MEDIUM' : 'LOW', status: problem ? 'confirmed' : 'verification required',
527
- headline: problem ? '"' + name + '" accepts ' + words + ', and the app saved it.'
576
+ // The same value taken into the same field by the same app, whichever
577
+ // flow sent it - a create and an edit of the same record.
578
+ same: saved[0].key ? kind + '|' + (w0.origin || w0.host || '') + recordStem(w0.path) + '|' + saved[0].key : null,
579
+ sameRequest: probe.writeWords(w0),
580
+ type: 'accepted', severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
581
+ headline: sure ? '"' + name + '" accepts ' + words + ', and the app saved it.'
582
+ : problem ? '"' + name + '" was sent ' + words + ' and the app did not refuse it - check whether it kept it.'
528
583
  : '"' + name + '" took ' + words + ' and the app ' + savedWords + ' - check that this is allowed.',
529
584
  proof: ['Flow: ' + plan.flow.title + ' (' + where + ')', 'Typed into "' + name + '": ' + show(plan.value)]
530
- .concat(proofLines(saved), why ? [why] : []),
585
+ .concat(proofLines(saved), pageLine, why && !weak ? [why] : []),
531
586
  }));
532
587
  }
533
588
  }
534
589
 
535
590
  if (findings.length) return { outcome: 'found', findings: findings };
536
- const refused = writesOf(variant).filter((w) => w.status >= 400 && w.status < 500);
591
+ // Refused: a 4xx, or a form's redirect to the sign-in page or with an error.
592
+ const refused = writesOf(variant).filter((w) => (w.status >= 400 && w.status < 500) || (w.status >= 300 && w.status < 400 && !ok(w)) || sentBackWrites.includes(w));
537
593
  // The form's own checks (required, min, max, type) stopped the submit:
538
594
  // nothing reached the server, so the server was not tested - which is not
539
595
  // the same as the app handling it. Found on blind round 17: an empty
@@ -545,7 +601,7 @@ function judgeVariant(clean, variant, plan) {
545
601
  }
546
602
  return {
547
603
  outcome: 'held',
548
- why: refused.length ? 'the app refused it (' + probe.writeWords(refused[0]) + ')'
604
+ why: refused.length ? (sentBackWrites.includes(refused[0]) ? 'the app sent the form back instead of saving it (' : 'the app refused it (') + probe.writeWords(refused[0]) + ')'
549
605
  : saved.length ? 'saved, and nothing broke' : 'nothing was saved and nothing broke',
550
606
  findings: [],
551
607
  };
@@ -677,7 +733,7 @@ async function run(root, flows, options) {
677
733
 
678
734
  const report = {
679
735
  check: 'edge', runId: runId, startedAt: new Date().toISOString(), endedAt: null,
680
- stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [],
736
+ stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [], writeHostsFrom: 'page+vendor',
681
737
  };
682
738
 
683
739
  // 1. The plain replay of every flow that has a field to try.
@@ -925,7 +981,19 @@ async function run(root, flows, options) {
925
981
  report.notTested.push(Object.assign(label, { why: !results[p.id] && stopped ? 'stopped before it finished' : v.verdict.why }));
926
982
  }
927
983
  else if (v.verdict.outcome === 'held') report.held.push(Object.assign(label, { why: v.verdict.why }));
928
- for (const f of v.verdict.findings) report.findings.push(f);
984
+ for (const f of v.verdict.findings) {
985
+ // A create flow and an edit flow posting to the same address with the
986
+ // same field are one problem in the server, said once (blind round 17:
987
+ // each value was reported twice).
988
+ const first = f.same && report.findings.find((g) => g.same === f.same && g.status === f.status);
989
+ if (first) {
990
+ first.alsoIn = (first.alsoIn || 0) + 1;
991
+ if (first.alsoIn <= 3) first.proof.push('The same in "' + f.flow + '" (' + f.where + '): ' + f.sameRequest + '.');
992
+ report.held.push(Object.assign({}, label, { why: 'the same as in "' + first.flow + '" (' + first.where + ') - said once, there' }));
993
+ continue;
994
+ }
995
+ report.findings.push(f);
996
+ }
929
997
  for (const w of writesOf(results[p.id])) if (leftBehind(w)) report.testData.push({ flow: p.flow.title, value: KINDS[p.kind].words, request: probe.writeWords(w) });
930
998
  }
931
999
  // Markup, judged across every run of this press.
@@ -1102,6 +1170,7 @@ module.exports = { formStops: formStops, straightManySource: straightManySource,
1102
1170
  straightSource: straightSource,
1103
1171
  judgeMarkers: judgeMarkers,
1104
1172
  fixPrompt: fixPrompt,
1173
+ recordStem: recordStem,
1105
1174
  run: run,
1106
1175
  recheck: recheck,
1107
1176
  save: save,
package/kryptheon-heal.js CHANGED
@@ -203,6 +203,46 @@ function watch(page, options) {
203
203
  return real.apply(locator, args);
204
204
  }
205
205
 
206
+ // A text step whose words hold a count: "Projects3", "Shared Launch room 0".
207
+ // Found on blind round 19: three recordings clicked such text, each replay
208
+ // added a record, the count moved, and they never passed again - so none of
209
+ // the checks that replay them ever ran. Followed only when the same words
210
+ // with other numbers are on the page exactly once.
211
+ const byText = page.getByText.bind(page);
212
+ async function textAct(locator, real, method, args, text) {
213
+ const there = await locator
214
+ .first()
215
+ .waitFor({ state: 'attached', timeout: probeMs })
216
+ .then(() => true, () => false);
217
+ if (!there) {
218
+ const pattern = countPattern(text);
219
+ if (pattern) {
220
+ const replacement = byText(pattern);
221
+ const count = await replacement.count().catch(() => 0);
222
+ if (count === 1) {
223
+ const now = String(await replacement.innerText({ timeout: 2000 }).catch(() => '')).replace(/\s+/g, ' ').trim().slice(0, 120);
224
+ if (now) {
225
+ renamed.push({ role: 'text', was: text, now: now, counted: true });
226
+ return replacement[method].apply(replacement, args);
227
+ }
228
+ }
229
+ }
230
+ }
231
+ return real.apply(locator, args);
232
+ }
233
+ page.getByText = function (text, textOptions) {
234
+ const locator = byText(text, textOptions);
235
+ if (typeof text !== 'string' || !countPattern(text)) return locator;
236
+ for (const method of ACTIONS) {
237
+ const real = locator[method];
238
+ if (typeof real !== 'function') continue;
239
+ locator[method] = function () {
240
+ return textAct(locator, real, method, Array.prototype.slice.call(arguments), text);
241
+ };
242
+ }
243
+ return locator;
244
+ };
245
+
206
246
  page.getByRole = function (role, roleOptions) {
207
247
  const locator = byRole(role, roleOptions);
208
248
  const name = roleOptions && typeof roleOptions.name === 'string' ? roleOptions.name : null;
@@ -236,8 +276,24 @@ function mergeMemory(previous, learned) {
236
276
  return out;
237
277
  }
238
278
 
279
+ /**
280
+ * The same words with any number in them: a regular expression for text whose
281
+ * numbers are a count, or null when the words without their numbers are too
282
+ * few to say which text it is (at least four letters must be left).
283
+ */
284
+ function countPattern(text) {
285
+ const value = String(text == null ? '' : text);
286
+ if (!/\d/.test(value)) return null;
287
+ if (value.replace(/[^\p{L}]/gu, '').length < 4) return null;
288
+ const parts = value.split(/\d+/).map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '\\s*'));
289
+ return new RegExp(parts.join('\\s*\\d+\\s*'), 'i');
290
+ }
291
+
239
292
  /** The report line, shared by the terminal and the history. */
240
293
  function describeRename(entry) {
294
+ if (entry.role === 'text' && entry.counted) {
295
+ return 'The text "' + entry.was + '" now reads "' + entry.now + '" - only its number changed, so I clicked it and carried on.';
296
+ }
241
297
  const what = entry.role === 'link' ? 'link' : entry.role === 'button' ? 'button' : entry.role;
242
298
  return 'The ' + what + ' "' + entry.was + '" is now called "' + entry.now + '" - I used "' + entry.now + '" and carried on.';
243
299
  }
@@ -246,6 +302,7 @@ module.exports = {
246
302
  PROBE_MS: PROBE_MS,
247
303
  namesByRole: namesByRole,
248
304
  decideRename: decideRename,
305
+ countPattern: countPattern,
249
306
  samePage: samePage,
250
307
  pageNames: pageNames,
251
308
  watch: watch,