kryptheon 0.1.46 → 0.1.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -265,6 +265,38 @@ it notices on the first replay - you are warned and must answer yourself.
265
265
  `npx kryptheon recheck <id>` tries a finding's value again: FIXED, or STILL
266
266
  OPEN with the new proof.
267
267
 
268
+ ## Reliability
269
+
270
+ ```
271
+ npx kryptheon reliability
272
+ ```
273
+
274
+ Asks two questions of your recorded flows:
275
+
276
+ - **Does it work every time?** Each flow is replayed three times. One that
277
+ fails and then works again, with nothing changed, is reported - with the
278
+ failed run as proof. It is called a problem only when your app answered
279
+ differently in the failed run (a 5xx or a page error the good runs did not
280
+ get); without that, the recording's own timing could be the cause, and it
281
+ is said as a thing to check. A flow that fails all three times is broken,
282
+ which `npx kryptheon check` reports. One that works once and then stops is
283
+ a thing to check: the first run's own data may be in the way.
284
+ - **What happens on a double-click?** Each step that saves (Save, Add,
285
+ Create, Submit...) is clicked twice, quickly. When that saves twice - more
286
+ successful POSTs to the same place than one click makes - it is reported
287
+ with both answers, and the two record ids the server named when it names
288
+ them. A server that answers both clicks with the same record saved it once,
289
+ and that is not reported. A double save is tried a second time before it is
290
+ called a problem.
291
+
292
+ Slower networks are not tried yet.
293
+
294
+ Like the input check, **it adds data to your app** - your flows' own saves,
295
+ replayed - and leaves it there. It runs only when you say yes, with the same
296
+ rules: `--yes` only for an app on this computer, at most 40 runs, Stop in the
297
+ dashboard. `npx kryptheon recheck <id>` replays a flaky flow five times (FIXED
298
+ only if all five work), or double-clicks the step again.
299
+
268
300
  ## What goes where
269
301
 
270
302
  Only your recordings go in your project, in `tests/`. Everything Kryptheon
package/bin/kryptheon.js CHANGED
@@ -101,6 +101,7 @@ function usage() {
101
101
  console.log(' npx kryptheon recheck [id] prove a fix: run the check again, and say FIXED or not');
102
102
  console.log(' npx kryptheon code read your frontend code for text inserted as HTML');
103
103
  console.log(' npx kryptheon edge try unusual values in your recorded forms (adds test data; asks first)');
104
+ console.log(' npx kryptheon reliability replay each flow 3 times, double-click what saves (adds test data; asks first)');
104
105
  console.log(' npx kryptheon dashboard the page above (also: npx kryptheon, in a terminal)');
105
106
  console.log('');
106
107
  console.log(' add --quiet to check for one line when everything passes');
@@ -1938,10 +1939,10 @@ function probeFlows() {
1938
1939
  * that are not this computer - none for --yes, which is only ever for this
1939
1940
  * computer - or null when the answer is no.
1940
1941
  */
1941
- async function probeConsent(what, addresses, yes) {
1942
+ async function probeConsent(what, addresses, yes, words) {
1942
1943
  const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
1943
1944
  console.log('');
1944
- for (const line of probe.consentLines(what, addresses)) console.log(' ' + line);
1945
+ for (const line of probe.consentLines(what, addresses, words)) console.log(' ' + line);
1945
1946
  console.log('');
1946
1947
  const live = probe.liveLooking(addresses);
1947
1948
  if (yes && !live.length) return [];
@@ -1962,27 +1963,47 @@ async function probeConsent(what, addresses, yes) {
1962
1963
  return said ? live : null;
1963
1964
  }
1964
1965
 
1965
- async function edgeCommand(args) {
1966
+ // The checks that run changed copies of the recordings, and what is said
1967
+ // about each before it starts. Both add data to the app; both ask first.
1968
+ const PROBE_CHECKS = {
1969
+ edge: {
1970
+ module: 'kryptheon-edge.js',
1971
+ what: 'Input and edge cases: replays your recorded forms with one field at a time given an unusual value ' +
1972
+ '(empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most 40 runs.',
1973
+ none: 'There is no recording to try values in. Record a flow that fills a form:',
1974
+ },
1975
+ reliability: {
1976
+ module: 'kryptheon-reliability.js',
1977
+ what: 'Reliability: replays each recorded flow three times, and clicks each step that saves (Save, Add, Create...) twice, quickly. At most 40 runs.',
1978
+ none: 'There is no recording to replay. Record one of your flows:',
1979
+ // It types nothing of its own, so nothing it saves carries the marker.
1980
+ words: { marker: false },
1981
+ },
1982
+ };
1983
+
1984
+ function probeModule(check) {
1985
+ return require(path.join(PACKAGE_DIR, PROBE_CHECKS[check].module));
1986
+ }
1987
+
1988
+ async function probeCommand(check, args) {
1966
1989
  const here = project.inspectProject(USER_DIR);
1967
1990
  if (!here.ok) {
1968
1991
  for (const line of project.noProjectLines(here)) console.error(line);
1969
1992
  return 1;
1970
1993
  }
1971
1994
  if (!openStore()) return 1;
1972
- const edge = require(path.join(PACKAGE_DIR, 'kryptheon-edge.js'));
1995
+ const runner = probeModule(check);
1973
1996
  const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
1974
1997
  const { flows } = probeFlows();
1975
1998
  if (!flows.length) {
1976
1999
  console.log('');
1977
- console.log(' There is no recording to try values in. Record a flow that fills a form:');
2000
+ console.log(' ' + PROBE_CHECKS[check].none);
1978
2001
  console.log(' npx kryptheon record <url>');
1979
2002
  console.log('');
1980
2003
  return 2;
1981
2004
  }
1982
- const addresses = edge.knownAddresses(USER_DIR, flows.map((f) => f.source));
1983
- const what = 'Input and edge cases: replays your recorded forms with one field at a time given an unusual value ' +
1984
- '(empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most ' + probe.MAX_RUNS + ' runs.';
1985
- const confirmed = await probeConsent(what, addresses, args.includes('--yes'));
2005
+ const addresses = probe.knownAddresses(USER_DIR, flows.map((f) => f.source));
2006
+ const confirmed = await probeConsent(PROBE_CHECKS[check].what, addresses, args.includes('--yes'), PROBE_CHECKS[check].words);
1986
2007
  if (!confirmed) return 2;
1987
2008
  if (!ensureBrowser()) return 2;
1988
2009
 
@@ -1991,14 +2012,14 @@ async function edgeCommand(args) {
1991
2012
  if (control) control.stop();
1992
2013
  };
1993
2014
  process.once('SIGINT', onSigint);
1994
- const report = await edge.run(USER_DIR, flows, {
2015
+ const report = await runner.run(USER_DIR, flows, {
1995
2016
  say: (line) => console.log(' ' + line),
1996
2017
  onControl: (c) => (control = c),
1997
2018
  confirmed: confirmed,
1998
2019
  });
1999
2020
  process.removeListener('SIGINT', onSigint);
2000
- edge.save(USER_DIR, report);
2001
- for (const line of edge.summaryLines(report)) console.log(' ' + line);
2021
+ runner.save(USER_DIR, report);
2022
+ for (const line of runner.summaryLines(report)) console.log(' ' + line);
2002
2023
  if (report.findings.some((f) => f.status === 'confirmed')) return 1;
2003
2024
  return report.flows.some((f) => f.state === 'tested') ? 0 : 2;
2004
2025
  }
@@ -2385,19 +2406,23 @@ const VERDICT_WORDS = {
2385
2406
  'could not confirm': 'COULD NOT CONFIRM',
2386
2407
  };
2387
2408
 
2388
- async function edgeRecheckCommand(target, options) {
2409
+ async function probeRecheckCommand(target, options) {
2389
2410
  const probe = require(path.join(PACKAGE_DIR, 'kryptheon-probe.js'));
2390
2411
  const { flows } = probeFlows();
2391
2412
  const file = String(target.where).replace(/:\d+$/, '');
2392
2413
  const flow = flows.find((f) => f.file === file);
2393
- const addresses = require(path.join(PACKAGE_DIR, 'kryptheon-edge.js')).knownAddresses(USER_DIR, flow ? [flow.source] : []);
2394
- const what = 'Re-check ' + target.id + ': replays "' + (flow ? flow.title : file) + '" as recorded, then once more with the value that found this. Two runs.';
2395
- const confirmed = await probeConsent(what, addresses, Boolean(options && options.yes));
2414
+ const addresses = probe.knownAddresses(USER_DIR, flow ? [flow.source] : []);
2415
+ const name = '"' + (flow ? flow.title : file) + '"';
2416
+ const what = 'Re-check ' + target.id + ': ' + (
2417
+ target.check === 'edge' ? 'replays ' + name + ' as recorded, then once more with the value that found this. Two runs.'
2418
+ : target.kind === 'flaky' ? 'replays ' + name + ' five times. It is fixed only if all five work.'
2419
+ : 'replays ' + name + ' once, then once more with this step double-clicked. Two runs.');
2420
+ const confirmed = await probeConsent(what, addresses, Boolean(options && options.yes), PROBE_CHECKS[target.check].words);
2396
2421
  if (!confirmed) return 2;
2397
2422
  let control = null;
2398
2423
  const onSigint = () => control && control.stop();
2399
2424
  process.once('SIGINT', onSigint);
2400
- const r = await edgeRecheck(target.id, (line) => console.log(' ' + line), { onStop: (fn) => (control = { stop: fn }) }, confirmed);
2425
+ const r = await probeRecheck(target.id, (line) => console.log(' ' + line), { onStop: (fn) => (control = { stop: fn }) }, confirmed);
2401
2426
  process.removeListener('SIGINT', onSigint);
2402
2427
  console.log('');
2403
2428
  console.log(' RE-CHECK ' + target.id + ' ' + target.where);
@@ -2445,8 +2470,8 @@ function recheckCommand(id, options) {
2445
2470
 
2446
2471
  // An input finding is re-checked by trying its value again, which adds test
2447
2472
  // data to the app - so it asks first, the way `edge` does.
2448
- const edgeTarget = dash.buildState(USER_DIR).findings.find((f) => f.id === String(id).trim() && f.check === 'edge');
2449
- if (edgeTarget) return edgeRecheckCommand(edgeTarget, options);
2473
+ const probeTarget = dash.buildState(USER_DIR).findings.find((f) => f.id === String(id).trim() && PROBE_CHECKS[f.check]);
2474
+ if (probeTarget) return probeRecheckCommand(probeTarget, options);
2450
2475
 
2451
2476
  const r = looker.recheck(USER_DIR, { frontend: () => readCodeAndSave(opened), regression: replayForLoop }, String(id).trim());
2452
2477
  console.log('');
@@ -2608,18 +2633,18 @@ function runFromDashboard(id, opened, say) {
2608
2633
  * this process, where its Stop can reach the browser run.
2609
2634
  */
2610
2635
  async function probeFromDashboard(id, say, control, confirmed) {
2611
- if (id !== 'edge') return { ok: false, why: '"' + id + '" cannot be run from the dashboard' };
2612
- const edge = require(path.join(PACKAGE_DIR, 'kryptheon-edge.js'));
2636
+ if (!PROBE_CHECKS[id]) return { ok: false, why: '"' + id + '" cannot be run from the dashboard' };
2637
+ const runner = probeModule(id);
2613
2638
  const { flows } = probeFlows();
2614
- if (!flows.length) return { ok: false, why: 'there is no recording that can be replayed - record a flow that fills a form' };
2639
+ if (!flows.length) return { ok: false, why: 'there is no recording that can be replayed - record one of your flows' };
2615
2640
  if (!ensureBrowser()) return { ok: false, why: 'the browser could not be downloaded' };
2616
- const report = await edge.run(USER_DIR, flows, {
2641
+ const report = await runner.run(USER_DIR, flows, {
2617
2642
  say: say,
2618
2643
  onControl: (c) => control.onStop(() => c.stop()),
2619
2644
  confirmed: confirmed || [],
2620
2645
  });
2621
- edge.save(USER_DIR, report);
2622
- for (const line of edge.summaryLines(report)) say(line);
2646
+ runner.save(USER_DIR, report);
2647
+ for (const line of runner.summaryLines(report)) say(line);
2623
2648
  const problems = report.findings.filter((f) => f.status === 'confirmed').length;
2624
2649
  const tested = report.flows.some((f) => f.state === 'tested');
2625
2650
  if (!tested) return { ok: false, why: 'nothing was tested - see what it printed' };
@@ -2631,23 +2656,23 @@ async function probeFromDashboard(id, say, control, confirmed) {
2631
2656
  };
2632
2657
  }
2633
2658
 
2634
- /** One input finding, tried again: its flow as recorded, and the one value that found it. */
2635
- async function edgeRecheck(id, say, control, confirmed) {
2659
+ /** One input or reliability finding, tried again: only what found it, never the whole check. */
2660
+ async function probeRecheck(id, say, control, confirmed) {
2636
2661
  const dash = require(path.join(PACKAGE_DIR, 'kryptheon-dashboard.js'));
2637
- const edge = require(path.join(PACKAGE_DIR, 'kryptheon-edge.js'));
2638
- const target = dash.buildState(USER_DIR).findings.find((f) => f.id === id && f.check === 'edge');
2662
+ const target = dash.buildState(USER_DIR).findings.find((f) => f.id === id && PROBE_CHECKS[f.check]);
2639
2663
  if (!target) return null;
2664
+ const runner = probeModule(target.check);
2640
2665
  const { flows } = probeFlows();
2641
2666
  if (!ensureBrowser()) return { target: target, verdict: 'could not confirm', why: 'the browser could not be downloaded', newProblems: [] };
2642
- const r = await edge.recheck(USER_DIR, flows, target, {
2667
+ const r = await runner.recheck(USER_DIR, flows, target, {
2643
2668
  say: say,
2644
2669
  onControl: (c) => control && control.onStop(() => c.stop()),
2645
2670
  confirmed: confirmed || [],
2646
2671
  });
2647
2672
  const attempt = {
2648
- id: id, at: new Date().toISOString(), check: 'edge', where: target.where, headline: target.headline,
2673
+ id: id, at: new Date().toISOString(), check: target.check, where: target.where, headline: target.headline,
2649
2674
  verdict: r.verdict, why: r.why || '',
2650
- newProblems: (r.newProblems || []).map((f) => ({ check: 'edge', severity: f.severity, status: f.status, where: f.where, headline: f.headline })),
2675
+ newProblems: (r.newProblems || []).map((f) => ({ check: target.check, severity: f.severity, status: f.status, where: f.where, headline: f.headline })),
2651
2676
  };
2652
2677
  try {
2653
2678
  fs.appendFileSync(store.pathsFor(USER_DIR).fixes, JSON.stringify(attempt) + '\n', 'utf8');
@@ -2657,9 +2682,9 @@ async function edgeRecheck(id, say, control, confirmed) {
2657
2682
  return Object.assign({ target: target, testData: r.testData || [] }, attempt);
2658
2683
  }
2659
2684
 
2660
- async function edgeRecheckFromDashboard(id, say, control, confirmed) {
2661
- const r = await edgeRecheck(id, say, control, confirmed);
2662
- if (!r) return { ok: false, why: 'no open input finding with that id' };
2685
+ async function probeRecheckFromDashboard(id, say, control, confirmed) {
2686
+ const r = await probeRecheck(id, say, control, confirmed);
2687
+ if (!r) return { ok: false, why: 'no open finding with that id that can be re-checked this way' };
2663
2688
  say('VERDICT ' + VERDICT_WORDS[r.verdict] + (r.why ? ' - ' + r.why : ''));
2664
2689
  return Object.assign({ ok: true, summary: r.verdict }, r, { target: undefined });
2665
2690
  }
@@ -3022,7 +3047,7 @@ async function dashboardCommand() {
3022
3047
  run: (id, say) => runFromDashboard(id, opened, say),
3023
3048
  recheck: (id, say) => recheckFromDashboard(id, opened, say),
3024
3049
  probe: (id, say, control, confirmed) => probeFromDashboard(id, say, control, confirmed),
3025
- probeRecheck: (id, say, control, confirmed) => edgeRecheckFromDashboard(id, say, control, confirmed),
3050
+ probeRecheck: (id, say, control, confirmed) => probeRecheckFromDashboard(id, say, control, confirmed),
3026
3051
  record: (address, say, options) => recordFromDashboard(address, say, options),
3027
3052
  db: (action, connection, say) => databaseFromDashboard(action, connection, say),
3028
3053
  words: (connection) => nightWords(connection),
@@ -3349,7 +3374,8 @@ async function main() {
3349
3374
  case 'recheck':
3350
3375
  return finishWith(await recheckCommand(rest.find((a) => !/^--/.test(a)), { yes: rest.includes('--yes') }));
3351
3376
  case 'edge':
3352
- return finishWith(await edgeCommand(rest));
3377
+ case 'reliability':
3378
+ return finishWith(await probeCommand(command, rest));
3353
3379
  case '-v':
3354
3380
  case '--version':
3355
3381
  case 'version':
@@ -302,8 +302,8 @@
302
302
  });
303
303
  }
304
304
 
305
- var CHECK_NAME = { regression: 'Recorded flows', frontend: 'Frontend code', data: 'Database', edge: 'Input and edge cases' };
306
- var CHECK_TONE = { regression: 'sun', frontend: 'amber', data: 'dusk', edge: 'ember' };
305
+ var CHECK_NAME = { regression: 'Recorded flows', frontend: 'Frontend code', data: 'Database', edge: 'Input and edge cases', reliability: 'Reliability' };
306
+ var CHECK_TONE = { regression: 'sun', frontend: 'amber', data: 'dusk', edge: 'ember', reliability: 'ember' };
307
307
 
308
308
  function counts() {
309
309
  var confirmed = state.findings.filter(function (f) { return f.status === 'confirmed'; }).length;
@@ -871,7 +871,7 @@
871
871
  var askingProbe = el('div', 'controls f-quick');
872
872
  askingProbe.appendChild(probeConsentBox(probeAsking.check, 'Yes, try it again', function (body) {
873
873
  runJob('/api/recheck', Object.assign({ id: f.id }, body)).then(function (done) { heard(f, (done && done.result) || {}); }, function () {});
874
- }, 'Re-check: replays this flow as recorded, then once more with the value that found this. Two runs.'));
874
+ }, RECHECK_WORDS[f.check === 'reliability' ? f.kind : 'edge']));
875
875
  card.appendChild(askingProbe);
876
876
  } else if (pending && pending.step === 'finding' && pending.finding && pending.finding.id === f.id) {
877
877
  var asking = el('div', 'controls f-quick');
@@ -949,7 +949,13 @@
949
949
  }
950
950
 
951
951
  var STATE_TONE = { 'problems found': 'ember', 'things to check': 'amber', 'last run held': 'leaf', 'nothing found': 'leaf', 'stopped part way': 'amber', 'nothing was tested': 'amber' };
952
- var PROBE_LABEL = { edge: 'Try inputs - adds test data' };
952
+ var PROBE_LABEL = { edge: 'Try inputs - adds test data', reliability: 'Replay 3x - adds test data' };
953
+ // What one re-check press does, said before the yes.
954
+ var RECHECK_WORDS = {
955
+ edge: 'Re-check: replays this flow as recorded, then once more with the value that found this. Two runs.',
956
+ flaky: 'Re-check: replays this flow five times. It is fixed only if all five work.',
957
+ double: 'Re-check: replays this flow once, then once more with this step double-clicked. Two runs.',
958
+ };
953
959
 
954
960
  // Which probe check is asking for a yes right now, and for which finding
955
961
  // when it is a re-check. Nothing that adds data to the app runs without it.
@@ -1120,7 +1126,7 @@
1120
1126
  if (f.check === 'data') { prepare('recheck', 'finding', f); return; }
1121
1127
  // An input finding is re-checked by typing its value again, which adds
1122
1128
  // test data to the app: the same yes as the check's own button, first.
1123
- if (f.check === 'edge') { probeAsking = { check: 'edge', finding: f }; openRow = null; renderFindings(); return; }
1129
+ if (f.check === 'edge' || f.check === 'reliability') { probeAsking = { check: f.check, finding: f }; openRow = null; renderFindings(); return; }
1124
1130
  runJob('/api/recheck', { id: f.id }).then(function (done) { heard(f, (done && done.result) || {}); }, function () {});
1125
1131
  }
1126
1132
  function heard(f, r) {
@@ -1281,7 +1287,15 @@
1281
1287
  function load() {
1282
1288
  return fetch('/api/state', { cache: 'no-store' })
1283
1289
  .then(function (res) { if (!res.ok) throw new Error('the dashboard answered ' + res.status); return res.json(); })
1284
- .then(render)
1290
+ .then(function (next) {
1291
+ // Drawn again only when something changed: a redraw every few
1292
+ // seconds replaced every button under the pointer, and a press that
1293
+ // landed on two drawings of one button was lost.
1294
+ var text = JSON.stringify(next);
1295
+ if (text === load.last) return;
1296
+ load.last = text;
1297
+ render(next);
1298
+ })
1285
1299
  .catch(function (err) { byId('subline').textContent = 'Could not load: ' + err.message; });
1286
1300
  }
1287
1301
 
@@ -212,7 +212,7 @@
212
212
  .detail-box { margin: 0 8px 22px 20px; padding: 18px 0 0; border-top: 1px dashed var(--line); }
213
213
  .d-step { margin-top: 18px; font-size: 10px; color: var(--orange-deep); }
214
214
  .d-step:first-child { margin-top: 0; }
215
- .d-text { margin-top: 6px; font-size: 14px; line-height: 1.6; color: var(--muted); overflow-wrap: anywhere; }
215
+ .d-text { margin-top: 6px; font-size: 14px; line-height: 1.6; color: var(--muted); overflow-wrap: anywhere; white-space: pre-line; }
216
216
  .detail-box pre { max-height: 360px; margin: 10px 0 0; padding: 18px; overflow: auto; font: 12px/1.7 var(--mono); white-space: pre-wrap; overflow-wrap: anywhere; color: var(--on-dark); background: var(--ink); }
217
217
  .d-actions { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 14px; }
218
218
  .attempts { display: grid; gap: 8px; margin-top: 14px; font-size: 13px; color: var(--muted); }
@@ -48,7 +48,8 @@ const CHECKS = [
48
48
  { id: 'performance', label: 'Performance', available: false },
49
49
  { id: 'edge', label: 'Input and edge cases', available: true, asks: true,
50
50
  what: 'Replays your recorded forms with one field at a time given an unusual value - empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist - and reports only what it can prove: a page error, a 5xx, a bad value saved, a tag shown as HTML. It adds test data to your app, so it runs only when you press its button.' },
51
- { id: 'reliability', label: 'Reliability', available: false },
51
+ { id: 'reliability', label: 'Reliability', available: true, asks: true,
52
+ what: 'Replays each recorded flow three times and reports one that sometimes works and sometimes fails, with the failed run as proof; and clicks each step that saves twice, quickly, reporting when that saves twice, with both answers as proof. It adds test data to your app, so it runs only when you press its button.' },
52
53
  { id: 'api', label: 'API', available: false },
53
54
  { id: 'backend', label: 'Backend', available: false },
54
55
  { id: 'synthetic', label: 'Synthetic test users', available: false },
@@ -194,7 +195,7 @@ function countRecordings(root) {
194
195
  * and the words said about them (kryptheon-probe.js). Read from the
195
196
  * recordings every time, so what the page shows is what would really run.
196
197
  */
197
- function probeAsk(root, what, env) {
198
+ function probeAsk(root, what, env, options) {
198
199
  const probe = require('./kryptheon-probe.js');
199
200
  const sources = [];
200
201
  try {
@@ -203,14 +204,19 @@ function probeAsk(root, what, env) {
203
204
  } catch (err) {
204
205
  /* no recordings: nothing to ask about */
205
206
  }
206
- // The pages, and where the app was last seen saving (kryptheon-edge.js).
207
- const addresses = require('./kryptheon-edge.js').knownAddresses(root, sources, env);
208
- return { addresses: addresses, live: probe.liveLooking(addresses), lines: probe.consentLines(what, addresses) };
207
+ // The pages, and where the app was last seen saving (kryptheon-probe.js).
208
+ const addresses = probe.knownAddresses(root, sources, env);
209
+ return { addresses: addresses, live: probe.liveLooking(addresses), lines: probe.consentLines(what, addresses, options) };
209
210
  }
210
211
 
211
212
  const PROBE_WHAT = {
212
213
  edge: 'Input and edge cases: replays your recorded forms with one field at a time given an unusual value (empty, 5000 characters, an HTML tag, emoji, a negative number, a date that does not exist). At most 40 runs.',
214
+ reliability: 'Reliability: replays each recorded flow three times, and clicks each step that saves (Save, Add, Create...) twice, quickly. At most 40 runs.',
213
215
  };
216
+ // The reliability check types nothing of its own, so nothing it saves can
217
+ // carry the kryptheon-test marker - and its words do not promise one.
218
+ const PROBE_WORDS = { reliability: { marker: false } };
219
+ const PROBE_CHECKS = ['edge', 'reliability'];
214
220
 
215
221
  /**
216
222
  * The order someone new should go in, one step at a time. A step is "done"
@@ -448,14 +454,17 @@ function buildState(root, env) {
448
454
  }
449
455
  }
450
456
 
451
- // Input and edge cases: the last press of its button. Each finding carries
452
- // its proof, and what kind of value and field it was about, so a re-check
453
- // can try exactly that again.
454
- const edgeLast = readJson(path.join(paths.dir, 'edge-last.json'));
455
- if (edgeLast && config.enabled.edge) {
456
- for (const f of edgeLast.findings || []) {
457
+ // Input and edge cases, and reliability: the last press of each one's
458
+ // button. Each finding carries its proof, and what it was about - the
459
+ // field and value, the flow, the step - so a re-check can try exactly that
460
+ // again.
461
+ const probeLast = {};
462
+ for (const id of PROBE_CHECKS) {
463
+ const last = (probeLast[id] = readJson(path.join(paths.dir, require('./kryptheon-probe.js').REPORT_FILES[id])));
464
+ if (!last || !config.enabled[id]) continue;
465
+ for (const f of last.findings || []) {
457
466
  findings.push({
458
- check: 'edge',
467
+ check: id,
459
468
  severity: f.severity,
460
469
  status: f.status,
461
470
  evidence: f.status === 'confirmed' ? 'runtime confirmed' : 'seen once',
@@ -464,7 +473,7 @@ function buildState(root, env) {
464
473
  headline: f.headline,
465
474
  detail: (f.proof || []).join('\n'),
466
475
  fixPrompt: f.fixPrompt || '',
467
- when: edgeLast.endedAt || edgeLast.startedAt,
476
+ when: last.endedAt || last.startedAt,
468
477
  kind: f.kind,
469
478
  type: f.type,
470
479
  });
@@ -534,12 +543,12 @@ function buildState(root, env) {
534
543
  f.id = changes.findingId(f);
535
544
  f.attempts = attempts.filter((a) => a.id === f.id).map((a) => ({ at: a.at, verdict: a.verdict, why: a.why || '', newProblems: (a.newProblems || []).length }));
536
545
  forAll.push({ id: f.id, severity: f.severity, headline: f.headline, where: f.where, prompt: f.fixPrompt || '' });
537
- if (f.fixPrompt && f.check === 'edge') {
538
- // Re-checking types the value into the app once more, so it adds a
539
- // little test data: --yes says yes for an app on this computer, and is
540
- // refused for any other address - there the person must answer.
546
+ if (f.fixPrompt && PROBE_CHECKS.includes(f.check)) {
547
+ // Re-checking runs the flow in the app again, so it adds a little test
548
+ // data: --yes says yes for an app on this computer, and is refused for
549
+ // any other address - there the person must answer.
541
550
  f.fixPrompt += '\n\nWhen you have fixed it, prove it: run npx kryptheon recheck ' + f.id + ' --yes' +
542
- ' (it types the value into the app once more, adding a little test data; --yes works only for an app running on this computer). ' +
551
+ ' (it runs the flow in the app again, adding a little test data; --yes works only for an app running on this computer). ' +
543
552
  'It is fixed only when that says FIXED.';
544
553
  } else if (f.fixPrompt) {
545
554
  f.fixPrompt += '\n\nWhen you have fixed it, prove it: run npx kryptheon recheck ' + f.id +
@@ -557,12 +566,13 @@ function buildState(root, env) {
557
566
  if (!c.available) state = 'coming soon';
558
567
  else if (!config.enabled[c.id]) state = 'switched off';
559
568
  else if (c.id === 'regression') state = !recordings ? 'no recordings yet' : !lastRun ? 'never run' : lastRun.failed ? 'problems found' : 'last run held';
560
- else if (c.id === 'edge') {
561
- state = !edgeLast ? 'never run'
562
- : findings.some((f) => f.check === 'edge' && f.status === 'confirmed') ? 'problems found'
563
- : findings.some((f) => f.check === 'edge') ? 'things to check'
564
- : edgeLast.stopped ? 'stopped part way'
565
- : !(edgeLast.flows || []).some((x) => x.state === 'tested') ? 'nothing was tested'
569
+ else if (PROBE_CHECKS.includes(c.id)) {
570
+ const last = probeLast[c.id];
571
+ state = !last ? 'never run'
572
+ : findings.some((f) => f.check === c.id && f.status === 'confirmed') ? 'problems found'
573
+ : findings.some((f) => f.check === c.id) ? 'things to check'
574
+ : last.stopped ? 'stopped part way'
575
+ : !(last.flows || []).some((x) => x.state === 'tested') ? 'nothing was tested'
566
576
  : 'last run held';
567
577
  }
568
578
  else if (c.id === 'frontend') state = !code ? 'never run' : (code.findings || []).length ? 'things to check' : 'nothing found';
@@ -576,11 +586,12 @@ function buildState(root, env) {
576
586
  nightly: c.id === 'data' ? nightlyLine(nightly) : null,
577
587
  // A check that adds data to the app: never in "Run all", never on its
578
588
  // own - only its own button, after the page has shown these words.
579
- asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env) : null,
580
- lastRun: c.id === 'edge' && edgeLast ? {
581
- at: edgeLast.endedAt, stopped: !!edgeLast.stopped, runsDone: edgeLast.runsDone, runsPlanned: edgeLast.runsPlanned,
582
- held: (edgeLast.held || []).length, notTested: (edgeLast.notTested || []).length + (edgeLast.flows || []).filter((x) => x.state !== 'tested').length,
583
- testData: (edgeLast.testData || []).length,
589
+ asks: c.asks && config.enabled[c.id] && recordings > 0 ? probeAsk(root, PROBE_WHAT[c.id], env, PROBE_WORDS[c.id]) : null,
590
+ lastRun: probeLast[c.id] ? {
591
+ at: probeLast[c.id].endedAt, stopped: !!probeLast[c.id].stopped, runsDone: probeLast[c.id].runsDone, runsPlanned: probeLast[c.id].runsPlanned,
592
+ held: (probeLast[c.id].held || []).length,
593
+ notTested: (probeLast[c.id].notTested || []).length + (probeLast[c.id].flows || []).filter((x) => x.state !== 'tested').length,
594
+ testData: (probeLast[c.id].testData || []).length,
584
595
  } : null,
585
596
  // Runnable from the page: what needs nothing the page cannot supply. The
586
597
  // database check needs a connection string, which stays in the terminal.
@@ -653,8 +664,8 @@ function buildState(root, env) {
653
664
 
654
665
  // What each run is called on the page, and the command that does the same in
655
666
  // a terminal - shown under every button, for when the page cannot be used.
656
- const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms' };
657
- const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge' };
667
+ const RUN_TITLE = { frontend: 'Reading your code', regression: 'Replaying your recordings', edge: 'Trying unusual inputs in your forms', reliability: 'Replaying your flows three times, and double-clicking' };
668
+ const RUN_COMMAND = { frontend: 'npx kryptheon code', regression: 'npx kryptheon check', edge: 'npx kryptheon edge', reliability: 'npx kryptheon reliability' };
658
669
  const DB_ACTIONS = {
659
670
  scan: { title: 'Checking your database', command: 'npx kryptheon-night' },
660
671
  install: { title: 'Setting up the nightly check', command: 'npx kryptheon-night install' },
@@ -991,8 +1002,8 @@ function createServer(root, options) {
991
1002
  if (!/^[0-9a-f]{8}$/.test(id)) return json(res, 400, { ok: false, why: 'not a finding id' });
992
1003
  const open = buildState(root, env).findings.find((f) => f.id === id);
993
1004
  if (!open) return json(res, 404, { ok: false, why: 'no open finding with that id' });
994
- if (open.check === 'edge') {
995
- const refused = probeRefusal('edge');
1005
+ if (PROBE_CHECKS.includes(open.check)) {
1006
+ const refused = probeRefusal(open.check);
996
1007
  if (refused) return json(res, 400, { ok: false, why: refused });
997
1008
  if (typeof opts.probeRecheck !== 'function') return json(res, 400, { ok: false, why: 'this finding cannot be re-checked from here' });
998
1009
  return begin('recheck', 'Re-checking: ' + open.headline, 'npx kryptheon recheck ' + id, (say, control) => opts.probeRecheck(id, say, control, confirmed));
package/kryptheon-edge.js CHANGED
@@ -271,11 +271,32 @@ function diffLine(s) {
271
271
  return (s.key ? s.key + ': ' + show(s.was) + ' in your recorded run, ' + show(s.now) + ' in this one' : 'the request body holds the typed value');
272
272
  }
273
273
 
274
+ /**
275
+ * What the server said it stored for this field, from its answer - the row it
276
+ * saved, when it sends one back. The request shows what was sent; this shows
277
+ * what was kept, which is the stronger proof.
278
+ */
279
+ function stored(s) {
280
+ if (!s.key || s.write.answer == null) return null;
281
+ const last = s.key.split('.').pop();
282
+ const answer = flatten(s.write.answer);
283
+ const key = Object.prototype.hasOwnProperty.call(answer, s.key) ? s.key
284
+ : Object.keys(answer).find((k) => k === last || k.endsWith('.' + last));
285
+ return key ? { key: key, value: answer[key] } : null;
286
+ }
287
+
288
+ function storedLine(s) {
289
+ const kept = stored(s);
290
+ return kept ? 'The server\'s answer: ' + kept.key + ' is ' + show(kept.value) + ' - that is what it stored.' : null;
291
+ }
292
+
274
293
  function proofLines(saved) {
275
294
  const lines = [];
276
295
  for (const s of saved.slice(0, 3)) {
277
296
  lines.push('Request: ' + probe.writeWords(s.write));
278
297
  lines.push('Difference from your recorded run: ' + diffLine(s));
298
+ const stored = storedLine(s);
299
+ if (stored) lines.push(stored);
279
300
  }
280
301
  return lines;
281
302
  }
@@ -327,7 +348,10 @@ function judgeVariant(clean, variant, plan) {
327
348
 
328
349
  const saved = savedValues(clean, variant, field, plan.value);
329
350
  if (saved.length) {
330
- const as = savedAs(kind, plan.value, saved[0].now);
351
+ // What the server says it kept decides, when it says: a -5 it turned
352
+ // into 0 was not saved as -5. Without an answer, the request decides.
353
+ const kept = stored(saved[0]);
354
+ const as = savedAs(kind, plan.value, kept ? kept.value : saved[0].now);
331
355
  const info = field.info || {};
332
356
  const problem =
333
357
  (kind === 'empty' && info.required && as !== 'changed') ||
@@ -681,18 +705,9 @@ function carryOver(previous, report, root) {
681
705
  return report;
682
706
  }
683
707
 
684
- /**
685
- * Every address a press would reach: the pages the recordings open, and the
686
- * places the last plain replay saw the app save to. This is what the person
687
- * is asked about, so a database somewhere else is named before it is written.
688
- */
689
- function knownAddresses(root, sources, env) {
690
- const out = [];
691
- for (const source of sources) for (const a of probe.addressesOf(source)) if (!out.includes(a)) out.push(a);
692
- const last = load(root, env);
693
- for (const h of (last && last.writeHosts) || []) if (!out.includes(h)) out.push(h);
694
- return out;
695
- }
708
+ // Kept here for callers of this module; the list itself is shared by every
709
+ // probe check (kryptheon-probe.js).
710
+ const knownAddresses = probe.knownAddresses;
696
711
 
697
712
  /** Keeps the report where the dashboard reads it - with what it did not try still said. */
698
713
  function save(root, report, env, options) {
@@ -117,7 +117,13 @@ const test = base.test.extend({
117
117
  const events = [];
118
118
  const fields = {};
119
119
  let seq = 0;
120
- const add = (e) => events.push(Object.assign({ seq: ++seq }, e));
120
+ // Returns the event as kept, so what arrives later (a write's answer) is
121
+ // added to the very object that is saved - not to a copy of it.
122
+ const add = (e) => {
123
+ const kept = Object.assign({ seq: ++seq }, e);
124
+ events.push(kept);
125
+ return kept;
126
+ };
121
127
 
122
128
  // A gap before every run, and a longer one after the app said "too many
123
129
  // requests": the app is being asked many times in a row, and a rate limit
@@ -158,13 +164,42 @@ const test = base.test.extend({
158
164
  // A write that carried a secret is the sign-in, not data the app
159
165
  // keeps - it is never listed as test data left behind.
160
166
  const signIn = JSON.stringify(kept).indexOf('[hidden]') !== -1;
161
- add(Object.assign({ kind: 'write', method: method, host: at.host, origin: at.origin, path: at.path, status: status, signIn: signIn }, kept));
167
+ const event = Object.assign({ kind: 'write', method: method, host: at.host, origin: at.origin, path: at.path, status: status, signIn: signIn }, kept);
168
+ const placed = add(event);
169
+ // What the server answered - often the record it saved, with its id.
170
+ // That is the proof that two saves made two records, and of what was
171
+ // really stored. Never for the sign-in, whose answer is a session.
172
+ // Read after the event is placed, so its order among the marks is
173
+ // the order the answer arrived in.
174
+ if (!signIn && /json/i.test((res.headers() || {})['content-type'] || '')) {
175
+ const reading = res.text().then((text) => {
176
+ if (text && text.length <= MAX_BODY * 5) {
177
+ try {
178
+ placed.answer = JSON.parse(hideValues(JSON.stringify(hideKeys(JSON.parse(text))), secrets));
179
+ } catch (err) {
180
+ /* not JSON after all: no answer kept */
181
+ }
182
+ }
183
+ }, () => {}).then(() => answers.delete(reading));
184
+ answers.add(reading);
185
+ }
162
186
  }
163
187
  });
188
+ // Answers still being read. Counted before a mark and before the end.
189
+ const answers = new Set();
190
+ const answersRead = async () => {
191
+ for (let waited = 0; answers.size && waited < 5000; waited += 100) await sleep(100);
192
+ };
164
193
 
165
194
  // Called from inside a probe copy, around the steps that matter.
166
195
  page.__kxMark = async (name) => {
167
- // Let what the last step started arrive before it is counted.
196
+ // Let what the last step started arrive before it is counted: a moment
197
+ // for a second click's request to leave, then every write still on its
198
+ // way. A load state alone is not enough - once a page has gone quiet it
199
+ // says so at once, even while a new save is in flight.
200
+ await sleep(300);
201
+ for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
202
+ await answersRead();
168
203
  try {
169
204
  await page.waitForLoadState('networkidle', { timeout: 3000 });
170
205
  } catch (err) {
@@ -211,6 +246,7 @@ const test = base.test.extend({
211
246
  await use(page);
212
247
 
213
248
  for (let waited = 0; inFlight.size && waited < 10000; waited += 100) await sleep(100);
249
+ await answersRead();
214
250
  try {
215
251
  await page.waitForLoadState('networkidle', { timeout: 3000 });
216
252
  } catch (err) {