kryptheon 0.1.49 → 0.1.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -51,16 +51,23 @@ const ROUNDS = 3;
51
51
  // The size a phone recording is made at (bin/kryptheon.js PHONE_SIZE).
52
52
  const PHONE = { width: 390, height: 844 };
53
53
  const CONFIRM_RUNS = 5;
54
+ // Flows replayed at phone size in one press.
55
+ const PHONE_FLOWS = 3;
54
56
  // A re-check of a flaky flow needs more than three runs to say anything.
55
57
  const RECHECK_ROUNDS = 5;
56
58
 
57
59
  const CLICK = /^(\s*)(await\s+)?(page\s*\..+?)\.click\(\s*\)\s*;?\s*$/;
58
60
 
59
- const ok = (w) => w.status >= 200 && w.status < 300;
61
+ // Taken by the app: a 2xx, or a form's redirect after it saved (probe.accepted).
62
+ const ok = (w) => probe.accepted(w);
60
63
  const eventsOf = (r) => (r && r.events) || [];
61
64
  const writesOf = (r) => eventsOf(r).filter((e) => e.kind === 'write');
62
65
  const leftBehind = (w) => ok(w) && w.method !== 'DELETE' && !w.signIn;
63
- const routeOf = (w) => w.method + ' ' + (w.origin || w.host || '') + w.path;
66
+ // Where a write went, a record's id left out: a step that saves to the third
67
+ // course in a list saves to another course once runs have added courses
68
+ // (blind round 21), and is still the same save.
69
+ const routeOf = (w) => w.method + ' ' + (w.origin || w.host || '') +
70
+ String(w.path || '').replace(/\/(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24})(?=\/|$)/gi, '/:id');
64
71
  const limited = (r) => eventsOf(r).some((e) => e.kind === 'limited');
65
72
 
66
73
  /**
@@ -72,8 +79,7 @@ const limited = (r) => eventsOf(r).some((e) => e.kind === 'limited');
72
79
  function saveSteps(source) {
73
80
  const lines = String(source || '').split('\n');
74
81
  const out = [];
75
- for (const a of replay.readActions(source)) {
76
- if (a.method !== 'click' || !a.name) continue;
82
+ for (const a of clickSteps(source)) {
77
83
  if (!replay.CREATES.test(a.name.trim()) || replay.SIGNUP.test(a.name)) continue;
78
84
  const m = CLICK.exec(lines[a.line - 1].replace(/\r$/, ''));
79
85
  if (!m) continue;
@@ -82,6 +88,37 @@ function saveSteps(source) {
82
88
  return out;
83
89
  }
84
90
 
91
+ // A click into a field: it saves nothing, whatever the field is called
92
+ // (blind round 21: a click on "Add a quick class note" was taken for a save).
93
+ const FIELD_CLICK = /getBy(Placeholder|Label)\(|getByRole\(\s*['"](textbox|searchbox|combobox|spinbutton|checkbox|radio|option|listbox)['"]/;
94
+ // A button with no name of its own: getByRole('button') and nothing more.
95
+ const NAMELESS_BUTTON = /getByRole\(\s*['"]button['"]\s*\)\s*\.click\(\s*\)/;
96
+
97
+ /**
98
+ * Every click a person made on something that can save, with a name to call
99
+ * it by. A click into a field is left out. A button with no name, clicked
100
+ * right after typing into a field (a card's own send button), is named for
101
+ * that field: 'Save "Add a quick class note"' (blind round 21: the calendar's
102
+ * note button had no name, so the note it saved was never checked).
103
+ */
104
+ function clickSteps(source) {
105
+ const actions = replay.readActions(source);
106
+ const out = [];
107
+ actions.forEach((a, i) => {
108
+ if (a.method !== 'click' || FIELD_CLICK.test(a.text)) return;
109
+ if (a.name) {
110
+ out.push(a);
111
+ return;
112
+ }
113
+ if (!NAMELESS_BUTTON.test(a.text)) return;
114
+ const before = actions.slice(0, i).reverse().find((b) => !(b.method === 'click' && FIELD_CLICK.test(b.text)));
115
+ if (before && before.method === 'fill' && before.name && !replay.isSecretField(before.name)) {
116
+ out.push(Object.assign({}, a, { name: 'Save "' + before.name + '"' }));
117
+ }
118
+ });
119
+ return out;
120
+ }
121
+
85
122
  /**
86
123
  * What was typed before a saving step, to look for on the page after it:
87
124
  * each literal value filled in since the last step that saved, three
@@ -118,9 +155,189 @@ function plainSource(source, steps) {
118
155
  before[s.line] = (typed.length ? [seen('before')] : []).concat([s.indent + "await page.__kxMark('before-" + s.line + "');"]);
119
156
  after[s.line] = [s.indent + "await page.__kxMark('after-" + s.line + "');"].concat(typed.length ? [seen('after'), seen('fresh', true)] : []);
120
157
  }
158
+ // Each step that keeps what was typed: what each field held before it was
159
+ // typed into, and the form opened again after the step, read back.
160
+ const push = (map, line, text) => (map[line] = [].concat(map[line] || [], [text]));
161
+ for (const k of keepSteps(source)) {
162
+ const fields = keptFields(source, k);
163
+ if (!fields.length) continue;
164
+ for (const f of fields) push(before, f.firstLine, f.indent + 'await page.__kxWas(' + f.line + ', ' + f.locator + ');');
165
+ if (!steps.some((s) => s.line === k.line)) {
166
+ push(before, k.line, k.indent + "await page.__kxMark('before-" + k.line + "');");
167
+ push(after, k.line, k.indent + "await page.__kxMark('after-" + k.line + "');");
168
+ }
169
+ push(after, k.line, k.indent + 'await page.__kxReadBack(' + JSON.stringify('kept-' + k.line) + ', [' +
170
+ fields.map((f) => '{ line: ' + f.line + ', get: (p) => ' + f.locator.replace(/^page\s*\./, 'p.') + ' }').join(', ') + ']);');
171
+ }
121
172
  return probe.probeSource(source, { before: before, after: after });
122
173
  }
123
174
 
175
+ // A click that keeps what was typed: Save, Update, Apply...
176
+ const KEEPS = /^[^\p{L}\p{N}]*(save|update|apply|submit|done|confirm)\b/iu;
177
+
178
+ /** The steps after which what was typed is read back: a plain click on a button that saves or updates. */
179
+ function keepSteps(source) {
180
+ const lines = String(source || '').split('\n');
181
+ const out = [];
182
+ for (const a of clickSteps(source)) {
183
+ if (!(KEEPS.test(a.name.trim()) || replay.CREATES.test(a.name.trim())) || replay.SIGNUP.test(a.name)) continue;
184
+ const m = CLICK.exec(lines[a.line - 1].replace(/\r$/, ''));
185
+ if (!m) continue;
186
+ out.push({ line: a.line, indent: m[1], locator: m[3], name: a.name.trim() });
187
+ }
188
+ return out;
189
+ }
190
+
191
+ /**
192
+ * The fields typed into before a step that keeps them: each literal fill
193
+ * since the last such step, empty ones too (clearing a field is an edit),
194
+ * never the sign-in or a secret.
195
+ */
196
+ function keptFields(source, step) {
197
+ const steps = keepSteps(source);
198
+ const since = Math.max(0, ...steps.filter((s) => s.line < step.line).map((s) => s.line));
199
+ const lines = String(source || '').split('\n');
200
+ let signInEnd = 0;
201
+ lines.forEach((line, i) => {
202
+ if (/\.fill\(/.test(line) && (replay.isSecretField(replay.nameInLine(line)) || /process\.env\./.test(line))) signInEnd = i + 1;
203
+ });
204
+ // Each field once: its last fill is what was saved, and what it held is
205
+ // read before its first - a typo fixed by typing again is still one edit.
206
+ const byLocator = new Map();
207
+ for (const f of edge.fillSteps(source)) {
208
+ if (f.line <= since || f.line >= step.line || f.line <= signInEnd || replay.isSecretField(f.name) || !/^page\s*\./.test(f.locator)) continue;
209
+ const first = byLocator.has(f.locator) ? byLocator.get(f.locator).firstLine : f.line;
210
+ byLocator.set(f.locator, Object.assign({}, f, { firstLine: first }));
211
+ }
212
+ return Array.from(byLocator.values());
213
+ }
214
+
215
+ // The address of a form for something new: /pets/new, /orders/create.
216
+ const NEW_FORM = /\/(new|create|add)(\/|$)/i;
217
+
218
+ /** Two field values that say the same: the same text, or the same number. */
219
+ function sameValue(a, b) {
220
+ const x = String(a == null ? '' : a).trim();
221
+ const y = String(b == null ? '' : b).trim();
222
+ if (x === y) return true;
223
+ return x !== '' && y !== '' && Number.isFinite(Number(x)) && Number.isFinite(Number(y)) && Number(x) === Number(y);
224
+ }
225
+
226
+ /**
227
+ * Whether what was typed before a step that keeps it was kept: the form
228
+ * opened again after the step, against what each field held before it was
229
+ * typed into. Only an edit is judged - a field that held something - and only
230
+ * in a run whose save the app took. Returns { outcome: 'lost' | 'changed' |
231
+ * 'held' | 'not tested', finding?, why }.
232
+ */
233
+ function judgeKept(flow, step, runs) {
234
+ // What a field held is only what the app saved when the recording did not
235
+ // type it there itself, earlier: a value typed, refused by the form and
236
+ // left in the box is not the app's. Found on the round 19 bench: an invite
237
+ // form kept a bad address the browser had refused, and the next save was
238
+ // judged as an edit of it.
239
+ const typedEarlier = (f) => edge.fillSteps(flow.source).filter((x) => x.locator === f.locator && x.line < f.firstLine).map((x) => x.value);
240
+ const fields = keptFields(flow.source, step).map((f) => Object.assign({}, f, { earlier: typedEarlier(f) }));
241
+ const counted = (runs || []).filter((r) => r && !limited(r));
242
+ const lost = [];
243
+ const changed = [];
244
+ let kept = 0;
245
+ let judged = 0;
246
+ let sentNothing = 0;
247
+ for (const r of counted) {
248
+ const back = eventsOf(r).filter((e) => e.kind === 'readback' && e.name === 'kept-' + step.line);
249
+ if (!back.length) continue;
250
+ const writes = windowWrites(r, step.line) || [];
251
+ const took = writes.filter((w) => ok(w) && !w.signIn && w.method !== 'DELETE');
252
+ // An edit: the field held something the recording did not type there,
253
+ // and was given something else - on a form that is not for something
254
+ // new. A new-record form may show defaults (blind round 20: "Add pet" at
255
+ // /pets/new showed a weight of 5, and was read as an edit of it).
256
+ const notEdit = (f, e) => !f || String(e.was).trim() === '' || sameValue(f.value, e.was) || f.earlier.some((v) => sameValue(v, e.was)) || NEW_FORM.test(String(e.at || ''));
257
+ let edits = 0;
258
+ for (const e of back) {
259
+ const f = fields.find((x) => x.line === e.line);
260
+ if (notEdit(f, e)) continue;
261
+ edits += 1;
262
+ }
263
+ if (!edits) continue;
264
+ if (!took.length) {
265
+ sentNothing += 1;
266
+ continue;
267
+ }
268
+ judged += 1;
269
+ for (const e of back) {
270
+ const f = fields.find((x) => x.line === e.line);
271
+ if (notEdit(f, e) || e.now === null) continue;
272
+ const item = { run: counted.indexOf(r) + 1, field: f, was: e.was, now: e.now, write: took[0] };
273
+ if (sameValue(e.now, f.value)) kept += 1;
274
+ else if (sameValue(e.now, e.was)) lost.push(item);
275
+ else changed.push(item);
276
+ }
277
+ }
278
+ if (!judged) {
279
+ // A form for something new is not an edit: nothing to say, not even
280
+ // "not tested" - every create flow would say it.
281
+ return sentNothing ? { outcome: 'not tested', why: 'the app did not take the save in any run, so what it kept was not read back' }
282
+ : { outcome: 'not an edit', why: 'no field it typed into held anything before, so it is not an edit - nothing to read back' };
283
+ }
284
+ const thing = (item) => '"' + (item.field.name || 'a field') + '"';
285
+ const typedWords = (item) => (String(item.field.value).trim() === '' ? 'cleared' : 'set to ' + JSON.stringify(item.field.value));
286
+ // One finding per field: a description that is not saved and a date that
287
+ // cannot be cleared are two things to fix.
288
+ const byField = (list) => {
289
+ const out = new Map();
290
+ for (const x of list) if (!out.has(x.field.line)) out.set(x.field.line, list.filter((y) => y.field.line === x.field.line));
291
+ return Array.from(out.values());
292
+ };
293
+ const findings = [];
294
+ for (const items of byField(lost)) {
295
+ const first = items[0];
296
+ const runsLost = new Set(items.map((x) => x.run)).size;
297
+ const sure = runsLost >= 2;
298
+ findings.push({
299
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'kept', type: 'lost', step: step.name, field: first.field.name || 'line ' + first.field.line,
300
+ severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
301
+ headline: 'After "' + step.name + '", ' + thing(first) + ' was ' + typedWords(first) + ', but the form opened again still shows ' + JSON.stringify(first.was) + ' - the change was not kept.',
302
+ proof: [
303
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + first.field.line + ', saved at line ' + step.line + ')',
304
+ 'Before typing, ' + thing(first) + ' held ' + JSON.stringify(first.was) + '; the recording ' + (String(first.field.value).trim() === '' ? 'cleared it' : 'typed ' + JSON.stringify(first.field.value)) + '.',
305
+ 'The save: ' + words(first.write) + ' - the app took it.',
306
+ 'The same form opened again in a new tab: ' + thing(first) + ' shows ' + JSON.stringify(first.now) + ', what it held before.',
307
+ 'Seen in ' + runsLost + ' of ' + judged + (judged === 1 ? ' run' : ' runs') + ' that saved.',
308
+ ],
309
+ });
310
+ }
311
+ for (const items of byField(changed)) {
312
+ const first = items[0];
313
+ findings.push({
314
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'kept', type: 'changed', step: step.name, field: first.field.name || 'line ' + first.field.line,
315
+ severity: 'LOW', status: 'verification required',
316
+ headline: 'After "' + step.name + '", ' + thing(first) + ' was ' + typedWords(first) + ', and the form opened again shows ' + JSON.stringify(first.now) + ' - check that this is meant.',
317
+ proof: [
318
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + first.field.line + ', saved at line ' + step.line + ')',
319
+ 'Before typing it held ' + JSON.stringify(first.was) + '. The save: ' + words(first.write) + '.',
320
+ 'The same form opened again in a new tab shows ' + JSON.stringify(first.now) + ' - neither what was typed nor what it held.',
321
+ ],
322
+ });
323
+ }
324
+ if (findings.length) return { outcome: lost.length ? 'lost' : 'changed', findings: findings };
325
+ return { outcome: 'held', why: 'the form opened again after it shows what was typed' };
326
+ }
327
+
328
+ /**
329
+ * What was typed for a step, counted on the page before it and on the same
330
+ * address opened again after it - the lines that count it, for a probe copy.
331
+ * A form that redirects names no record in its answer; the page is what says
332
+ * how many were kept.
333
+ */
334
+ function countLines(source, step) {
335
+ const typed = typedFor(source, saveSteps(source), step);
336
+ if (!typed.length) return { before: [], after: [] };
337
+ const seen = (name, fresh) => step.indent + 'await page.__kxSeen(' + JSON.stringify(name + '-' + step.line) + ', ' + JSON.stringify(typed) + (fresh ? ', true' : '') + ');';
338
+ return { before: [seen('before')], after: [seen('fresh', true)] };
339
+ }
340
+
124
341
  /** The same, with one step clicked twice. */
125
342
  function doubleSource(source, steps, step) {
126
343
  const edits = {};
@@ -129,8 +346,9 @@ function doubleSource(source, steps, step) {
129
346
  const before = {};
130
347
  const after = {};
131
348
  for (const s of marked) {
132
- before[s.line] = s.indent + "await page.__kxMark('before-" + s.line + "');";
133
- after[s.line] = s.indent + "await page.__kxMark('after-" + s.line + "');";
349
+ const counts = countLines(source, s);
350
+ before[s.line] = counts.before.concat([s.indent + "await page.__kxMark('before-" + s.line + "');"]);
351
+ after[s.line] = [s.indent + "await page.__kxMark('after-" + s.line + "');"].concat(counts.after);
134
352
  }
135
353
  return probe.probeSource(source, { edits: edits, before: before, after: after });
136
354
  }
@@ -142,11 +360,46 @@ function doubleSource(source, steps, step) {
142
360
  function twiceSource(source, step, carrier) {
143
361
  const before = {};
144
362
  const after = {};
145
- before[step.line] = [step.indent + 'await page.__kxTwice(' + JSON.stringify(carrier) + ');', step.indent + "await page.__kxMark('before-" + step.line + "');"];
146
- after[step.line] = step.indent + "await page.__kxMark('after-" + step.line + "');";
363
+ const counts = countLines(source, step);
364
+ before[step.line] = counts.before.concat([step.indent + 'await page.__kxTwice(' + JSON.stringify(carrier) + ');', step.indent + "await page.__kxMark('before-" + step.line + "');"]);
365
+ after[step.line] = [step.indent + "await page.__kxMark('after-" + step.line + "');"].concat(counts.after);
147
366
  return probe.probeSource(source, { before: before, after: after });
148
367
  }
149
368
 
369
+ /**
370
+ * What the page says a run kept at a step: for each value typed for it, how
371
+ * many more times it is on the page opened again after the step than before
372
+ * it. Null when it was not counted.
373
+ */
374
+ function addedOnPage(flow, step, result) {
375
+ const typed = typedFor(flow.source, saveSteps(flow.source), step);
376
+ const seen = (name) => (eventsOf(result).find((e) => e.kind === 'seen' && e.name === name + '-' + step.line) || {}).counts;
377
+ const before = seen('before');
378
+ const fresh = seen('fresh');
379
+ if (!typed.length || !before || !fresh) return null;
380
+ return typed.map((t, i) => ({ text: t, added: fresh[i] - before[i], before: before[i], fresh: fresh[i] }));
381
+ }
382
+
383
+ /**
384
+ * Two saves answered with a redirect name no record: whether the page shows
385
+ * more of what was typed than a single click adds. { more, text, once, now }
386
+ * or null when the page does not say - nothing was counted, or what a single
387
+ * click saved is not on it.
388
+ */
389
+ function pageSays(flow, step, single, other) {
390
+ const one = addedOnPage(flow, step, single);
391
+ const two = addedOnPage(flow, step, other);
392
+ if (!one || !two) return null;
393
+ const shown = one.map((x, i) => i).filter((i) => one[i].added >= 1);
394
+ if (!shown.length) return null;
395
+ const more = shown.find((i) => two[i].added > one[i].added);
396
+ const i = more === undefined ? shown[0] : more;
397
+ return { more: more !== undefined, text: one[i].text, once: one[i].added, now: two[i].added };
398
+ }
399
+
400
+ const redirectOnly = (answers) => answers.every((w) => w.answer == null && w.status >= 300 && w.status < 400);
401
+ const timesWord = (n) => (n === 1 ? 'once' : n + ' times');
402
+
150
403
  /**
151
404
  * The recording as it is, at phone size, with every button on the pages it
152
405
  * went to clicked once it has finished - or stopped. In an afterEach, so a
@@ -169,13 +422,29 @@ function phoneSource(source) {
169
422
  * write was blocked, so what it would have done is said as not seen.
170
423
  * Returns { outcome: 'found' | 'held' | 'not tested', finding?, why, unclicked }.
171
424
  */
172
- function judgePhone(flow, result) {
425
+ function judgePhone(flow, result, desktopErrors) {
173
426
  if (!result) return { outcome: 'not tested', why: 'this run did not finish' };
174
427
  if (limited(result)) return { outcome: 'not tested', why: 'your app said "too many requests" (429)' };
175
428
  const sweep = eventsOf(result).find((e) => e.kind === 'sweep');
176
429
  if (!sweep) return { outcome: 'not tested', why: 'the buttons could not be clicked' };
177
430
  if (!sweep.pages.length) return { outcome: 'not tested', why: 'none of the pages it went to could be opened again' + (sweep.failed ? ': ' + sweep.failed : '') };
178
431
  const atLoad = new Set((sweep.onLoad || []).map((e) => e.text));
432
+ // A page that throws as soon as it loads at phone size, and not at desktop
433
+ // size, is broken on a phone before anyone clicks. Found on blind round
434
+ // 20: a menu script reached for a button only the desktop layout has, at
435
+ // <= 600px, on every page - and was left out as "already logged at load".
436
+ // Only with the desktop runs to compare: without them, an error at load
437
+ // may be the page's everywhere.
438
+ const phoneOnly = desktopErrors ? (sweep.onLoad || []).filter((e) => e.type === 'page' && !desktopErrors.has(e.text)) : [];
439
+ const loadFinding = phoneOnly.length ? {
440
+ flow: flow.title, where: flow.file, kind: 'phone', type: 'phone-load', severity: 'MEDIUM', status: 'confirmed',
441
+ headline: 'At phone size, ' + phoneOnly[0].path + ' throws an error as soon as it loads - it does not at desktop size.',
442
+ proof: [
443
+ 'Flow: ' + flow.title + ' (' + flow.file + '), its pages opened at ' + PHONE.width + ' by ' + PHONE.height + '.',
444
+ 'The page threw when it loaded, before any click: ' + phoneOnly[0].text,
445
+ ].concat(Array.from(new Set(phoneOnly.slice(1).map((e) => e.path))).slice(0, 3).map((p) => 'The same on ' + p + '.'),
446
+ ['The same pages replayed at desktop size threw no such error - so a phone screen is what breaks it (often a script that looks for something only one layout has).']),
447
+ } : null;
179
448
  const clean = (sweep.clicks || []).filter((c) => !c.blocked.length && !c.failed);
180
449
  const wrote = (sweep.clicks || []).filter((c) => c.blocked.length).map((c) => '"' + c.name + '" (' + c.blocked[0] + ')');
181
450
  const unclicked = [];
@@ -183,6 +452,7 @@ function judgePhone(flow, result) {
183
452
  if (wrote.length) unclicked.push('a click that tried to save was stopped, so what it does was not seen: ' + wrote.slice(0, 5).join(', '));
184
453
  const hits = clean.map((c) => Object.assign({}, c, { errors: c.errors.filter((e) => !atLoad.has(e.text)) })).filter((c) => c.errors.length);
185
454
  if (!hits.length) {
455
+ if (loadFinding) return { outcome: 'found', unclicked: unclicked, finding: loadFinding };
186
456
  return { outcome: clean.length ? 'held' : 'not tested', unclicked: unclicked,
187
457
  why: clean.length ? 'at phone size, ' + clean.length + (clean.length === 1 ? ' button was' : ' buttons were') + ' clicked on ' +
188
458
  sweep.pages.length + (sweep.pages.length === 1 ? ' page' : ' pages') + ', and none made the page log an error'
@@ -195,6 +465,7 @@ function judgePhone(flow, result) {
195
465
  return {
196
466
  outcome: 'found',
197
467
  unclicked: unclicked,
468
+ loadFinding: loadFinding,
198
469
  finding: {
199
470
  flow: flow.title, where: flow.file, kind: 'phone', type: 'phone',
200
471
  severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
@@ -210,6 +481,12 @@ function judgePhone(flow, result) {
210
481
  };
211
482
  }
212
483
 
484
+ /** A recording made at phone size: codegen wrote a narrow viewport into it. */
485
+ function isPhoneRecording(source) {
486
+ const m = /viewport\s*:\s*\{\s*width\s*:\s*(\d+)/.exec(String(source || ''));
487
+ return Boolean(m) && Number(m[1]) < 768;
488
+ }
489
+
213
490
  /** The writes a run made between the marks around one step. */
214
491
  function windowWrites(result, line) {
215
492
  const events = eventsOf(result);
@@ -236,7 +513,7 @@ function show(value) {
236
513
  }
237
514
 
238
515
  function words(w) {
239
- return w.method + ' ' + (w.host || '') + w.path + ' -> ' + w.status;
516
+ return probe.writeWords(w);
240
517
  }
241
518
 
242
519
  /**
@@ -246,6 +523,14 @@ function words(w) {
246
523
  */
247
524
  function recordIds(w) {
248
525
  const answer = w && w.answer;
526
+ // A form's redirect names the record it made in the address it sends the
527
+ // browser to (POST /projects -> /projects/13): a part of that address that
528
+ // looks like an id and was not in the address the form posted to.
529
+ if (answer == null && w && w.to && w.status >= 300 && w.status < 400) {
530
+ const posted = String(w.path || '').split('/');
531
+ return String(w.to).split(/[?#]/)[0].split('/').filter((p) => p && !posted.includes(p) &&
532
+ (/^\d+$/.test(p) || /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(p) || /^[0-9a-f]{24}$/i.test(p))).slice(-1);
533
+ }
249
534
  if (answer == null) return [];
250
535
  const rows = Array.isArray(answer) ? answer : answer && typeof answer === 'object' && answer.data !== undefined
251
536
  ? [].concat(answer.data) : [answer];
@@ -286,7 +571,9 @@ function singleClickWords(writes) {
286
571
 
287
572
  /** "Create work order" -> "work order": what the step makes, for the headline. */
288
573
  function thingOf(name) {
289
- const rest = String(name || '').replace(/^[^\p{L}\p{N}]*(add|create|new|save|submit|post|upload|send|register)\b\s*/iu, '').trim();
574
+ // An arrow or icon after the words is not part of the thing (blind round
575
+ // 20: "Add note →" made "2 note →s").
576
+ const rest = String(name || '').replace(/^[^\p{L}\p{N}]*(add|create|new|save|submit|post|upload|send|register)\b\s*/iu, '').replace(/[^\p{L}\p{N}]+$/u, '').trim();
290
577
  return rest || 'it';
291
578
  }
292
579
 
@@ -374,26 +661,39 @@ function judgeDouble(flow, step, single, double) {
374
661
  // The server naming one record twice saved it once and answered twice:
375
662
  // that is the safe way to handle a double-click, not a duplicate.
376
663
  const ids = answers.map(recordIds);
377
- const named = ids.every((list) => list.length);
378
664
  const distinct = Array.from(new Set([].concat.apply([], ids)));
665
+ // A redirect may name the page a record belongs to (POST /comments ->
666
+ // /tasks/5), not the record: the same address twice says nothing, two
667
+ // addresses say two records.
668
+ const named = ids.every((list) => list.length) && !(redirectOnly(answers) && distinct.length < 2);
379
669
  if (named && distinct.length === 1) {
380
670
  return { outcome: 'held', why: 'the server answered both clicks with the same record (id ' + distinct[0] + ') - it was saved once' };
381
671
  }
672
+ // A form's redirect names no record: the page opened again decides.
673
+ const byPage = !named && redirectOnly(answers);
674
+ const page = byPage ? pageSays(flow, step, single, double) : null;
675
+ if (byPage && page && !page.more) {
676
+ return { outcome: 'held', why: 'both clicks were answered with a redirect, and the page opened again shows "' + page.text + '" added ' + timesWord(page.now) + ', as after one click - it was kept once' };
677
+ }
678
+ const unsure = byPage && !page;
382
679
  const same = answers.length >= 2 && JSON.stringify(answers[0].data) === JSON.stringify(answers[1].data);
383
680
  const thing = thingOf(step.name);
384
681
  return {
385
682
  outcome: 'duplicate',
386
683
  finding: {
387
684
  flow: flow.title, where: flow.file + ':' + step.line, kind: 'double', type: 'double', step: step.name,
388
- severity: 'HIGH', status: 'confirmed',
389
- headline: 'Double-clicking "' + step.name + '" saves it twice.',
685
+ severity: unsure ? 'LOW' : 'HIGH', status: unsure ? 'verification required' : 'confirmed',
686
+ headline: unsure ? 'Double-clicking "' + step.name + '" was answered as two saves - check whether it keeps two.' : 'Double-clicking "' + step.name + '" saves it twice.',
390
687
  proof: [
391
688
  'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
392
689
  'One click on "' + step.name + '": ' + before + ' x ' + route.split(' ')[0] + ' ' + (answers[0].host || '') + answers[0].path + ' answered ' + (one[route] || []).map((w) => w.status).join(', ') + '.',
393
- 'A quick double-click: ' + answers.length + ' x, answered ' + answers.map((w) => w.status).join(', ') + ' - ' + answers.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant.',
690
+ 'A quick double-click: ' + answers.length + ' x, answered ' + answers.map((w) => w.status).join(', ') + ' - ' +
691
+ (unsure ? 'both taken; whether two were kept is not shown' : answers.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant') + '.',
394
692
  ].concat(answers.slice(0, 2).map((w, i) => 'Answer ' + (i + 1) + ': ' + words(w) + ', sent ' + show(w.data !== undefined ? w.data : w.text))).concat(
395
693
  same ? ['Both requests carried the same data: the same thing, saved twice.'] : [],
396
694
  named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
695
+ : page ? ['The page opened again after the double-click shows "' + page.text + '" added ' + timesWord(page.now) + '; after one click, ' + timesWord(page.once) + '.']
696
+ : unsure ? ['Both were answered with a redirect, which names no record, and the page does not show what was saved - so whether two were kept is not proven.']
397
697
  : ['The server\'s answers do not say which record each one made, so the proof is the ' + answers.length + ' successful answers.']),
398
698
  },
399
699
  };
@@ -422,29 +722,65 @@ function judgeTwice(flow, step, single, twice, pageHeld) {
422
722
  const route = routeOf(copy);
423
723
  const pair = both.filter((w) => routeOf(w) === route && w.method === copy.method && !w.signIn);
424
724
  const okOnes = pair.filter(ok);
725
+ // A server error is not a refusal: the server failed on two of the same
726
+ // save at once, where one alone was answered fine. Found on blind round
727
+ // 20: a note numbered MAX()+1 in two steps answered 201 and 500, and was
728
+ // said as "the other was refused".
729
+ const crashed = pair.find((w) => w.status >= 500);
730
+ const fine = (once || []).find((w) => routeOf(w) === route && ok(w));
731
+ if (crashed && fine) {
732
+ const thing = thingOf(step.name);
733
+ return {
734
+ outcome: 'duplicate',
735
+ finding: {
736
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'race', type: 'race-error', step: step.name,
737
+ severity: 'MEDIUM', status: 'confirmed',
738
+ headline: 'Sent twice at the same moment, "' + step.name + '" makes your server fail - one of the two was answered ' + crashed.status + '.',
739
+ proof: [
740
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
741
+ 'One click on "' + step.name + '" alone: ' + words(fine) + '.',
742
+ 'That request and a copy of it - same body, same sign-in - sent at the same moment: answered ' + pair.map((w) => w.status).join(', ') + '.',
743
+ 'A ' + crashed.status + ' is your server failing, not refusing: ' + (okOnes.length ? okOnes.length + ' ' + thing + (okOnes.length === 1 ? ' was' : 's were') + ' saved, and the other request crashed.' : 'neither was saved.'),
744
+ ],
745
+ },
746
+ };
747
+ }
425
748
  if (okOnes.length < 2) {
426
749
  const refused = pair.find((w) => !ok(w));
427
750
  return { outcome: 'held', why: 'sent twice at the same moment, it was saved once' + (refused ? ' - the other was refused (' + words(refused) + ')' : '') };
428
751
  }
429
752
  const ids = okOnes.map(recordIds);
430
- const named = ids.every((list) => list.length);
431
753
  const distinct = Array.from(new Set([].concat.apply([], ids)));
754
+ // The same redirect twice may only name the page both belong to.
755
+ const named = ids.every((list) => list.length) && !(redirectOnly(okOnes) && distinct.length < 2);
432
756
  if (named && distinct.length === 1) {
433
757
  return { outcome: 'held', why: 'sent twice at the same moment, the server answered both with the same record (id ' + distinct[0] + ') - it was saved once' };
434
758
  }
759
+ // A form's redirect names no record: the page opened again decides.
760
+ const byPage = !named && redirectOnly(okOnes);
761
+ const page = byPage ? pageSays(flow, step, single, twice) : null;
762
+ if (byPage && page && !page.more) {
763
+ return { outcome: 'held', why: 'sent twice at the same moment, both were answered with a redirect, and the page opened again shows "' + page.text + '" added ' + timesWord(page.now) + ', as after one click - it was kept once' };
764
+ }
765
+ const unsure = byPage && !page;
435
766
  const thing = thingOf(step.name);
436
767
  return {
437
768
  outcome: 'duplicate',
438
769
  finding: {
439
770
  flow: flow.title, where: flow.file + ':' + step.line, kind: 'race', type: 'race', step: step.name,
440
- severity: 'MEDIUM', status: 'confirmed',
441
- headline: 'Sent twice at the same moment, "' + step.name + '" saves twice - your server lets a repeated save through.',
771
+ severity: unsure ? 'LOW' : 'MEDIUM', status: unsure ? 'verification required' : 'confirmed',
772
+ headline: unsure ? 'Sent twice at the same moment, "' + step.name + '" was answered as two saves - check whether your server keeps two.'
773
+ : 'Sent twice at the same moment, "' + step.name + '" saves twice - your server lets a repeated save through.',
442
774
  proof: [
443
775
  'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
444
776
  'One click on "' + step.name + '" sends ' + copy.method + ' ' + (copy.host || '') + copy.path + '.',
777
+ // Not proven two: said as two taken, never as "2 settings created"
778
+ // for a save that may only change one record (blind round 21).
445
779
  'That request and a copy of it - same body, same sign-in - sent at the same moment: answered ' + okOnes.map((w) => w.status).join(', ') + ' - ' +
446
- okOnes.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant.',
780
+ (unsure ? 'both taken; whether two were kept is not shown' : okOnes.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant') + '.',
447
781
  ].concat(named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
782
+ : page ? ['The page opened again afterwards shows "' + page.text + '" added ' + timesWord(page.now) + '; after one click, ' + timesWord(page.once) + '.']
783
+ : unsure ? ['Both were answered with a redirect, which names no record, and the page does not show what was saved - so whether two were kept is not proven.']
448
784
  : ['The server\'s answers do not say which record each one made, so the proof is the ' + okOnes.length + ' successful answers.'],
449
785
  pageHeld ? ['A double-click on the page saved it once, so the page guards against this and the server does not. A slow network, a retry or a second tab sends the same request twice.'] : []),
450
786
  },
@@ -517,6 +853,9 @@ function fixPrompt(f) {
517
853
  double: 'Make saving safe against a double-click: disable the button while the save is in flight, and make the server ignore a repeat of the same request (an idempotency key, or a unique constraint on what makes it the same).',
518
854
  race: 'Make the server refuse a repeat of the same save: send an idempotency key with each save and keep the first answer for it, or add a unique constraint on what makes two the same. Disabling the button is not enough on its own.',
519
855
  phone: 'Make that button work at phone size without an error: find what the click runs at a narrow screen (a menu, a drawer, a panel that is hidden or not there yet) and handle it. Check it at 390 by 844.',
856
+ 'race-error': 'Find why two of the same save at once make the server fail: look for a value that is read and then written in two steps (a next number from MAX()+1, a check-then-insert) and make it one step - a sequence or identity column, INSERT ... ON CONFLICT, or a transaction that locks the row. When a unique rule refuses a repeat, answer 409 with a message, never 500.',
857
+ lost: 'Find why this field is not saved by this form: check that the server reads it from the request and writes it in the update (an empty value too - clearing a field must clear it, not keep the old value). Do not change any other field.',
858
+ changed: 'Check whether the value saved for this field is what the person typed. If the app changes it on purpose (rounding, a format), say so on the form; otherwise save it as typed.',
520
859
  late: 'After a save succeeds, update what the page shows - add the new item to the list from the server\'s answer, or load the list again - so the person sees it without opening the page again. Otherwise they think it did not save, and save it again.',
521
860
  }[f.type];
522
861
  return [
@@ -540,8 +879,15 @@ async function run(root, flows, options) {
540
879
  const opts = options || {};
541
880
  const say = typeof opts.say === 'function' ? opts.say : () => {};
542
881
  const budget = Math.min(probe.MAX_RUNS, opts.budget || probe.MAX_RUNS);
882
+ // Many flows: two plain runs each, not three - a failure next to a pass is
883
+ // seen in two - so the cap goes on the saves (twice each: a double-click
884
+ // and two at once) rather than on a third look. Found on blind round 20:
885
+ // twelve recordings, and the invitation and appointment saves were over
886
+ // the cap, never sent twice.
887
+ const savingCost = flows.reduce((n, f) => n + (saveSteps(f.source).length ? ROUNDS + 2 * saveSteps(f.source).length : 0), 0);
543
888
  const rounds = opts.only && opts.only.kind === 'flaky' ? RECHECK_ROUNDS
544
- : opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race' || opts.only.kind === 'phone') ? 1 : ROUNDS;
889
+ : opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race' || opts.only.kind === 'phone') ? 1
890
+ : !opts.only && savingCost > budget * 0.75 ? 2 : ROUNDS;
545
891
  let stopped = false;
546
892
  let current = null;
547
893
  const control = {
@@ -557,7 +903,7 @@ async function run(root, flows, options) {
557
903
  }
558
904
  const report = {
559
905
  check: 'reliability', startedAt: new Date().toISOString(), endedAt: null,
560
- stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [],
906
+ stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [], writeHostsFrom: 'page+vendor',
561
907
  };
562
908
 
563
909
  const runWith = async (probes, labels) => {
@@ -573,7 +919,10 @@ async function run(root, flows, options) {
573
919
  // least their saving steps once.
574
920
  const chosen = [];
575
921
  let planned = 0;
576
- for (const f of flows) {
922
+ // Flows that save first: a flow that only looks has less to find, and
923
+ // takes its place in the cap after them.
924
+ const ordered = flows.filter((f) => saveSteps(f.source).length).concat(flows.filter((f) => !saveSteps(f.source).length));
925
+ for (const f of ordered) {
577
926
  if (opts.only && f.file !== opts.only.file) continue;
578
927
  if (replay.findReplayRisks(f.source).some((r) => r.kind === 'signup')) {
579
928
  report.flows.push({ title: f.title, file: f.file, state: 'skipped', why: 'it creates an account, and this check never creates accounts' });
@@ -737,7 +1086,13 @@ async function run(root, flows, options) {
737
1086
  // write blocked. Within what is left of the cap.
738
1087
  if (!(opts.only && opts.only.kind !== 'phone')) {
739
1088
  const phones = [];
740
- for (const f of chosen) {
1089
+ // At most PHONE_FLOWS: the pages repeat from flow to flow, and the runs
1090
+ // are better spent on saves. Phone recordings first.
1091
+ const phoneFlows = opts.only ? chosen : chosen.filter((f) => isPhoneRecording(f.source)).concat(chosen.filter((f) => !isPhoneRecording(f.source))).slice(0, PHONE_FLOWS);
1092
+ for (const f of chosen.filter((x) => !phoneFlows.includes(x))) {
1093
+ report.notTested.push({ flow: f.title, where: f.file, kind: 'phone', why: 'only ' + PHONE_FLOWS + ' flows are replayed at phone size in one press - their pages are mostly the same' });
1094
+ }
1095
+ for (const f of phoneFlows) {
741
1096
  const label = { flow: f.title, where: f.file, kind: 'phone' };
742
1097
  if (stopped || report.runsDone + phones.length + 1 > budget) {
743
1098
  report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it ran' : 'over the limit of ' + budget + ' runs' }));
@@ -754,21 +1109,32 @@ async function run(root, flows, options) {
754
1109
  // problem, however many flows pass by it (blind round 17: one menu
755
1110
  // button, said five times).
756
1111
  const samePhone = new Map();
1112
+ // What the pages throw at desktop size: every page error in the plain
1113
+ // runs of flows recorded at desktop size (a phone recording replays at
1114
+ // phone size, so it says nothing about the desktop).
1115
+ const atDesktop = chosen.filter((f) => !isPhoneRecording(f.source));
1116
+ const desktopErrors = atDesktop.length ? new Set([].concat.apply([], atDesktop.map((f) => [].concat.apply([], f.runs.map((r) => eventsOf(r).filter((e) => e.kind === 'error').map((e) => e.text)))))) : null;
757
1117
  for (const p of phones) {
758
- const v = judgePhone(p.flow, swept[p.id]);
1118
+ const v = judgePhone(p.flow, swept[p.id], desktopErrors);
759
1119
  const label = { flow: p.flow.title, where: p.flow.file, kind: 'phone' };
760
- const same = v.finding ? v.finding.headline + '|' + v.finding.proof[2] : null;
761
- if (same && samePhone.has(same)) {
762
- const first = samePhone.get(same);
763
- first.alsoIn = (first.alsoIn || 0) + 1;
764
- if (first.alsoIn <= 3) first.proof.splice(first.proof.length - 1, 0, 'The same in "' + p.flow.title + '" (' + p.flow.file + ').');
765
- report.held.push(Object.assign(label, { why: 'at phone size, the same error as in "' + first.flow + '" - said once, there' }));
766
- } else if (v.finding) {
767
- if (same) samePhone.set(same, v.finding);
768
- report.findings.push(v.finding);
1120
+ let reported = false;
1121
+ for (const f of [v.finding, v.loadFinding].filter(Boolean)) {
1122
+ // One error on one page is one problem, however many flows pass
1123
+ // by it; one thrown at load is one problem wherever it loads.
1124
+ const same = f.type === 'phone-load' ? 'load|' + f.proof[1] : f.headline + '|' + f.proof[2];
1125
+ if (samePhone.has(same)) {
1126
+ const first = samePhone.get(same);
1127
+ first.alsoIn = (first.alsoIn || 0) + 1;
1128
+ if (first.alsoIn <= 3) first.proof.splice(first.proof.length - 1, 0, 'The same in "' + p.flow.title + '" (' + p.flow.file + ').');
1129
+ if (f === v.finding || !v.finding) report.held.push(Object.assign({}, label, { why: 'at phone size, the same error as in "' + first.flow + '" - said once, there' }));
1130
+ } else {
1131
+ samePhone.set(same, f);
1132
+ report.findings.push(f);
1133
+ }
1134
+ reported = true;
769
1135
  }
770
- else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
771
- else report.notTested.push(Object.assign(label, { why: !swept[p.id] && stopped ? 'stopped before it finished' : v.why }));
1136
+ if (!reported && v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
1137
+ else if (!reported) report.notTested.push(Object.assign(label, { why: !swept[p.id] && stopped ? 'stopped before it finished' : v.why }));
772
1138
  // What was left unclicked is its own line, so a re-check is not held
773
1139
  // up by it.
774
1140
  for (const why of v.unclicked || []) report.notTested.push(Object.assign({}, label, { kind: 'phone-unclicked', why: why }));
@@ -791,6 +1157,23 @@ async function run(root, flows, options) {
791
1157
  }
792
1158
  }
793
1159
 
1160
+ // 4d. What each edit typed, read back from the form opened again after its
1161
+ // save - from the plain runs, so it costs no run of its own.
1162
+ if (!(opts.only && opts.only.kind !== 'kept')) {
1163
+ for (const f of chosen) {
1164
+ for (const k of keepSteps(f.source)) {
1165
+ if (!keptFields(f.source, k).length) continue;
1166
+ if (opts.only && opts.only.line && opts.only.line !== k.line) continue;
1167
+ const v = judgeKept(f, k, f.runs);
1168
+ const label = { flow: f.title, where: f.file + ':' + k.line, kind: 'kept', step: k.name };
1169
+ if (v.outcome === 'not an edit') continue;
1170
+ if (v.findings) for (const x of v.findings) report.findings.push(x);
1171
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
1172
+ else report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it finished' : v.why }));
1173
+ }
1174
+ }
1175
+ }
1176
+
794
1177
  // 5. The three runs of each flow. A re-check of a double-click ran its
795
1178
  // flow once, only to have a single click to compare with - one run says
796
1179
  // nothing about flakiness, so the flow counts as tested when that run
@@ -807,7 +1190,7 @@ async function run(root, flows, options) {
807
1190
  }
808
1191
  // A re-check of what a save shows needs the runs to have finished, not a
809
1192
  // verdict on flakiness.
810
- if (opts.only && opts.only.kind === 'shown') {
1193
+ if (opts.only && (opts.only.kind === 'shown' || opts.only.kind === 'kept')) {
811
1194
  const finished = f.runs.some((r) => r && !limited(r));
812
1195
  report.flows.push({ title: f.title, file: f.file, state: finished ? 'tested' : 'not tested', why: finished ? '' : 'none of the runs finished' });
813
1196
  continue;
@@ -832,7 +1215,9 @@ async function run(root, flows, options) {
832
1215
 
833
1216
  // What a finding is about, for keeping it across presses: the flow for a
834
1217
  // flaky one, the step for a double-click.
835
- const keyOf = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind;
1218
+ const keyOf = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind + (x.kind === 'kept' && x.field ? '|' + x.field : '');
1219
+ // A step held or not tested says so for every field it read back.
1220
+ const stepKey = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind;
836
1221
 
837
1222
  /**
838
1223
  * What a press did not try is not cleared by it: a finding from an earlier
@@ -842,7 +1227,7 @@ function carryOver(previous, report, root) {
842
1227
  if (!previous || !Array.isArray(previous.findings)) return report;
843
1228
  const tried = new Set(report.held.concat(report.findings).map(keyOf));
844
1229
  for (const f of previous.findings) {
845
- if (tried.has(keyOf(f))) continue;
1230
+ if (tried.has(keyOf(f)) || (f.kind === 'kept' && tried.has(stepKey(f)))) continue;
846
1231
  const file = String(f.where).replace(/:\d+$/, '');
847
1232
  if (root && !fs.existsSync(path.join(root, file))) continue;
848
1233
  const kept = Object.assign({}, f, { carried: f.carried || previous.endedAt || previous.startedAt });
@@ -876,7 +1261,7 @@ async function recheck(root, flows, finding, options) {
876
1261
  const unsure = (why) => ({ verdict: 'could not confirm', why: why, newProblems: [], testData: [] });
877
1262
  const flow = flows.find((f) => f.file === file);
878
1263
  if (!flow) return unsure('the recording ' + file + ' is not there, or cannot be run now');
879
- if (finding.kind !== 'flaky' && finding.kind !== 'phone' && !saveSteps(flow.source).some((s) => s.line === line)) return unsure('that step is no longer at line ' + line + ' of ' + file);
1264
+ if (finding.kind !== 'flaky' && finding.kind !== 'phone' && !(finding.kind === 'kept' ? keepSteps(flow.source) : saveSteps(flow.source)).some((s) => s.line === line)) return unsure('that step is no longer at line ' + line + ' of ' + file);
880
1265
  const report = await run(root, [flow], Object.assign({}, opts, {
881
1266
  only: finding.kind === 'flaky' || finding.kind === 'phone' ? { file: file, kind: finding.kind } : { file: file, kind: finding.kind, line: line },
882
1267
  }));
@@ -888,13 +1273,14 @@ async function recheck(root, flows, finding, options) {
888
1273
  if (report.stopped) return unsure('it was stopped before it finished');
889
1274
  const ran = report.flows.find((f) => f.file === file);
890
1275
  if (!ran || ran.state !== 'tested') return unsure(ran ? ran.why : 'the flow did not run');
891
- const left = report.notTested.find((x) => keyOf(x) === keyOf(finding));
1276
+ const left = report.notTested.find((x) => keyOf(x) === keyOf(finding) || (finding.kind === 'kept' && stepKey(x) === stepKey(finding)));
892
1277
  if (left) return unsure(left.why);
893
1278
  const same = report.findings.find((f) => keyOf(f) === keyOf(finding));
894
1279
  const others = report.findings.filter((f) => f !== same && f.status === 'confirmed');
895
1280
  const result = same ? { verdict: 'still open', why: same.proof.slice(1).join(' '), newProblems: others }
896
1281
  : { verdict: 'fixed', why: finding.kind === 'flaky' ? 'it worked all ' + RECHECK_ROUNDS + ' times'
897
1282
  : finding.kind === 'shown' || finding.kind === 'race' || finding.kind === 'phone' ? (report.held.find((x) => keyOf(x) === keyOf(finding)) || {}).why || 'it held'
1283
+ : finding.kind === 'kept' ? 'the form opened again after the save shows what was typed'
898
1284
  : 'a double-click saved it once', newProblems: others };
899
1285
  const saved = load(root, opts.env);
900
1286
  if (saved && result.verdict === 'fixed') {
@@ -909,6 +1295,7 @@ async function recheck(root, flows, finding, options) {
909
1295
  function heldLabel(x) {
910
1296
  return x.kind === 'flaky' ? x.flow + ' - '
911
1297
  : x.kind === 'shown' ? 'After "' + x.step + '" (' + x.where + '), looked for on the page - '
1298
+ : x.kind === 'kept' ? 'After "' + x.step + '" (' + x.where + '), the form read back - '
912
1299
  : x.kind === 'race' ? '"' + x.step + '" (' + x.where + ') sent twice at once - '
913
1300
  : x.kind === 'phone' || x.kind === 'phone-unclicked' ? 'At phone size, ' + x.flow + ' - '
914
1301
  : '"' + x.step + '" (' + x.where + ') double-clicked - ';
@@ -968,6 +1355,9 @@ module.exports = {
968
1355
  judgeFlaky: judgeFlaky,
969
1356
  judgeDouble: judgeDouble,
970
1357
  judgeShown: judgeShown,
1358
+ judgeKept: judgeKept,
1359
+ keepSteps: keepSteps,
1360
+ keptFields: keptFields,
971
1361
  judgeTwice: judgeTwice,
972
1362
  judgePhone: judgePhone,
973
1363
  phoneSource: phoneSource,