kryptheon 0.1.49 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +435 -333
- package/bin/kryptheon.js +90 -4
- package/kryptheon-code.js +158 -2
- package/kryptheon-dashboard-page.js +78 -6
- package/kryptheon-dashboard.html +9 -0
- package/kryptheon-dashboard.js +240 -12
- package/kryptheon-edge.js +88 -14
- package/kryptheon-heal.js +57 -0
- package/kryptheon-payments-run.js +501 -0
- package/kryptheon-payments.js +626 -0
- package/kryptheon-probe-fixture.js +175 -8
- package/kryptheon-probe.js +368 -317
- package/kryptheon-reliability.js +430 -40
- package/kryptheon-replay.js +0 -0
- package/kryptheon-secrets.js +14 -3
- package/package.json +12 -5
package/kryptheon-reliability.js
CHANGED
|
@@ -51,16 +51,23 @@ const ROUNDS = 3;
|
|
|
51
51
|
// The size a phone recording is made at (bin/kryptheon.js PHONE_SIZE).
|
|
52
52
|
const PHONE = { width: 390, height: 844 };
|
|
53
53
|
const CONFIRM_RUNS = 5;
|
|
54
|
+
// Flows replayed at phone size in one press.
|
|
55
|
+
const PHONE_FLOWS = 3;
|
|
54
56
|
// A re-check of a flaky flow needs more than three runs to say anything.
|
|
55
57
|
const RECHECK_ROUNDS = 5;
|
|
56
58
|
|
|
57
59
|
const CLICK = /^(\s*)(await\s+)?(page\s*\..+?)\.click\(\s*\)\s*;?\s*$/;
|
|
58
60
|
|
|
59
|
-
|
|
61
|
+
// Taken by the app: a 2xx, or a form's redirect after it saved (probe.accepted).
|
|
62
|
+
const ok = (w) => probe.accepted(w);
|
|
60
63
|
const eventsOf = (r) => (r && r.events) || [];
|
|
61
64
|
const writesOf = (r) => eventsOf(r).filter((e) => e.kind === 'write');
|
|
62
65
|
const leftBehind = (w) => ok(w) && w.method !== 'DELETE' && !w.signIn;
|
|
63
|
-
|
|
66
|
+
// Where a write went, a record's id left out: a step that saves to the third
|
|
67
|
+
// course in a list saves to another course once runs have added courses
|
|
68
|
+
// (blind round 21), and is still the same save.
|
|
69
|
+
const routeOf = (w) => w.method + ' ' + (w.origin || w.host || '') +
|
|
70
|
+
String(w.path || '').replace(/\/(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24})(?=\/|$)/gi, '/:id');
|
|
64
71
|
const limited = (r) => eventsOf(r).some((e) => e.kind === 'limited');
|
|
65
72
|
|
|
66
73
|
/**
|
|
@@ -72,8 +79,7 @@ const limited = (r) => eventsOf(r).some((e) => e.kind === 'limited');
|
|
|
72
79
|
function saveSteps(source) {
|
|
73
80
|
const lines = String(source || '').split('\n');
|
|
74
81
|
const out = [];
|
|
75
|
-
for (const a of
|
|
76
|
-
if (a.method !== 'click' || !a.name) continue;
|
|
82
|
+
for (const a of clickSteps(source)) {
|
|
77
83
|
if (!replay.CREATES.test(a.name.trim()) || replay.SIGNUP.test(a.name)) continue;
|
|
78
84
|
const m = CLICK.exec(lines[a.line - 1].replace(/\r$/, ''));
|
|
79
85
|
if (!m) continue;
|
|
@@ -82,6 +88,37 @@ function saveSteps(source) {
|
|
|
82
88
|
return out;
|
|
83
89
|
}
|
|
84
90
|
|
|
91
|
+
// A click into a field: it saves nothing, whatever the field is called
|
|
92
|
+
// (blind round 21: a click on "Add a quick class note" was taken for a save).
|
|
93
|
+
const FIELD_CLICK = /getBy(Placeholder|Label)\(|getByRole\(\s*['"](textbox|searchbox|combobox|spinbutton|checkbox|radio|option|listbox)['"]/;
|
|
94
|
+
// A button with no name of its own: getByRole('button') and nothing more.
|
|
95
|
+
const NAMELESS_BUTTON = /getByRole\(\s*['"]button['"]\s*\)\s*\.click\(\s*\)/;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Every click a person made on something that can save, with a name to call
|
|
99
|
+
* it by. A click into a field is left out. A button with no name, clicked
|
|
100
|
+
* right after typing into a field (a card's own send button), is named for
|
|
101
|
+
* that field: 'Save "Add a quick class note"' (blind round 21: the calendar's
|
|
102
|
+
* note button had no name, so the note it saved was never checked).
|
|
103
|
+
*/
|
|
104
|
+
function clickSteps(source) {
|
|
105
|
+
const actions = replay.readActions(source);
|
|
106
|
+
const out = [];
|
|
107
|
+
actions.forEach((a, i) => {
|
|
108
|
+
if (a.method !== 'click' || FIELD_CLICK.test(a.text)) return;
|
|
109
|
+
if (a.name) {
|
|
110
|
+
out.push(a);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
if (!NAMELESS_BUTTON.test(a.text)) return;
|
|
114
|
+
const before = actions.slice(0, i).reverse().find((b) => !(b.method === 'click' && FIELD_CLICK.test(b.text)));
|
|
115
|
+
if (before && before.method === 'fill' && before.name && !replay.isSecretField(before.name)) {
|
|
116
|
+
out.push(Object.assign({}, a, { name: 'Save "' + before.name + '"' }));
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
|
|
85
122
|
/**
|
|
86
123
|
* What was typed before a saving step, to look for on the page after it:
|
|
87
124
|
* each literal value filled in since the last step that saved, three
|
|
@@ -118,9 +155,189 @@ function plainSource(source, steps) {
|
|
|
118
155
|
before[s.line] = (typed.length ? [seen('before')] : []).concat([s.indent + "await page.__kxMark('before-" + s.line + "');"]);
|
|
119
156
|
after[s.line] = [s.indent + "await page.__kxMark('after-" + s.line + "');"].concat(typed.length ? [seen('after'), seen('fresh', true)] : []);
|
|
120
157
|
}
|
|
158
|
+
// Each step that keeps what was typed: what each field held before it was
|
|
159
|
+
// typed into, and the form opened again after the step, read back.
|
|
160
|
+
const push = (map, line, text) => (map[line] = [].concat(map[line] || [], [text]));
|
|
161
|
+
for (const k of keepSteps(source)) {
|
|
162
|
+
const fields = keptFields(source, k);
|
|
163
|
+
if (!fields.length) continue;
|
|
164
|
+
for (const f of fields) push(before, f.firstLine, f.indent + 'await page.__kxWas(' + f.line + ', ' + f.locator + ');');
|
|
165
|
+
if (!steps.some((s) => s.line === k.line)) {
|
|
166
|
+
push(before, k.line, k.indent + "await page.__kxMark('before-" + k.line + "');");
|
|
167
|
+
push(after, k.line, k.indent + "await page.__kxMark('after-" + k.line + "');");
|
|
168
|
+
}
|
|
169
|
+
push(after, k.line, k.indent + 'await page.__kxReadBack(' + JSON.stringify('kept-' + k.line) + ', [' +
|
|
170
|
+
fields.map((f) => '{ line: ' + f.line + ', get: (p) => ' + f.locator.replace(/^page\s*\./, 'p.') + ' }').join(', ') + ']);');
|
|
171
|
+
}
|
|
121
172
|
return probe.probeSource(source, { before: before, after: after });
|
|
122
173
|
}
|
|
123
174
|
|
|
175
|
+
// A click that keeps what was typed: Save, Update, Apply...
|
|
176
|
+
const KEEPS = /^[^\p{L}\p{N}]*(save|update|apply|submit|done|confirm)\b/iu;
|
|
177
|
+
|
|
178
|
+
/** The steps after which what was typed is read back: a plain click on a button that saves or updates. */
|
|
179
|
+
function keepSteps(source) {
|
|
180
|
+
const lines = String(source || '').split('\n');
|
|
181
|
+
const out = [];
|
|
182
|
+
for (const a of clickSteps(source)) {
|
|
183
|
+
if (!(KEEPS.test(a.name.trim()) || replay.CREATES.test(a.name.trim())) || replay.SIGNUP.test(a.name)) continue;
|
|
184
|
+
const m = CLICK.exec(lines[a.line - 1].replace(/\r$/, ''));
|
|
185
|
+
if (!m) continue;
|
|
186
|
+
out.push({ line: a.line, indent: m[1], locator: m[3], name: a.name.trim() });
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* The fields typed into before a step that keeps them: each literal fill
|
|
193
|
+
* since the last such step, empty ones too (clearing a field is an edit),
|
|
194
|
+
* never the sign-in or a secret.
|
|
195
|
+
*/
|
|
196
|
+
function keptFields(source, step) {
|
|
197
|
+
const steps = keepSteps(source);
|
|
198
|
+
const since = Math.max(0, ...steps.filter((s) => s.line < step.line).map((s) => s.line));
|
|
199
|
+
const lines = String(source || '').split('\n');
|
|
200
|
+
let signInEnd = 0;
|
|
201
|
+
lines.forEach((line, i) => {
|
|
202
|
+
if (/\.fill\(/.test(line) && (replay.isSecretField(replay.nameInLine(line)) || /process\.env\./.test(line))) signInEnd = i + 1;
|
|
203
|
+
});
|
|
204
|
+
// Each field once: its last fill is what was saved, and what it held is
|
|
205
|
+
// read before its first - a typo fixed by typing again is still one edit.
|
|
206
|
+
const byLocator = new Map();
|
|
207
|
+
for (const f of edge.fillSteps(source)) {
|
|
208
|
+
if (f.line <= since || f.line >= step.line || f.line <= signInEnd || replay.isSecretField(f.name) || !/^page\s*\./.test(f.locator)) continue;
|
|
209
|
+
const first = byLocator.has(f.locator) ? byLocator.get(f.locator).firstLine : f.line;
|
|
210
|
+
byLocator.set(f.locator, Object.assign({}, f, { firstLine: first }));
|
|
211
|
+
}
|
|
212
|
+
return Array.from(byLocator.values());
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// The address of a form for something new: /pets/new, /orders/create.
|
|
216
|
+
const NEW_FORM = /\/(new|create|add)(\/|$)/i;
|
|
217
|
+
|
|
218
|
+
/** Two field values that say the same: the same text, or the same number. */
|
|
219
|
+
function sameValue(a, b) {
|
|
220
|
+
const x = String(a == null ? '' : a).trim();
|
|
221
|
+
const y = String(b == null ? '' : b).trim();
|
|
222
|
+
if (x === y) return true;
|
|
223
|
+
return x !== '' && y !== '' && Number.isFinite(Number(x)) && Number.isFinite(Number(y)) && Number(x) === Number(y);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Whether what was typed before a step that keeps it was kept: the form
|
|
228
|
+
* opened again after the step, against what each field held before it was
|
|
229
|
+
* typed into. Only an edit is judged - a field that held something - and only
|
|
230
|
+
* in a run whose save the app took. Returns { outcome: 'lost' | 'changed' |
|
|
231
|
+
* 'held' | 'not tested', finding?, why }.
|
|
232
|
+
*/
|
|
233
|
+
function judgeKept(flow, step, runs) {
|
|
234
|
+
// What a field held is only what the app saved when the recording did not
|
|
235
|
+
// type it there itself, earlier: a value typed, refused by the form and
|
|
236
|
+
// left in the box is not the app's. Found on the round 19 bench: an invite
|
|
237
|
+
// form kept a bad address the browser had refused, and the next save was
|
|
238
|
+
// judged as an edit of it.
|
|
239
|
+
const typedEarlier = (f) => edge.fillSteps(flow.source).filter((x) => x.locator === f.locator && x.line < f.firstLine).map((x) => x.value);
|
|
240
|
+
const fields = keptFields(flow.source, step).map((f) => Object.assign({}, f, { earlier: typedEarlier(f) }));
|
|
241
|
+
const counted = (runs || []).filter((r) => r && !limited(r));
|
|
242
|
+
const lost = [];
|
|
243
|
+
const changed = [];
|
|
244
|
+
let kept = 0;
|
|
245
|
+
let judged = 0;
|
|
246
|
+
let sentNothing = 0;
|
|
247
|
+
for (const r of counted) {
|
|
248
|
+
const back = eventsOf(r).filter((e) => e.kind === 'readback' && e.name === 'kept-' + step.line);
|
|
249
|
+
if (!back.length) continue;
|
|
250
|
+
const writes = windowWrites(r, step.line) || [];
|
|
251
|
+
const took = writes.filter((w) => ok(w) && !w.signIn && w.method !== 'DELETE');
|
|
252
|
+
// An edit: the field held something the recording did not type there,
|
|
253
|
+
// and was given something else - on a form that is not for something
|
|
254
|
+
// new. A new-record form may show defaults (blind round 20: "Add pet" at
|
|
255
|
+
// /pets/new showed a weight of 5, and was read as an edit of it).
|
|
256
|
+
const notEdit = (f, e) => !f || String(e.was).trim() === '' || sameValue(f.value, e.was) || f.earlier.some((v) => sameValue(v, e.was)) || NEW_FORM.test(String(e.at || ''));
|
|
257
|
+
let edits = 0;
|
|
258
|
+
for (const e of back) {
|
|
259
|
+
const f = fields.find((x) => x.line === e.line);
|
|
260
|
+
if (notEdit(f, e)) continue;
|
|
261
|
+
edits += 1;
|
|
262
|
+
}
|
|
263
|
+
if (!edits) continue;
|
|
264
|
+
if (!took.length) {
|
|
265
|
+
sentNothing += 1;
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
judged += 1;
|
|
269
|
+
for (const e of back) {
|
|
270
|
+
const f = fields.find((x) => x.line === e.line);
|
|
271
|
+
if (notEdit(f, e) || e.now === null) continue;
|
|
272
|
+
const item = { run: counted.indexOf(r) + 1, field: f, was: e.was, now: e.now, write: took[0] };
|
|
273
|
+
if (sameValue(e.now, f.value)) kept += 1;
|
|
274
|
+
else if (sameValue(e.now, e.was)) lost.push(item);
|
|
275
|
+
else changed.push(item);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (!judged) {
|
|
279
|
+
// A form for something new is not an edit: nothing to say, not even
|
|
280
|
+
// "not tested" - every create flow would say it.
|
|
281
|
+
return sentNothing ? { outcome: 'not tested', why: 'the app did not take the save in any run, so what it kept was not read back' }
|
|
282
|
+
: { outcome: 'not an edit', why: 'no field it typed into held anything before, so it is not an edit - nothing to read back' };
|
|
283
|
+
}
|
|
284
|
+
const thing = (item) => '"' + (item.field.name || 'a field') + '"';
|
|
285
|
+
const typedWords = (item) => (String(item.field.value).trim() === '' ? 'cleared' : 'set to ' + JSON.stringify(item.field.value));
|
|
286
|
+
// One finding per field: a description that is not saved and a date that
|
|
287
|
+
// cannot be cleared are two things to fix.
|
|
288
|
+
const byField = (list) => {
|
|
289
|
+
const out = new Map();
|
|
290
|
+
for (const x of list) if (!out.has(x.field.line)) out.set(x.field.line, list.filter((y) => y.field.line === x.field.line));
|
|
291
|
+
return Array.from(out.values());
|
|
292
|
+
};
|
|
293
|
+
const findings = [];
|
|
294
|
+
for (const items of byField(lost)) {
|
|
295
|
+
const first = items[0];
|
|
296
|
+
const runsLost = new Set(items.map((x) => x.run)).size;
|
|
297
|
+
const sure = runsLost >= 2;
|
|
298
|
+
findings.push({
|
|
299
|
+
flow: flow.title, where: flow.file + ':' + step.line, kind: 'kept', type: 'lost', step: step.name, field: first.field.name || 'line ' + first.field.line,
|
|
300
|
+
severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
|
|
301
|
+
headline: 'After "' + step.name + '", ' + thing(first) + ' was ' + typedWords(first) + ', but the form opened again still shows ' + JSON.stringify(first.was) + ' - the change was not kept.',
|
|
302
|
+
proof: [
|
|
303
|
+
'Flow: ' + flow.title + ' (' + flow.file + ':' + first.field.line + ', saved at line ' + step.line + ')',
|
|
304
|
+
'Before typing, ' + thing(first) + ' held ' + JSON.stringify(first.was) + '; the recording ' + (String(first.field.value).trim() === '' ? 'cleared it' : 'typed ' + JSON.stringify(first.field.value)) + '.',
|
|
305
|
+
'The save: ' + words(first.write) + ' - the app took it.',
|
|
306
|
+
'The same form opened again in a new tab: ' + thing(first) + ' shows ' + JSON.stringify(first.now) + ', what it held before.',
|
|
307
|
+
'Seen in ' + runsLost + ' of ' + judged + (judged === 1 ? ' run' : ' runs') + ' that saved.',
|
|
308
|
+
],
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
for (const items of byField(changed)) {
|
|
312
|
+
const first = items[0];
|
|
313
|
+
findings.push({
|
|
314
|
+
flow: flow.title, where: flow.file + ':' + step.line, kind: 'kept', type: 'changed', step: step.name, field: first.field.name || 'line ' + first.field.line,
|
|
315
|
+
severity: 'LOW', status: 'verification required',
|
|
316
|
+
headline: 'After "' + step.name + '", ' + thing(first) + ' was ' + typedWords(first) + ', and the form opened again shows ' + JSON.stringify(first.now) + ' - check that this is meant.',
|
|
317
|
+
proof: [
|
|
318
|
+
'Flow: ' + flow.title + ' (' + flow.file + ':' + first.field.line + ', saved at line ' + step.line + ')',
|
|
319
|
+
'Before typing it held ' + JSON.stringify(first.was) + '. The save: ' + words(first.write) + '.',
|
|
320
|
+
'The same form opened again in a new tab shows ' + JSON.stringify(first.now) + ' - neither what was typed nor what it held.',
|
|
321
|
+
],
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
if (findings.length) return { outcome: lost.length ? 'lost' : 'changed', findings: findings };
|
|
325
|
+
return { outcome: 'held', why: 'the form opened again after it shows what was typed' };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* What was typed for a step, counted on the page before it and on the same
|
|
330
|
+
* address opened again after it - the lines that count it, for a probe copy.
|
|
331
|
+
* A form that redirects names no record in its answer; the page is what says
|
|
332
|
+
* how many were kept.
|
|
333
|
+
*/
|
|
334
|
+
function countLines(source, step) {
|
|
335
|
+
const typed = typedFor(source, saveSteps(source), step);
|
|
336
|
+
if (!typed.length) return { before: [], after: [] };
|
|
337
|
+
const seen = (name, fresh) => step.indent + 'await page.__kxSeen(' + JSON.stringify(name + '-' + step.line) + ', ' + JSON.stringify(typed) + (fresh ? ', true' : '') + ');';
|
|
338
|
+
return { before: [seen('before')], after: [seen('fresh', true)] };
|
|
339
|
+
}
|
|
340
|
+
|
|
124
341
|
/** The same, with one step clicked twice. */
|
|
125
342
|
function doubleSource(source, steps, step) {
|
|
126
343
|
const edits = {};
|
|
@@ -129,8 +346,9 @@ function doubleSource(source, steps, step) {
|
|
|
129
346
|
const before = {};
|
|
130
347
|
const after = {};
|
|
131
348
|
for (const s of marked) {
|
|
132
|
-
|
|
133
|
-
|
|
349
|
+
const counts = countLines(source, s);
|
|
350
|
+
before[s.line] = counts.before.concat([s.indent + "await page.__kxMark('before-" + s.line + "');"]);
|
|
351
|
+
after[s.line] = [s.indent + "await page.__kxMark('after-" + s.line + "');"].concat(counts.after);
|
|
134
352
|
}
|
|
135
353
|
return probe.probeSource(source, { edits: edits, before: before, after: after });
|
|
136
354
|
}
|
|
@@ -142,11 +360,46 @@ function doubleSource(source, steps, step) {
|
|
|
142
360
|
function twiceSource(source, step, carrier) {
|
|
143
361
|
const before = {};
|
|
144
362
|
const after = {};
|
|
145
|
-
|
|
146
|
-
|
|
363
|
+
const counts = countLines(source, step);
|
|
364
|
+
before[step.line] = counts.before.concat([step.indent + 'await page.__kxTwice(' + JSON.stringify(carrier) + ');', step.indent + "await page.__kxMark('before-" + step.line + "');"]);
|
|
365
|
+
after[step.line] = [step.indent + "await page.__kxMark('after-" + step.line + "');"].concat(counts.after);
|
|
147
366
|
return probe.probeSource(source, { before: before, after: after });
|
|
148
367
|
}
|
|
149
368
|
|
|
369
|
+
/**
|
|
370
|
+
* What the page says a run kept at a step: for each value typed for it, how
|
|
371
|
+
* many more times it is on the page opened again after the step than before
|
|
372
|
+
* it. Null when it was not counted.
|
|
373
|
+
*/
|
|
374
|
+
function addedOnPage(flow, step, result) {
|
|
375
|
+
const typed = typedFor(flow.source, saveSteps(flow.source), step);
|
|
376
|
+
const seen = (name) => (eventsOf(result).find((e) => e.kind === 'seen' && e.name === name + '-' + step.line) || {}).counts;
|
|
377
|
+
const before = seen('before');
|
|
378
|
+
const fresh = seen('fresh');
|
|
379
|
+
if (!typed.length || !before || !fresh) return null;
|
|
380
|
+
return typed.map((t, i) => ({ text: t, added: fresh[i] - before[i], before: before[i], fresh: fresh[i] }));
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Two saves answered with a redirect name no record: whether the page shows
|
|
385
|
+
* more of what was typed than a single click adds. { more, text, once, now }
|
|
386
|
+
* or null when the page does not say - nothing was counted, or what a single
|
|
387
|
+
* click saved is not on it.
|
|
388
|
+
*/
|
|
389
|
+
function pageSays(flow, step, single, other) {
|
|
390
|
+
const one = addedOnPage(flow, step, single);
|
|
391
|
+
const two = addedOnPage(flow, step, other);
|
|
392
|
+
if (!one || !two) return null;
|
|
393
|
+
const shown = one.map((x, i) => i).filter((i) => one[i].added >= 1);
|
|
394
|
+
if (!shown.length) return null;
|
|
395
|
+
const more = shown.find((i) => two[i].added > one[i].added);
|
|
396
|
+
const i = more === undefined ? shown[0] : more;
|
|
397
|
+
return { more: more !== undefined, text: one[i].text, once: one[i].added, now: two[i].added };
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const redirectOnly = (answers) => answers.every((w) => w.answer == null && w.status >= 300 && w.status < 400);
|
|
401
|
+
const timesWord = (n) => (n === 1 ? 'once' : n + ' times');
|
|
402
|
+
|
|
150
403
|
/**
|
|
151
404
|
* The recording as it is, at phone size, with every button on the pages it
|
|
152
405
|
* went to clicked once it has finished - or stopped. In an afterEach, so a
|
|
@@ -169,13 +422,29 @@ function phoneSource(source) {
|
|
|
169
422
|
* write was blocked, so what it would have done is said as not seen.
|
|
170
423
|
* Returns { outcome: 'found' | 'held' | 'not tested', finding?, why, unclicked }.
|
|
171
424
|
*/
|
|
172
|
-
function judgePhone(flow, result) {
|
|
425
|
+
function judgePhone(flow, result, desktopErrors) {
|
|
173
426
|
if (!result) return { outcome: 'not tested', why: 'this run did not finish' };
|
|
174
427
|
if (limited(result)) return { outcome: 'not tested', why: 'your app said "too many requests" (429)' };
|
|
175
428
|
const sweep = eventsOf(result).find((e) => e.kind === 'sweep');
|
|
176
429
|
if (!sweep) return { outcome: 'not tested', why: 'the buttons could not be clicked' };
|
|
177
430
|
if (!sweep.pages.length) return { outcome: 'not tested', why: 'none of the pages it went to could be opened again' + (sweep.failed ? ': ' + sweep.failed : '') };
|
|
178
431
|
const atLoad = new Set((sweep.onLoad || []).map((e) => e.text));
|
|
432
|
+
// A page that throws as soon as it loads at phone size, and not at desktop
|
|
433
|
+
// size, is broken on a phone before anyone clicks. Found on blind round
|
|
434
|
+
// 20: a menu script reached for a button only the desktop layout has, at
|
|
435
|
+
// <= 600px, on every page - and was left out as "already logged at load".
|
|
436
|
+
// Only with the desktop runs to compare: without them, an error at load
|
|
437
|
+
// may be the page's everywhere.
|
|
438
|
+
const phoneOnly = desktopErrors ? (sweep.onLoad || []).filter((e) => e.type === 'page' && !desktopErrors.has(e.text)) : [];
|
|
439
|
+
const loadFinding = phoneOnly.length ? {
|
|
440
|
+
flow: flow.title, where: flow.file, kind: 'phone', type: 'phone-load', severity: 'MEDIUM', status: 'confirmed',
|
|
441
|
+
headline: 'At phone size, ' + phoneOnly[0].path + ' throws an error as soon as it loads - it does not at desktop size.',
|
|
442
|
+
proof: [
|
|
443
|
+
'Flow: ' + flow.title + ' (' + flow.file + '), its pages opened at ' + PHONE.width + ' by ' + PHONE.height + '.',
|
|
444
|
+
'The page threw when it loaded, before any click: ' + phoneOnly[0].text,
|
|
445
|
+
].concat(Array.from(new Set(phoneOnly.slice(1).map((e) => e.path))).slice(0, 3).map((p) => 'The same on ' + p + '.'),
|
|
446
|
+
['The same pages replayed at desktop size threw no such error - so a phone screen is what breaks it (often a script that looks for something only one layout has).']),
|
|
447
|
+
} : null;
|
|
179
448
|
const clean = (sweep.clicks || []).filter((c) => !c.blocked.length && !c.failed);
|
|
180
449
|
const wrote = (sweep.clicks || []).filter((c) => c.blocked.length).map((c) => '"' + c.name + '" (' + c.blocked[0] + ')');
|
|
181
450
|
const unclicked = [];
|
|
@@ -183,6 +452,7 @@ function judgePhone(flow, result) {
|
|
|
183
452
|
if (wrote.length) unclicked.push('a click that tried to save was stopped, so what it does was not seen: ' + wrote.slice(0, 5).join(', '));
|
|
184
453
|
const hits = clean.map((c) => Object.assign({}, c, { errors: c.errors.filter((e) => !atLoad.has(e.text)) })).filter((c) => c.errors.length);
|
|
185
454
|
if (!hits.length) {
|
|
455
|
+
if (loadFinding) return { outcome: 'found', unclicked: unclicked, finding: loadFinding };
|
|
186
456
|
return { outcome: clean.length ? 'held' : 'not tested', unclicked: unclicked,
|
|
187
457
|
why: clean.length ? 'at phone size, ' + clean.length + (clean.length === 1 ? ' button was' : ' buttons were') + ' clicked on ' +
|
|
188
458
|
sweep.pages.length + (sweep.pages.length === 1 ? ' page' : ' pages') + ', and none made the page log an error'
|
|
@@ -195,6 +465,7 @@ function judgePhone(flow, result) {
|
|
|
195
465
|
return {
|
|
196
466
|
outcome: 'found',
|
|
197
467
|
unclicked: unclicked,
|
|
468
|
+
loadFinding: loadFinding,
|
|
198
469
|
finding: {
|
|
199
470
|
flow: flow.title, where: flow.file, kind: 'phone', type: 'phone',
|
|
200
471
|
severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
|
|
@@ -210,6 +481,12 @@ function judgePhone(flow, result) {
|
|
|
210
481
|
};
|
|
211
482
|
}
|
|
212
483
|
|
|
484
|
+
/** A recording made at phone size: codegen wrote a narrow viewport into it. */
|
|
485
|
+
function isPhoneRecording(source) {
|
|
486
|
+
const m = /viewport\s*:\s*\{\s*width\s*:\s*(\d+)/.exec(String(source || ''));
|
|
487
|
+
return Boolean(m) && Number(m[1]) < 768;
|
|
488
|
+
}
|
|
489
|
+
|
|
213
490
|
/** The writes a run made between the marks around one step. */
|
|
214
491
|
function windowWrites(result, line) {
|
|
215
492
|
const events = eventsOf(result);
|
|
@@ -236,7 +513,7 @@ function show(value) {
|
|
|
236
513
|
}
|
|
237
514
|
|
|
238
515
|
function words(w) {
|
|
239
|
-
return
|
|
516
|
+
return probe.writeWords(w);
|
|
240
517
|
}
|
|
241
518
|
|
|
242
519
|
/**
|
|
@@ -246,6 +523,14 @@ function words(w) {
|
|
|
246
523
|
*/
|
|
247
524
|
function recordIds(w) {
|
|
248
525
|
const answer = w && w.answer;
|
|
526
|
+
// A form's redirect names the record it made in the address it sends the
|
|
527
|
+
// browser to (POST /projects -> /projects/13): a part of that address that
|
|
528
|
+
// looks like an id and was not in the address the form posted to.
|
|
529
|
+
if (answer == null && w && w.to && w.status >= 300 && w.status < 400) {
|
|
530
|
+
const posted = String(w.path || '').split('/');
|
|
531
|
+
return String(w.to).split(/[?#]/)[0].split('/').filter((p) => p && !posted.includes(p) &&
|
|
532
|
+
(/^\d+$/.test(p) || /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(p) || /^[0-9a-f]{24}$/i.test(p))).slice(-1);
|
|
533
|
+
}
|
|
249
534
|
if (answer == null) return [];
|
|
250
535
|
const rows = Array.isArray(answer) ? answer : answer && typeof answer === 'object' && answer.data !== undefined
|
|
251
536
|
? [].concat(answer.data) : [answer];
|
|
@@ -286,7 +571,9 @@ function singleClickWords(writes) {
|
|
|
286
571
|
|
|
287
572
|
/** "Create work order" -> "work order": what the step makes, for the headline. */
|
|
288
573
|
function thingOf(name) {
|
|
289
|
-
|
|
574
|
+
// An arrow or icon after the words is not part of the thing (blind round
|
|
575
|
+
// 20: "Add note →" made "2 note →s").
|
|
576
|
+
const rest = String(name || '').replace(/^[^\p{L}\p{N}]*(add|create|new|save|submit|post|upload|send|register)\b\s*/iu, '').replace(/[^\p{L}\p{N}]+$/u, '').trim();
|
|
290
577
|
return rest || 'it';
|
|
291
578
|
}
|
|
292
579
|
|
|
@@ -374,26 +661,39 @@ function judgeDouble(flow, step, single, double) {
|
|
|
374
661
|
// The server naming one record twice saved it once and answered twice:
|
|
375
662
|
// that is the safe way to handle a double-click, not a duplicate.
|
|
376
663
|
const ids = answers.map(recordIds);
|
|
377
|
-
const named = ids.every((list) => list.length);
|
|
378
664
|
const distinct = Array.from(new Set([].concat.apply([], ids)));
|
|
665
|
+
// A redirect may name the page a record belongs to (POST /comments ->
|
|
666
|
+
// /tasks/5), not the record: the same address twice says nothing, two
|
|
667
|
+
// addresses say two records.
|
|
668
|
+
const named = ids.every((list) => list.length) && !(redirectOnly(answers) && distinct.length < 2);
|
|
379
669
|
if (named && distinct.length === 1) {
|
|
380
670
|
return { outcome: 'held', why: 'the server answered both clicks with the same record (id ' + distinct[0] + ') - it was saved once' };
|
|
381
671
|
}
|
|
672
|
+
// A form's redirect names no record: the page opened again decides.
|
|
673
|
+
const byPage = !named && redirectOnly(answers);
|
|
674
|
+
const page = byPage ? pageSays(flow, step, single, double) : null;
|
|
675
|
+
if (byPage && page && !page.more) {
|
|
676
|
+
return { outcome: 'held', why: 'both clicks were answered with a redirect, and the page opened again shows "' + page.text + '" added ' + timesWord(page.now) + ', as after one click - it was kept once' };
|
|
677
|
+
}
|
|
678
|
+
const unsure = byPage && !page;
|
|
382
679
|
const same = answers.length >= 2 && JSON.stringify(answers[0].data) === JSON.stringify(answers[1].data);
|
|
383
680
|
const thing = thingOf(step.name);
|
|
384
681
|
return {
|
|
385
682
|
outcome: 'duplicate',
|
|
386
683
|
finding: {
|
|
387
684
|
flow: flow.title, where: flow.file + ':' + step.line, kind: 'double', type: 'double', step: step.name,
|
|
388
|
-
severity: 'HIGH', status: 'confirmed',
|
|
389
|
-
headline: 'Double-clicking "' + step.name + '" saves it twice.',
|
|
685
|
+
severity: unsure ? 'LOW' : 'HIGH', status: unsure ? 'verification required' : 'confirmed',
|
|
686
|
+
headline: unsure ? 'Double-clicking "' + step.name + '" was answered as two saves - check whether it keeps two.' : 'Double-clicking "' + step.name + '" saves it twice.',
|
|
390
687
|
proof: [
|
|
391
688
|
'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
|
|
392
689
|
'One click on "' + step.name + '": ' + before + ' x ' + route.split(' ')[0] + ' ' + (answers[0].host || '') + answers[0].path + ' answered ' + (one[route] || []).map((w) => w.status).join(', ') + '.',
|
|
393
|
-
'A quick double-click: ' + answers.length + ' x, answered ' + answers.map((w) => w.status).join(', ') + ' - ' +
|
|
690
|
+
'A quick double-click: ' + answers.length + ' x, answered ' + answers.map((w) => w.status).join(', ') + ' - ' +
|
|
691
|
+
(unsure ? 'both taken; whether two were kept is not shown' : answers.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant') + '.',
|
|
394
692
|
].concat(answers.slice(0, 2).map((w, i) => 'Answer ' + (i + 1) + ': ' + words(w) + ', sent ' + show(w.data !== undefined ? w.data : w.text))).concat(
|
|
395
693
|
same ? ['Both requests carried the same data: the same thing, saved twice.'] : [],
|
|
396
694
|
named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
|
|
695
|
+
: page ? ['The page opened again after the double-click shows "' + page.text + '" added ' + timesWord(page.now) + '; after one click, ' + timesWord(page.once) + '.']
|
|
696
|
+
: unsure ? ['Both were answered with a redirect, which names no record, and the page does not show what was saved - so whether two were kept is not proven.']
|
|
397
697
|
: ['The server\'s answers do not say which record each one made, so the proof is the ' + answers.length + ' successful answers.']),
|
|
398
698
|
},
|
|
399
699
|
};
|
|
@@ -422,29 +722,65 @@ function judgeTwice(flow, step, single, twice, pageHeld) {
|
|
|
422
722
|
const route = routeOf(copy);
|
|
423
723
|
const pair = both.filter((w) => routeOf(w) === route && w.method === copy.method && !w.signIn);
|
|
424
724
|
const okOnes = pair.filter(ok);
|
|
725
|
+
// A server error is not a refusal: the server failed on two of the same
|
|
726
|
+
// save at once, where one alone was answered fine. Found on blind round
|
|
727
|
+
// 20: a note numbered MAX()+1 in two steps answered 201 and 500, and was
|
|
728
|
+
// said as "the other was refused".
|
|
729
|
+
const crashed = pair.find((w) => w.status >= 500);
|
|
730
|
+
const fine = (once || []).find((w) => routeOf(w) === route && ok(w));
|
|
731
|
+
if (crashed && fine) {
|
|
732
|
+
const thing = thingOf(step.name);
|
|
733
|
+
return {
|
|
734
|
+
outcome: 'duplicate',
|
|
735
|
+
finding: {
|
|
736
|
+
flow: flow.title, where: flow.file + ':' + step.line, kind: 'race', type: 'race-error', step: step.name,
|
|
737
|
+
severity: 'MEDIUM', status: 'confirmed',
|
|
738
|
+
headline: 'Sent twice at the same moment, "' + step.name + '" makes your server fail - one of the two was answered ' + crashed.status + '.',
|
|
739
|
+
proof: [
|
|
740
|
+
'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
|
|
741
|
+
'One click on "' + step.name + '" alone: ' + words(fine) + '.',
|
|
742
|
+
'That request and a copy of it - same body, same sign-in - sent at the same moment: answered ' + pair.map((w) => w.status).join(', ') + '.',
|
|
743
|
+
'A ' + crashed.status + ' is your server failing, not refusing: ' + (okOnes.length ? okOnes.length + ' ' + thing + (okOnes.length === 1 ? ' was' : 's were') + ' saved, and the other request crashed.' : 'neither was saved.'),
|
|
744
|
+
],
|
|
745
|
+
},
|
|
746
|
+
};
|
|
747
|
+
}
|
|
425
748
|
if (okOnes.length < 2) {
|
|
426
749
|
const refused = pair.find((w) => !ok(w));
|
|
427
750
|
return { outcome: 'held', why: 'sent twice at the same moment, it was saved once' + (refused ? ' - the other was refused (' + words(refused) + ')' : '') };
|
|
428
751
|
}
|
|
429
752
|
const ids = okOnes.map(recordIds);
|
|
430
|
-
const named = ids.every((list) => list.length);
|
|
431
753
|
const distinct = Array.from(new Set([].concat.apply([], ids)));
|
|
754
|
+
// The same redirect twice may only name the page both belong to.
|
|
755
|
+
const named = ids.every((list) => list.length) && !(redirectOnly(okOnes) && distinct.length < 2);
|
|
432
756
|
if (named && distinct.length === 1) {
|
|
433
757
|
return { outcome: 'held', why: 'sent twice at the same moment, the server answered both with the same record (id ' + distinct[0] + ') - it was saved once' };
|
|
434
758
|
}
|
|
759
|
+
// A form's redirect names no record: the page opened again decides.
|
|
760
|
+
const byPage = !named && redirectOnly(okOnes);
|
|
761
|
+
const page = byPage ? pageSays(flow, step, single, twice) : null;
|
|
762
|
+
if (byPage && page && !page.more) {
|
|
763
|
+
return { outcome: 'held', why: 'sent twice at the same moment, both were answered with a redirect, and the page opened again shows "' + page.text + '" added ' + timesWord(page.now) + ', as after one click - it was kept once' };
|
|
764
|
+
}
|
|
765
|
+
const unsure = byPage && !page;
|
|
435
766
|
const thing = thingOf(step.name);
|
|
436
767
|
return {
|
|
437
768
|
outcome: 'duplicate',
|
|
438
769
|
finding: {
|
|
439
770
|
flow: flow.title, where: flow.file + ':' + step.line, kind: 'race', type: 'race', step: step.name,
|
|
440
|
-
severity: 'MEDIUM', status: 'confirmed',
|
|
441
|
-
headline: 'Sent twice at the same moment, "' + step.name + '" saves
|
|
771
|
+
severity: unsure ? 'LOW' : 'MEDIUM', status: unsure ? 'verification required' : 'confirmed',
|
|
772
|
+
headline: unsure ? 'Sent twice at the same moment, "' + step.name + '" was answered as two saves - check whether your server keeps two.'
|
|
773
|
+
: 'Sent twice at the same moment, "' + step.name + '" saves twice - your server lets a repeated save through.',
|
|
442
774
|
proof: [
|
|
443
775
|
'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
|
|
444
776
|
'One click on "' + step.name + '" sends ' + copy.method + ' ' + (copy.host || '') + copy.path + '.',
|
|
777
|
+
// Not proven two: said as two taken, never as "2 settings created"
|
|
778
|
+
// for a save that may only change one record (blind round 21).
|
|
445
779
|
'That request and a copy of it - same body, same sign-in - sent at the same moment: answered ' + okOnes.map((w) => w.status).join(', ') + ' - ' +
|
|
446
|
-
okOnes.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant.',
|
|
780
|
+
(unsure ? 'both taken; whether two were kept is not shown' : okOnes.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant') + '.',
|
|
447
781
|
].concat(named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
|
|
782
|
+
: page ? ['The page opened again afterwards shows "' + page.text + '" added ' + timesWord(page.now) + '; after one click, ' + timesWord(page.once) + '.']
|
|
783
|
+
: unsure ? ['Both were answered with a redirect, which names no record, and the page does not show what was saved - so whether two were kept is not proven.']
|
|
448
784
|
: ['The server\'s answers do not say which record each one made, so the proof is the ' + okOnes.length + ' successful answers.'],
|
|
449
785
|
pageHeld ? ['A double-click on the page saved it once, so the page guards against this and the server does not. A slow network, a retry or a second tab sends the same request twice.'] : []),
|
|
450
786
|
},
|
|
@@ -517,6 +853,9 @@ function fixPrompt(f) {
|
|
|
517
853
|
double: 'Make saving safe against a double-click: disable the button while the save is in flight, and make the server ignore a repeat of the same request (an idempotency key, or a unique constraint on what makes it the same).',
|
|
518
854
|
race: 'Make the server refuse a repeat of the same save: send an idempotency key with each save and keep the first answer for it, or add a unique constraint on what makes two the same. Disabling the button is not enough on its own.',
|
|
519
855
|
phone: 'Make that button work at phone size without an error: find what the click runs at a narrow screen (a menu, a drawer, a panel that is hidden or not there yet) and handle it. Check it at 390 by 844.',
|
|
856
|
+
'race-error': 'Find why two of the same save at once make the server fail: look for a value that is read and then written in two steps (a next number from MAX()+1, a check-then-insert) and make it one step - a sequence or identity column, INSERT ... ON CONFLICT, or a transaction that locks the row. When a unique rule refuses a repeat, answer 409 with a message, never 500.',
|
|
857
|
+
lost: 'Find why this field is not saved by this form: check that the server reads it from the request and writes it in the update (an empty value too - clearing a field must clear it, not keep the old value). Do not change any other field.',
|
|
858
|
+
changed: 'Check whether the value saved for this field is what the person typed. If the app changes it on purpose (rounding, a format), say so on the form; otherwise save it as typed.',
|
|
520
859
|
late: 'After a save succeeds, update what the page shows - add the new item to the list from the server\'s answer, or load the list again - so the person sees it without opening the page again. Otherwise they think it did not save, and save it again.',
|
|
521
860
|
}[f.type];
|
|
522
861
|
return [
|
|
@@ -540,8 +879,15 @@ async function run(root, flows, options) {
|
|
|
540
879
|
const opts = options || {};
|
|
541
880
|
const say = typeof opts.say === 'function' ? opts.say : () => {};
|
|
542
881
|
const budget = Math.min(probe.MAX_RUNS, opts.budget || probe.MAX_RUNS);
|
|
882
|
+
// Many flows: two plain runs each, not three - a failure next to a pass is
|
|
883
|
+
// seen in two - so the cap goes on the saves (twice each: a double-click
|
|
884
|
+
// and two at once) rather than on a third look. Found on blind round 20:
|
|
885
|
+
// twelve recordings, and the invitation and appointment saves were over
|
|
886
|
+
// the cap, never sent twice.
|
|
887
|
+
const savingCost = flows.reduce((n, f) => n + (saveSteps(f.source).length ? ROUNDS + 2 * saveSteps(f.source).length : 0), 0);
|
|
543
888
|
const rounds = opts.only && opts.only.kind === 'flaky' ? RECHECK_ROUNDS
|
|
544
|
-
: opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race' || opts.only.kind === 'phone') ? 1
|
|
889
|
+
: opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race' || opts.only.kind === 'phone') ? 1
|
|
890
|
+
: !opts.only && savingCost > budget * 0.75 ? 2 : ROUNDS;
|
|
545
891
|
let stopped = false;
|
|
546
892
|
let current = null;
|
|
547
893
|
const control = {
|
|
@@ -557,7 +903,7 @@ async function run(root, flows, options) {
|
|
|
557
903
|
}
|
|
558
904
|
const report = {
|
|
559
905
|
check: 'reliability', startedAt: new Date().toISOString(), endedAt: null,
|
|
560
|
-
stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [],
|
|
906
|
+
stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [], writeHostsFrom: 'page+vendor',
|
|
561
907
|
};
|
|
562
908
|
|
|
563
909
|
const runWith = async (probes, labels) => {
|
|
@@ -573,7 +919,10 @@ async function run(root, flows, options) {
|
|
|
573
919
|
// least their saving steps once.
|
|
574
920
|
const chosen = [];
|
|
575
921
|
let planned = 0;
|
|
576
|
-
|
|
922
|
+
// Flows that save first: a flow that only looks has less to find, and
|
|
923
|
+
// takes its place in the cap after them.
|
|
924
|
+
const ordered = flows.filter((f) => saveSteps(f.source).length).concat(flows.filter((f) => !saveSteps(f.source).length));
|
|
925
|
+
for (const f of ordered) {
|
|
577
926
|
if (opts.only && f.file !== opts.only.file) continue;
|
|
578
927
|
if (replay.findReplayRisks(f.source).some((r) => r.kind === 'signup')) {
|
|
579
928
|
report.flows.push({ title: f.title, file: f.file, state: 'skipped', why: 'it creates an account, and this check never creates accounts' });
|
|
@@ -737,7 +1086,13 @@ async function run(root, flows, options) {
|
|
|
737
1086
|
// write blocked. Within what is left of the cap.
|
|
738
1087
|
if (!(opts.only && opts.only.kind !== 'phone')) {
|
|
739
1088
|
const phones = [];
|
|
740
|
-
|
|
1089
|
+
// At most PHONE_FLOWS: the pages repeat from flow to flow, and the runs
|
|
1090
|
+
// are better spent on saves. Phone recordings first.
|
|
1091
|
+
const phoneFlows = opts.only ? chosen : chosen.filter((f) => isPhoneRecording(f.source)).concat(chosen.filter((f) => !isPhoneRecording(f.source))).slice(0, PHONE_FLOWS);
|
|
1092
|
+
for (const f of chosen.filter((x) => !phoneFlows.includes(x))) {
|
|
1093
|
+
report.notTested.push({ flow: f.title, where: f.file, kind: 'phone', why: 'only ' + PHONE_FLOWS + ' flows are replayed at phone size in one press - their pages are mostly the same' });
|
|
1094
|
+
}
|
|
1095
|
+
for (const f of phoneFlows) {
|
|
741
1096
|
const label = { flow: f.title, where: f.file, kind: 'phone' };
|
|
742
1097
|
if (stopped || report.runsDone + phones.length + 1 > budget) {
|
|
743
1098
|
report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it ran' : 'over the limit of ' + budget + ' runs' }));
|
|
@@ -754,21 +1109,32 @@ async function run(root, flows, options) {
|
|
|
754
1109
|
// problem, however many flows pass by it (blind round 17: one menu
|
|
755
1110
|
// button, said five times).
|
|
756
1111
|
const samePhone = new Map();
|
|
1112
|
+
// What the pages throw at desktop size: every page error in the plain
|
|
1113
|
+
// runs of flows recorded at desktop size (a phone recording replays at
|
|
1114
|
+
// phone size, so it says nothing about the desktop).
|
|
1115
|
+
const atDesktop = chosen.filter((f) => !isPhoneRecording(f.source));
|
|
1116
|
+
const desktopErrors = atDesktop.length ? new Set([].concat.apply([], atDesktop.map((f) => [].concat.apply([], f.runs.map((r) => eventsOf(r).filter((e) => e.kind === 'error').map((e) => e.text)))))) : null;
|
|
757
1117
|
for (const p of phones) {
|
|
758
|
-
const v = judgePhone(p.flow, swept[p.id]);
|
|
1118
|
+
const v = judgePhone(p.flow, swept[p.id], desktopErrors);
|
|
759
1119
|
const label = { flow: p.flow.title, where: p.flow.file, kind: 'phone' };
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
1120
|
+
let reported = false;
|
|
1121
|
+
for (const f of [v.finding, v.loadFinding].filter(Boolean)) {
|
|
1122
|
+
// One error on one page is one problem, however many flows pass
|
|
1123
|
+
// by it; one thrown at load is one problem wherever it loads.
|
|
1124
|
+
const same = f.type === 'phone-load' ? 'load|' + f.proof[1] : f.headline + '|' + f.proof[2];
|
|
1125
|
+
if (samePhone.has(same)) {
|
|
1126
|
+
const first = samePhone.get(same);
|
|
1127
|
+
first.alsoIn = (first.alsoIn || 0) + 1;
|
|
1128
|
+
if (first.alsoIn <= 3) first.proof.splice(first.proof.length - 1, 0, 'The same in "' + p.flow.title + '" (' + p.flow.file + ').');
|
|
1129
|
+
if (f === v.finding || !v.finding) report.held.push(Object.assign({}, label, { why: 'at phone size, the same error as in "' + first.flow + '" - said once, there' }));
|
|
1130
|
+
} else {
|
|
1131
|
+
samePhone.set(same, f);
|
|
1132
|
+
report.findings.push(f);
|
|
1133
|
+
}
|
|
1134
|
+
reported = true;
|
|
769
1135
|
}
|
|
770
|
-
|
|
771
|
-
else report.notTested.push(Object.assign(label, { why: !swept[p.id] && stopped ? 'stopped before it finished' : v.why }));
|
|
1136
|
+
if (!reported && v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
|
|
1137
|
+
else if (!reported) report.notTested.push(Object.assign(label, { why: !swept[p.id] && stopped ? 'stopped before it finished' : v.why }));
|
|
772
1138
|
// What was left unclicked is its own line, so a re-check is not held
|
|
773
1139
|
// up by it.
|
|
774
1140
|
for (const why of v.unclicked || []) report.notTested.push(Object.assign({}, label, { kind: 'phone-unclicked', why: why }));
|
|
@@ -791,6 +1157,23 @@ async function run(root, flows, options) {
|
|
|
791
1157
|
}
|
|
792
1158
|
}
|
|
793
1159
|
|
|
1160
|
+
// 4d. What each edit typed, read back from the form opened again after its
|
|
1161
|
+
// save - from the plain runs, so it costs no run of its own.
|
|
1162
|
+
if (!(opts.only && opts.only.kind !== 'kept')) {
|
|
1163
|
+
for (const f of chosen) {
|
|
1164
|
+
for (const k of keepSteps(f.source)) {
|
|
1165
|
+
if (!keptFields(f.source, k).length) continue;
|
|
1166
|
+
if (opts.only && opts.only.line && opts.only.line !== k.line) continue;
|
|
1167
|
+
const v = judgeKept(f, k, f.runs);
|
|
1168
|
+
const label = { flow: f.title, where: f.file + ':' + k.line, kind: 'kept', step: k.name };
|
|
1169
|
+
if (v.outcome === 'not an edit') continue;
|
|
1170
|
+
if (v.findings) for (const x of v.findings) report.findings.push(x);
|
|
1171
|
+
else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
|
|
1172
|
+
else report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it finished' : v.why }));
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
|
|
794
1177
|
// 5. The three runs of each flow. A re-check of a double-click ran its
|
|
795
1178
|
// flow once, only to have a single click to compare with - one run says
|
|
796
1179
|
// nothing about flakiness, so the flow counts as tested when that run
|
|
@@ -807,7 +1190,7 @@ async function run(root, flows, options) {
|
|
|
807
1190
|
}
|
|
808
1191
|
// A re-check of what a save shows needs the runs to have finished, not a
|
|
809
1192
|
// verdict on flakiness.
|
|
810
|
-
if (opts.only && opts.only.kind === 'shown') {
|
|
1193
|
+
if (opts.only && (opts.only.kind === 'shown' || opts.only.kind === 'kept')) {
|
|
811
1194
|
const finished = f.runs.some((r) => r && !limited(r));
|
|
812
1195
|
report.flows.push({ title: f.title, file: f.file, state: finished ? 'tested' : 'not tested', why: finished ? '' : 'none of the runs finished' });
|
|
813
1196
|
continue;
|
|
@@ -832,7 +1215,9 @@ async function run(root, flows, options) {
|
|
|
832
1215
|
|
|
833
1216
|
// What a finding is about, for keeping it across presses: the flow for a
|
|
834
1217
|
// flaky one, the step for a double-click.
|
|
835
|
-
const keyOf = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind;
|
|
1218
|
+
const keyOf = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind + (x.kind === 'kept' && x.field ? '|' + x.field : '');
|
|
1219
|
+
// A step held or not tested says so for every field it read back.
|
|
1220
|
+
const stepKey = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind;
|
|
836
1221
|
|
|
837
1222
|
/**
|
|
838
1223
|
* What a press did not try is not cleared by it: a finding from an earlier
|
|
@@ -842,7 +1227,7 @@ function carryOver(previous, report, root) {
|
|
|
842
1227
|
if (!previous || !Array.isArray(previous.findings)) return report;
|
|
843
1228
|
const tried = new Set(report.held.concat(report.findings).map(keyOf));
|
|
844
1229
|
for (const f of previous.findings) {
|
|
845
|
-
if (tried.has(keyOf(f))) continue;
|
|
1230
|
+
if (tried.has(keyOf(f)) || (f.kind === 'kept' && tried.has(stepKey(f)))) continue;
|
|
846
1231
|
const file = String(f.where).replace(/:\d+$/, '');
|
|
847
1232
|
if (root && !fs.existsSync(path.join(root, file))) continue;
|
|
848
1233
|
const kept = Object.assign({}, f, { carried: f.carried || previous.endedAt || previous.startedAt });
|
|
@@ -876,7 +1261,7 @@ async function recheck(root, flows, finding, options) {
|
|
|
876
1261
|
const unsure = (why) => ({ verdict: 'could not confirm', why: why, newProblems: [], testData: [] });
|
|
877
1262
|
const flow = flows.find((f) => f.file === file);
|
|
878
1263
|
if (!flow) return unsure('the recording ' + file + ' is not there, or cannot be run now');
|
|
879
|
-
if (finding.kind !== 'flaky' && finding.kind !== 'phone' && !saveSteps(flow.source).some((s) => s.line === line)) return unsure('that step is no longer at line ' + line + ' of ' + file);
|
|
1264
|
+
if (finding.kind !== 'flaky' && finding.kind !== 'phone' && !(finding.kind === 'kept' ? keepSteps(flow.source) : saveSteps(flow.source)).some((s) => s.line === line)) return unsure('that step is no longer at line ' + line + ' of ' + file);
|
|
880
1265
|
const report = await run(root, [flow], Object.assign({}, opts, {
|
|
881
1266
|
only: finding.kind === 'flaky' || finding.kind === 'phone' ? { file: file, kind: finding.kind } : { file: file, kind: finding.kind, line: line },
|
|
882
1267
|
}));
|
|
@@ -888,13 +1273,14 @@ async function recheck(root, flows, finding, options) {
|
|
|
888
1273
|
if (report.stopped) return unsure('it was stopped before it finished');
|
|
889
1274
|
const ran = report.flows.find((f) => f.file === file);
|
|
890
1275
|
if (!ran || ran.state !== 'tested') return unsure(ran ? ran.why : 'the flow did not run');
|
|
891
|
-
const left = report.notTested.find((x) => keyOf(x) === keyOf(finding));
|
|
1276
|
+
const left = report.notTested.find((x) => keyOf(x) === keyOf(finding) || (finding.kind === 'kept' && stepKey(x) === stepKey(finding)));
|
|
892
1277
|
if (left) return unsure(left.why);
|
|
893
1278
|
const same = report.findings.find((f) => keyOf(f) === keyOf(finding));
|
|
894
1279
|
const others = report.findings.filter((f) => f !== same && f.status === 'confirmed');
|
|
895
1280
|
const result = same ? { verdict: 'still open', why: same.proof.slice(1).join(' '), newProblems: others }
|
|
896
1281
|
: { verdict: 'fixed', why: finding.kind === 'flaky' ? 'it worked all ' + RECHECK_ROUNDS + ' times'
|
|
897
1282
|
: finding.kind === 'shown' || finding.kind === 'race' || finding.kind === 'phone' ? (report.held.find((x) => keyOf(x) === keyOf(finding)) || {}).why || 'it held'
|
|
1283
|
+
: finding.kind === 'kept' ? 'the form opened again after the save shows what was typed'
|
|
898
1284
|
: 'a double-click saved it once', newProblems: others };
|
|
899
1285
|
const saved = load(root, opts.env);
|
|
900
1286
|
if (saved && result.verdict === 'fixed') {
|
|
@@ -909,6 +1295,7 @@ async function recheck(root, flows, finding, options) {
|
|
|
909
1295
|
function heldLabel(x) {
|
|
910
1296
|
return x.kind === 'flaky' ? x.flow + ' - '
|
|
911
1297
|
: x.kind === 'shown' ? 'After "' + x.step + '" (' + x.where + '), looked for on the page - '
|
|
1298
|
+
: x.kind === 'kept' ? 'After "' + x.step + '" (' + x.where + '), the form read back - '
|
|
912
1299
|
: x.kind === 'race' ? '"' + x.step + '" (' + x.where + ') sent twice at once - '
|
|
913
1300
|
: x.kind === 'phone' || x.kind === 'phone-unclicked' ? 'At phone size, ' + x.flow + ' - '
|
|
914
1301
|
: '"' + x.step + '" (' + x.where + ') double-clicked - ';
|
|
@@ -968,6 +1355,9 @@ module.exports = {
|
|
|
968
1355
|
judgeFlaky: judgeFlaky,
|
|
969
1356
|
judgeDouble: judgeDouble,
|
|
970
1357
|
judgeShown: judgeShown,
|
|
1358
|
+
judgeKept: judgeKept,
|
|
1359
|
+
keepSteps: keepSteps,
|
|
1360
|
+
keptFields: keptFields,
|
|
971
1361
|
judgeTwice: judgeTwice,
|
|
972
1362
|
judgePhone: judgePhone,
|
|
973
1363
|
phoneSource: phoneSource,
|