kryptheon 0.1.46 → 0.1.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,984 @@
1
+ // Reliability: does the app do the same thing every time?
2
+ //
3
+ // Two questions, asked of the person's own recorded flows:
4
+ //
5
+ // 1. Flaky. Each flow is replayed three times. A flow that fails and then
6
+ // works again - with nothing changed - does not work every time, and the
7
+ // failed run is the proof: which step, what it said, and anything the
8
+ // app answered differently in it (a 5xx the good runs did not get).
9
+ // Three failures are not flaky - the flow is broken, and the regression
10
+ // check says so. Works-then-stops-working is said as a thing to check,
11
+ // never as flaky: the first run's own data may be what is in the way.
12
+ //
13
+ // 2. Double-click. Each step that saves (Save, Add, Create, Submit...) is
14
+ // clicked twice, quickly, the way an impatient person does. When the app
15
+ // then saves twice - more successful POSTs to the same place than a
16
+ // single click makes - that is reported, with both answers as proof. A
17
+ // duplicate is run once more before it is called a problem.
18
+ //
19
+ // 3. Two at once. On an app on this computer only, each save a single
20
+ // click makes is sent twice at the same moment - the page's own request
21
+ // and a copy with the same sign-in. A page that disables its button
22
+ // stops a double-click; this asks the server. Two records made is
23
+ // reported, with both answers.
24
+ //
25
+ // 4. At phone size. Each flow is replayed once at 390 by 844, and every
26
+ // button on the pages it went to is clicked, one at a time, with every
27
+ // write blocked - nothing is saved, changed or removed. A click after
28
+ // which the page throws an error is reported; one that only logs an
29
+ // error is a thing to check. Buttons that save, remove or sign out are
30
+ // not clicked, and are named.
31
+ //
32
+ // 5. Shows right away. After each step that saves, what was typed for it
33
+ // is looked for on the page: before the click, right after it, and on
34
+ // the same address opened again. Found only when the page is opened
35
+ // again, it was saved but the page did not show it - a person thinks it
36
+ // did not save, and saves it again. A page that does not show it even
37
+ // when opened again says nothing either way, and is said as not tested.
38
+ //
39
+ // Slower networks are not tried in this release.
40
+ //
41
+ // Every run adds data to the app (the flows save what they were recorded
42
+ // saving), so nothing here starts on its own: kryptheon-probe.js asks first.
43
+
44
+ const fs = require('fs');
45
+ const path = require('path');
46
+ const replay = require('./kryptheon-replay.js');
47
+ const probe = require('./kryptheon-probe.js');
48
+ const edge = require('./kryptheon-edge.js');
49
+
50
+ const ROUNDS = 3;
51
+ // The size a phone recording is made at (bin/kryptheon.js PHONE_SIZE).
52
+ const PHONE = { width: 390, height: 844 };
53
+ const CONFIRM_RUNS = 5;
54
+ // A re-check of a flaky flow needs more than three runs to say anything.
55
+ const RECHECK_ROUNDS = 5;
56
+
57
+ const CLICK = /^(\s*)(await\s+)?(page\s*\..+?)\.click\(\s*\)\s*;?\s*$/;
58
+
59
+ const ok = (w) => w.status >= 200 && w.status < 300;
60
+ const eventsOf = (r) => (r && r.events) || [];
61
+ const writesOf = (r) => eventsOf(r).filter((e) => e.kind === 'write');
62
+ const leftBehind = (w) => ok(w) && w.method !== 'DELETE' && !w.signIn;
63
+ const routeOf = (w) => w.method + ' ' + (w.origin || w.host || '') + w.path;
64
+ const limited = (r) => eventsOf(r).some((e) => e.kind === 'limited');
65
+
66
+ /**
67
+ * The steps that save, one per line: a plain click on a button whose name
68
+ * says it adds or saves something. Not a sign-up - that makes an account -
69
+ * and not one that only opens a form; that is found out at run time, when a
70
+ * single click of it adds no record (singleClickWords says what it did).
71
+ */
72
+ function saveSteps(source) {
73
+ const lines = String(source || '').split('\n');
74
+ const out = [];
75
+ for (const a of replay.readActions(source)) {
76
+ if (a.method !== 'click' || !a.name) continue;
77
+ if (!replay.CREATES.test(a.name.trim()) || replay.SIGNUP.test(a.name)) continue;
78
+ const m = CLICK.exec(lines[a.line - 1].replace(/\r$/, ''));
79
+ if (!m) continue;
80
+ out.push({ line: a.line, indent: m[1], locator: m[3], name: a.name.trim() });
81
+ }
82
+ return out;
83
+ }
84
+
85
+ /**
86
+ * What was typed before a saving step, to look for on the page after it:
87
+ * each literal value filled in since the last step that saved, three
88
+ * characters or more, never the sign-in. Values from .env are not literals.
89
+ */
90
+ function typedFor(source, steps, step) {
91
+ const since = Math.max(0, ...steps.filter((s) => s.line < step.line).map((s) => s.line));
92
+ const lines = String(source || '').split('\n');
93
+ let signInEnd = 0;
94
+ lines.forEach((line, i) => {
95
+ if (/\.fill\(/.test(line) && (replay.isSecretField(replay.nameInLine(line)) || /process\.env\./.test(line))) signInEnd = i + 1;
96
+ });
97
+ const out = [];
98
+ for (const f of edge.fillSteps(source)) {
99
+ if (f.line <= since || f.line >= step.line || f.line <= signInEnd || replay.isSecretField(f.name)) continue;
100
+ const value = String(f.value).trim();
101
+ if (value.length >= 3 && !out.includes(value)) out.push(value);
102
+ }
103
+ return out;
104
+ }
105
+
106
+ /**
107
+ * The plain replay, with a mark before and after each step that saves - and
108
+ * what was typed for it counted on the page before the click, right after it,
109
+ * and on the same address opened again.
110
+ */
111
+ function plainSource(source, steps) {
112
+ const before = {};
113
+ const after = {};
114
+ const all = saveSteps(source);
115
+ for (const s of steps) {
116
+ const typed = typedFor(source, all, s);
117
+ const seen = (name, fresh) => s.indent + 'await page.__kxSeen(' + JSON.stringify(name + '-' + s.line) + ', ' + JSON.stringify(typed) + (fresh ? ', true' : '') + ');';
118
+ before[s.line] = (typed.length ? [seen('before')] : []).concat([s.indent + "await page.__kxMark('before-" + s.line + "');"]);
119
+ after[s.line] = [s.indent + "await page.__kxMark('after-" + s.line + "');"].concat(typed.length ? [seen('after'), seen('fresh', true)] : []);
120
+ }
121
+ return probe.probeSource(source, { before: before, after: after });
122
+ }
123
+
124
+ /** The same, with one step clicked twice. */
125
+ function doubleSource(source, steps, step) {
126
+ const edits = {};
127
+ edits[step.line] = step.indent + 'await ' + step.locator + '.dblclick();';
128
+ const marked = steps.filter((s) => s.line === step.line);
129
+ const before = {};
130
+ const after = {};
131
+ for (const s of marked) {
132
+ before[s.line] = s.indent + "await page.__kxMark('before-" + s.line + "');";
133
+ after[s.line] = s.indent + "await page.__kxMark('after-" + s.line + "');";
134
+ }
135
+ return probe.probeSource(source, { edits: edits, before: before, after: after });
136
+ }
137
+
138
+ /**
139
+ * The plain replay, with one step's save sent twice at the same moment: the
140
+ * request it makes (`carrier`: method, host, path) goes out with a copy.
141
+ */
142
+ function twiceSource(source, step, carrier) {
143
+ const before = {};
144
+ const after = {};
145
+ before[step.line] = [step.indent + 'await page.__kxTwice(' + JSON.stringify(carrier) + ');', step.indent + "await page.__kxMark('before-" + step.line + "');"];
146
+ after[step.line] = step.indent + "await page.__kxMark('after-" + step.line + "');";
147
+ return probe.probeSource(source, { before: before, after: after });
148
+ }
149
+
150
+ /**
151
+ * The recording as it is, at phone size, with every button on the pages it
152
+ * went to clicked once it has finished - or stopped. In an afterEach, so a
153
+ * flow that fails at phone size still has its pages' buttons clicked.
154
+ */
155
+ function phoneSource(source) {
156
+ const lines = String(source || '').split('\n');
157
+ const at = lines.findIndex((l) => /kryptheon-fixture|from\s*['"]kryptheon['"]/.test(l));
158
+ const after = {};
159
+ after[at === -1 ? 1 : at + 1] = [
160
+ 'test.use({ viewport: ' + JSON.stringify(PHONE).replace(/"/g, '') + ' });',
161
+ 'test.afterEach(async ({ page }) => { await page.__kxSweep(); });',
162
+ ];
163
+ return probe.probeSource(source, { after: after });
164
+ }
165
+
166
+ /**
167
+ * What clicking every button at phone size showed. Errors the page already
168
+ * logged when it loaded are not put down to a click; a click that tried to
169
+ * write was blocked, so what it would have done is said as not seen.
170
+ * Returns { outcome: 'found' | 'held' | 'not tested', finding?, why, unclicked }.
171
+ */
172
+ function judgePhone(flow, result) {
173
+ if (!result) return { outcome: 'not tested', why: 'this run did not finish' };
174
+ if (limited(result)) return { outcome: 'not tested', why: 'your app said "too many requests" (429)' };
175
+ const sweep = eventsOf(result).find((e) => e.kind === 'sweep');
176
+ if (!sweep) return { outcome: 'not tested', why: 'the buttons could not be clicked' };
177
+ if (!sweep.pages.length) return { outcome: 'not tested', why: 'none of the pages it went to could be opened again' + (sweep.failed ? ': ' + sweep.failed : '') };
178
+ const atLoad = new Set((sweep.onLoad || []).map((e) => e.text));
179
+ const clean = (sweep.clicks || []).filter((c) => !c.blocked.length && !c.failed);
180
+ const wrote = (sweep.clicks || []).filter((c) => c.blocked.length).map((c) => '"' + c.name + '" (' + c.blocked[0] + ')');
181
+ const unclicked = [];
182
+ if ((sweep.skipped || []).length) unclicked.push('buttons that save, remove or sign out were not clicked: ' + sweep.skipped.map((n) => '"' + n + '"').join(', '));
183
+ if (wrote.length) unclicked.push('a click that tried to save was stopped, so what it does was not seen: ' + wrote.slice(0, 5).join(', '));
184
+ const hits = clean.map((c) => Object.assign({}, c, { errors: c.errors.filter((e) => !atLoad.has(e.text)) })).filter((c) => c.errors.length);
185
+ if (!hits.length) {
186
+ return { outcome: clean.length ? 'held' : 'not tested', unclicked: unclicked,
187
+ why: clean.length ? 'at phone size, ' + clean.length + (clean.length === 1 ? ' button was' : ' buttons were') + ' clicked on ' +
188
+ sweep.pages.length + (sweep.pages.length === 1 ? ' page' : ' pages') + ', and none made the page log an error'
189
+ : 'no button could be clicked at phone size' };
190
+ }
191
+ const thrown = hits.filter((c) => c.errors.some((e) => e.type === 'page'));
192
+ const lead = thrown[0] || hits[0];
193
+ const error = lead.errors.find((e) => e.type === 'page') || lead.errors[0];
194
+ const sure = thrown.length > 0;
195
+ return {
196
+ outcome: 'found',
197
+ unclicked: unclicked,
198
+ finding: {
199
+ flow: flow.title, where: flow.file, kind: 'phone', type: 'phone',
200
+ severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
201
+ headline: sure ? 'At phone size, clicking "' + lead.name + '" on ' + lead.path + ' makes the page throw an error.'
202
+ : 'At phone size, clicking "' + lead.name + '" on ' + lead.path + ' makes the page log an error - check it.',
203
+ proof: [
204
+ 'Flow: ' + flow.title + ' (' + flow.file + '), replayed at ' + PHONE.width + ' by ' + PHONE.height + '.',
205
+ 'Clicked "' + lead.name + '" on ' + lead.path + ', with every save blocked so nothing was changed.',
206
+ (error.type === 'page' ? 'The page threw: ' : 'The page logged: ') + error.text,
207
+ ].concat(hits.filter((c) => c !== lead).slice(0, 2).map((c) => 'Also "' + c.name + '" on ' + c.path + ': ' + c.errors[0].text),
208
+ ['It did not log that when the page loaded, before the click.']),
209
+ },
210
+ };
211
+ }
212
+
213
+ /** The writes a run made between the marks around one step. */
214
+ function windowWrites(result, line) {
215
+ const events = eventsOf(result);
216
+ const start = events.findIndex((e) => e.kind === 'mark' && e.name === 'before-' + line);
217
+ const end = events.findIndex((e) => e.kind === 'mark' && e.name === 'after-' + line);
218
+ if (start === -1 || end === -1 || end < start) return null;
219
+ return events.slice(start + 1, end).filter((e) => e.kind === 'write');
220
+ }
221
+
222
+ /** Successful POSTs that are not the sign-in, counted by where they went. */
223
+ function created(writes) {
224
+ const out = {};
225
+ for (const w of writes || []) {
226
+ if (w.method !== 'POST' || !ok(w) || w.signIn) continue;
227
+ (out[routeOf(w)] = out[routeOf(w)] || []).push(w);
228
+ }
229
+ return out;
230
+ }
231
+
232
+ function show(value) {
233
+ if (value === undefined) return '(no body kept)';
234
+ const s = typeof value === 'string' ? value : JSON.stringify(value);
235
+ return s.length > 120 ? s.slice(0, 100) + '... (' + s.length + ' characters)' : s;
236
+ }
237
+
238
+ function words(w) {
239
+ return w.method + ' ' + (w.host || '') + w.path + ' -> ' + w.status;
240
+ }
241
+
242
+ /**
243
+ * The records a server's answer names: the id (or uuid, or _id) of what it
244
+ * saved, from the answer itself, an array of rows, or a { data } wrapper.
245
+ * Empty when the answer says nothing about what was made.
246
+ */
247
+ function recordIds(w) {
248
+ const answer = w && w.answer;
249
+ if (answer == null) return [];
250
+ const rows = Array.isArray(answer) ? answer : answer && typeof answer === 'object' && answer.data !== undefined
251
+ ? [].concat(answer.data) : [answer];
252
+ const out = [];
253
+ for (const row of rows) {
254
+ if (!row || typeof row !== 'object') continue;
255
+ for (const k of ['id', 'uuid', '_id']) {
256
+ if (typeof row[k] === 'string' || typeof row[k] === 'number') {
257
+ out.push(String(row[k]));
258
+ break;
259
+ }
260
+ }
261
+ }
262
+ return out;
263
+ }
264
+
265
+ /**
266
+ * What a single click of a step did, in words, when it added no record:
267
+ * nothing at all, or a write that adds nothing - a change to a record that
268
+ * is already there (PUT, PATCH), a removal (DELETE), a write the app refused.
269
+ * Every write method is counted. Found on blind round 17: a step that sent a
270
+ * PUT was said to save nothing.
271
+ */
272
+ function singleClickWords(writes) {
273
+ const sent = (writes || []).filter((w) => !w.signIn);
274
+ if (!sent.length) return 'a single click of it saves nothing by itself';
275
+ const what = [];
276
+ for (const w of sent) {
277
+ const said = !ok(w) ? 'sends ' + words(w) + ', which your app refused'
278
+ : w.method === 'DELETE' ? 'removes a record (' + words(w) + ')'
279
+ : w.method === 'POST' ? 'sends ' + words(w)
280
+ : 'changes a record that is already there (' + words(w) + ')';
281
+ if (!what.includes(said)) what.push(said);
282
+ }
283
+ return 'a single click of it ' + what.slice(0, 2).join(' and ') + (what.length > 2 ? ' and ' + (what.length - 2) + ' more' : '') +
284
+ ' - it adds no record, so there is none to save twice';
285
+ }
286
+
287
+ /** "Create work order" -> "work order": what the step makes, for the headline. */
288
+ function thingOf(name) {
289
+ const rest = String(name || '').replace(/^[^\p{L}\p{N}]*(add|create|new|save|submit|post|upload|send|register)\b\s*/iu, '').trim();
290
+ return rest || 'it';
291
+ }
292
+
293
+ // --- judging -------------------------------------------------------------
294
+
295
+ /**
296
+ * One flow's three runs, in the order they ran. Returns { outcome, finding?,
297
+ * why } - outcome 'flaky', 'stopped-working', 'held', 'broken' or 'not tested'.
298
+ */
299
+ function judgeFlaky(flow, runs) {
300
+ const counted = runs.filter((r) => r && !limited(r));
301
+ if (counted.length < 2) {
302
+ return { outcome: 'not tested', why: runs.some(limited) ? 'your app said "too many requests" (429), so too few runs counted' : 'too few of the runs finished' };
303
+ }
304
+ const passed = counted.map((r) => r.status === 'passed');
305
+ if (passed.every(Boolean)) return { outcome: 'held', why: 'it worked all ' + counted.length + ' times' };
306
+ if (!passed.some(Boolean)) return { outcome: 'broken', why: 'it failed every time - that is a broken flow, which the regression check reports' };
307
+ const failedRun = counted[passed.indexOf(false)];
308
+ const goodRuns = counted.filter((r) => r.status === 'passed');
309
+ // Proof the app itself answered differently in the run that failed.
310
+ const seenGood = new Set();
311
+ for (const r of goodRuns) for (const e of eventsOf(r)) if (e.kind === 'server' || e.kind === 'error') seenGood.add(e.kind + ' ' + (e.text || routeOf(e)));
312
+ const onlyBad = eventsOf(failedRun).filter((e) => (e.kind === 'server' || e.kind === 'error') && !seenGood.has(e.kind + ' ' + (e.text || routeOf(e))));
313
+ const pattern = counted.map((r, i) => 'run ' + (i + 1) + ' ' + (r.status === 'passed' ? 'worked' : 'failed')).join(', ');
314
+ const where = flow.file + (failedRun.failedLine ? ':' + failedRun.failedLine : '');
315
+ const proof = [
316
+ 'Flow: ' + flow.title + ' (' + flow.file + ')',
317
+ 'Replayed ' + counted.length + ' times, nothing changed in between: ' + pattern + '.',
318
+ 'The run that failed stopped' + (failedRun.failedLine ? ' at line ' + failedRun.failedLine : '') + ': ' + (failedRun.error || failedRun.status),
319
+ ];
320
+ for (const e of onlyBad.slice(0, 3)) {
321
+ proof.push(e.kind === 'server' ? 'In that run only, the server answered: ' + e.method + ' ' + (e.host || '') + e.path + ' -> ' + e.status
322
+ : 'In that run only, the page threw: ' + e.text);
323
+ }
324
+ // A failure followed by a success proves it is not the first run's data in
325
+ // the way. Without one, it is a thing to check.
326
+ const recovered = passed.indexOf(false) < passed.lastIndexOf(true);
327
+ // Only the app's own answer proves the app: a 5xx or a page error that
328
+ // the runs that worked did not get. Without one, a recording that clicked
329
+ // before the page was ready fails in just the same way - so it is said as
330
+ // a thing to check, never as a problem in the app.
331
+ const appEvidence = onlyBad.length > 0;
332
+ if (!recovered) {
333
+ return {
334
+ outcome: 'stopped-working',
335
+ finding: {
336
+ flow: flow.title, where: where, kind: 'flaky', type: 'stopped-working', severity: 'LOW', status: 'verification required',
337
+ headline: 'The flow "' + flow.title + '" worked, then stopped working when replayed again - check whether that is expected.',
338
+ proof: proof.concat(['It never worked again after failing, so the data the first run saved may be what stops it (a name that must be unique, say).']),
339
+ },
340
+ };
341
+ }
342
+ return {
343
+ outcome: 'flaky',
344
+ finding: {
345
+ flow: flow.title, where: where, kind: 'flaky', type: 'flaky',
346
+ severity: appEvidence ? 'MEDIUM' : 'LOW', status: appEvidence ? 'confirmed' : 'verification required',
347
+ headline: appEvidence ? 'The flow "' + flow.title + '" does not work every time.'
348
+ : 'The flow "' + flow.title + '" did not work every time, but your app showed no error - check whether it is the app or the recording\'s timing.',
349
+ proof: proof.concat([appEvidence
350
+ ? 'It failed and then worked again with nothing changed, and the app answered differently in the run that failed - so a person doing the same thing will sometimes see it fail.'
351
+ : 'Nothing your app answered was different in the run that failed: no 5xx, no page error. A step that ran before the page was ready fails the same way, so this is not proof the app is at fault.']),
352
+ },
353
+ };
354
+ }
355
+
356
+ /**
357
+ * A step clicked twice, against the same step clicked once. Returns
358
+ * { outcome, finding?, why } - 'duplicate', 'held', 'not tested'.
359
+ */
360
+ function judgeDouble(flow, step, single, double) {
361
+ if (!double) return { outcome: 'not tested', why: 'this run did not finish' };
362
+ if (limited(double)) return { outcome: 'not tested', why: 'your app said "too many requests" (429)' };
363
+ const once = windowWrites(single, step.line);
364
+ const twice = windowWrites(double, step.line);
365
+ if (!once) return { outcome: 'not tested', why: 'the plain replay did not reach this step' };
366
+ if (!twice) return { outcome: 'not tested', why: 'the double-click run did not get past this step' };
367
+ const one = created(once);
368
+ const two = created(twice);
369
+ for (const route of Object.keys(two)) {
370
+ const before = (one[route] || []).length;
371
+ if (!before) continue; // only what a single click saves is compared
372
+ if (two[route].length <= before) continue;
373
+ const answers = two[route];
374
+ // The server naming one record twice saved it once and answered twice:
375
+ // that is the safe way to handle a double-click, not a duplicate.
376
+ const ids = answers.map(recordIds);
377
+ const named = ids.every((list) => list.length);
378
+ const distinct = Array.from(new Set([].concat.apply([], ids)));
379
+ if (named && distinct.length === 1) {
380
+ return { outcome: 'held', why: 'the server answered both clicks with the same record (id ' + distinct[0] + ') - it was saved once' };
381
+ }
382
+ const same = answers.length >= 2 && JSON.stringify(answers[0].data) === JSON.stringify(answers[1].data);
383
+ const thing = thingOf(step.name);
384
+ return {
385
+ outcome: 'duplicate',
386
+ finding: {
387
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'double', type: 'double', step: step.name,
388
+ severity: 'HIGH', status: 'confirmed',
389
+ headline: 'Double-clicking "' + step.name + '" saves it twice.',
390
+ proof: [
391
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
392
+ 'One click on "' + step.name + '": ' + before + ' x ' + route.split(' ')[0] + ' ' + (answers[0].host || '') + answers[0].path + ' answered ' + (one[route] || []).map((w) => w.status).join(', ') + '.',
393
+ 'A quick double-click: ' + answers.length + ' x, answered ' + answers.map((w) => w.status).join(', ') + ' - ' + answers.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant.',
394
+ ].concat(answers.slice(0, 2).map((w, i) => 'Answer ' + (i + 1) + ': ' + words(w) + ', sent ' + show(w.data !== undefined ? w.data : w.text))).concat(
395
+ same ? ['Both requests carried the same data: the same thing, saved twice.'] : [],
396
+ named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
397
+ : ['The server\'s answers do not say which record each one made, so the proof is the ' + answers.length + ' successful answers.']),
398
+ },
399
+ };
400
+ }
401
+ const posts = Object.keys(one).length;
402
+ return { outcome: 'held', why: posts ? 'a double-click saved it once, as a single click does' : singleClickWords(once) };
403
+ }
404
+
405
+ /**
406
+ * One step's save sent twice at the same moment, against the same step
407
+ * clicked once. Returns { outcome: 'duplicate' | 'held' | 'not tested',
408
+ * finding?, why }. `pageHeld` is what the double-click said when the page
409
+ * itself saved once - the page guards against it; this is the server.
410
+ */
411
+ function judgeTwice(flow, step, single, twice, pageHeld) {
412
+ if (!twice) return { outcome: 'not tested', why: 'this run did not finish' };
413
+ if (limited(twice)) return { outcome: 'not tested', why: 'your app said "too many requests" (429)' };
414
+ const failed = eventsOf(twice).find((e) => e.kind === 'copy-failed');
415
+ if (failed) return { outcome: 'not tested', why: 'the second request could not be sent: ' + failed.why };
416
+ const once = windowWrites(single, step.line);
417
+ const both = windowWrites(twice, step.line);
418
+ if (!once) return { outcome: 'not tested', why: 'the plain replay did not reach this step' };
419
+ if (!both) return { outcome: 'not tested', why: 'the run did not get past this step' };
420
+ const copy = both.find((w) => w.copy);
421
+ if (!copy) return { outcome: 'not tested', why: 'the save was not sent the way it was in the plain replay, so no second one went with it' };
422
+ const route = routeOf(copy);
423
+ const pair = both.filter((w) => routeOf(w) === route && w.method === copy.method && !w.signIn);
424
+ const okOnes = pair.filter(ok);
425
+ if (okOnes.length < 2) {
426
+ const refused = pair.find((w) => !ok(w));
427
+ return { outcome: 'held', why: 'sent twice at the same moment, it was saved once' + (refused ? ' - the other was refused (' + words(refused) + ')' : '') };
428
+ }
429
+ const ids = okOnes.map(recordIds);
430
+ const named = ids.every((list) => list.length);
431
+ const distinct = Array.from(new Set([].concat.apply([], ids)));
432
+ if (named && distinct.length === 1) {
433
+ return { outcome: 'held', why: 'sent twice at the same moment, the server answered both with the same record (id ' + distinct[0] + ') - it was saved once' };
434
+ }
435
+ const thing = thingOf(step.name);
436
+ return {
437
+ outcome: 'duplicate',
438
+ finding: {
439
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'race', type: 'race', step: step.name,
440
+ severity: 'MEDIUM', status: 'confirmed',
441
+ headline: 'Sent twice at the same moment, "' + step.name + '" saves twice - your server lets a repeated save through.',
442
+ proof: [
443
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
444
+ 'One click on "' + step.name + '" sends ' + copy.method + ' ' + (copy.host || '') + copy.path + '.',
445
+ 'That request and a copy of it - same body, same sign-in - sent at the same moment: answered ' + okOnes.map((w) => w.status).join(', ') + ' - ' +
446
+ okOnes.length + ' ' + thing + (/s$/.test(thing) ? '' : 's') + ' created where one was meant.',
447
+ ].concat(named ? ['The server\'s answers name ' + distinct.length + ' different records: ' + distinct.map((id) => 'id ' + id).join(', ') + '.']
448
+ : ['The server\'s answers do not say which record each one made, so the proof is the ' + okOnes.length + ' successful answers.'],
449
+ pageHeld ? ['A double-click on the page saved it once, so the page guards against this and the server does not. A slow network, a retry or a second tab sends the same request twice.'] : []),
450
+ },
451
+ };
452
+ }
453
+
454
+ /**
455
+ * Whether what a step saved showed on the page right away, from the plain
456
+ * runs. Per run: the typed values the save carried, counted before the click,
457
+ * right after it, and on the page opened again. More of one opened again than
458
+ * before, and no more right after, is "late"; more right after is "shown".
459
+ * Returns { outcome: 'late' | 'held' | 'not tested', finding?, why }.
460
+ */
461
+ function judgeShown(flow, step, runs) {
462
+ const typed = typedFor(flow.source, saveSteps(flow.source), step);
463
+ const counted = (runs || []).filter((r) => r && !limited(r));
464
+ const late = [];
465
+ let shown = 0;
466
+ let saved = 0;
467
+ for (const r of counted) {
468
+ const writes = windowWrites(r, step.line);
469
+ const made = writes ? [].concat.apply([], Object.values(created(writes))) : [];
470
+ if (!made.length) continue;
471
+ saved += 1;
472
+ const seen = (name) => (eventsOf(r).find((e) => e.kind === 'seen' && e.name === name + '-' + step.line) || {}).counts;
473
+ const before = seen('before');
474
+ const after = seen('after');
475
+ const fresh = seen('fresh');
476
+ if (!before || !after || !fresh) continue;
477
+ const body = (w) => (w.data !== undefined ? JSON.stringify(w.data) : String(w.text || ''));
478
+ const carried = typed.map((t, i) => ({ text: t, i: i })).filter((x) => made.some((w) => body(w).indexOf(JSON.stringify(x.text).slice(1, -1)) !== -1));
479
+ if (carried.some((x) => after[x.i] > before[x.i])) shown += 1;
480
+ else {
481
+ const later = carried.find((x) => fresh[x.i] > before[x.i]);
482
+ if (later) late.push({ run: counted.indexOf(r) + 1, text: later.text, before: before[later.i], after: after[later.i], fresh: fresh[later.i], write: made[0] });
483
+ }
484
+ }
485
+ if (!saved) return { outcome: 'not tested', why: 'no run saved anything at this step, so there was nothing to look for' };
486
+ if (!late.length) {
487
+ return shown ? { outcome: 'held', why: 'what it saved showed on the page right away' + (shown > 1 ? ', all ' + shown + ' times' : '') }
488
+ : { outcome: 'not tested', why: 'this page does not show what was saved, even when opened again - so whether it shows right away was not tested' };
489
+ }
490
+ const first = late[0];
491
+ const thing = thingOf(step.name);
492
+ const sure = late.length >= 2 && !shown;
493
+ const times = (n) => (n === 1 ? 'once' : n + ' times');
494
+ return {
495
+ outcome: 'late',
496
+ finding: {
497
+ flow: flow.title, where: flow.file + ':' + step.line, kind: 'shown', type: 'late', step: step.name,
498
+ severity: sure ? 'MEDIUM' : 'LOW', status: sure ? 'confirmed' : 'verification required',
499
+ headline: sure ? 'After "' + step.name + '", the new ' + thing + ' does not show until the page is opened again.'
500
+ : 'After "' + step.name + '", the new ' + thing + ' did not show until the page was opened again - in ' + late.length + ' of ' + saved + ' runs. Check whether it is slow to appear.',
501
+ proof: [
502
+ 'Flow: ' + flow.title + ' (' + flow.file + ':' + step.line + ')',
503
+ 'Saved: ' + words(first.write) + ', carrying "' + first.text + '".',
504
+ 'Before the click, "' + first.text + '" was on the page ' + times(first.before) + '; right after the save, ' + times(first.after) + ' - nothing new.',
505
+ 'The same address opened again showed it ' + times(first.fresh) + ': it was saved, and the page did not show it.',
506
+ 'Seen in ' + late.length + ' of ' + saved + ' runs that saved' + (shown ? '; in ' + shown + ' it showed right away.' : '.'),
507
+ ],
508
+ },
509
+ };
510
+ }
511
+
512
+ /** The fix, said for an AI assistant: what was seen, the proof, and what not to do. */
513
+ function fixPrompt(f) {
514
+ const what = {
515
+ flaky: 'Find why this sometimes fails - a request that sometimes errors, a race between two loads, something not waited for - and make it work every time. Look first at the server answer named above, if there is one.',
516
+ 'stopped-working': 'Check whether replaying this flow is meant to fail the second time (for example a name that must be unique). If it is meant to, say so. If not, fix it.',
517
+ double: 'Make saving safe against a double-click: disable the button while the save is in flight, and make the server ignore a repeat of the same request (an idempotency key, or a unique constraint on what makes it the same).',
518
+ race: 'Make the server refuse a repeat of the same save: send an idempotency key with each save and keep the first answer for it, or add a unique constraint on what makes two the same. Disabling the button is not enough on its own.',
519
+ phone: 'Make that button work at phone size without an error: find what the click runs at a narrow screen (a menu, a drawer, a panel that is hidden or not there yet) and handle it. Check it at 390 by 844.',
520
+ late: 'After a save succeeds, update what the page shows - add the new item to the list from the server\'s answer, or load the list again - so the person sees it without opening the page again. Otherwise they think it did not save, and save it again.',
521
+ }[f.type];
522
+ return [
523
+ 'My app does not behave the same way every time. Please fix the app - and nothing else.',
524
+ '',
525
+ 'Kryptheon replayed my recorded flow "' + f.flow + '" and saw this:',
526
+ f.headline,
527
+ '',
528
+ ].concat(f.proof, ['', what, 'Do not edit the recording in tests/ to make this go away.']).join('\n');
529
+ }
530
+
531
+ // --- running -------------------------------------------------------------
532
+
533
+ /**
534
+ * The whole check. `flows` is [{ file, title, source }] - recordings the
535
+ * caller already found runnable. Same options as kryptheon-edge.js run():
536
+ * say, onControl, confirmed, budget, gapMs; plus `only` for a re-check:
537
+ * { file, kind: 'flaky' } or { file, kind: 'double', line }.
538
+ */
539
+ async function run(root, flows, options) {
540
+ const opts = options || {};
541
+ const say = typeof opts.say === 'function' ? opts.say : () => {};
542
+ const budget = Math.min(probe.MAX_RUNS, opts.budget || probe.MAX_RUNS);
543
+ const rounds = opts.only && opts.only.kind === 'flaky' ? RECHECK_ROUNDS
544
+ : opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race' || opts.only.kind === 'phone') ? 1 : ROUNDS;
545
+ let stopped = false;
546
+ let current = null;
547
+ const control = {
548
+ stop() {
549
+ stopped = true;
550
+ if (current) current.stop();
551
+ },
552
+ };
553
+ if (typeof opts.onControl === 'function') opts.onControl(control);
554
+ const secretNames = [];
555
+ for (const f of flows) {
556
+ for (const m of String(f.source).matchAll(/process\.env\.([A-Za-z_][A-Za-z0-9_]*)/g)) if (!secretNames.includes(m[1])) secretNames.push(m[1]);
557
+ }
558
+ const report = {
559
+ check: 'reliability', startedAt: new Date().toISOString(), endedAt: null,
560
+ stopped: false, runsPlanned: 0, runsDone: 0, flows: [], findings: [], held: [], notTested: [], testData: [], writeHosts: [],
561
+ };
562
+
563
+ const runWith = async (probes, labels) => {
564
+ if (stopped || !probes.length) return {};
565
+ current = probe.runProbes(root, probes, { say: say, labels: labels, secretNames: secretNames, gapMs: opts.gapMs, env: opts.env, hide: opts.hide, childEnv: opts.childEnv });
566
+ const done = await current.done;
567
+ current = null;
568
+ report.runsDone += Object.keys(done.results).length;
569
+ return done.results;
570
+ };
571
+
572
+ // Which flows, within the cap: three plain runs each, and room for at
573
+ // least their saving steps once.
574
+ const chosen = [];
575
+ let planned = 0;
576
+ for (const f of flows) {
577
+ if (opts.only && f.file !== opts.only.file) continue;
578
+ if (replay.findReplayRisks(f.source).some((r) => r.kind === 'signup')) {
579
+ report.flows.push({ title: f.title, file: f.file, state: 'skipped', why: 'it creates an account, and this check never creates accounts' });
580
+ continue;
581
+ }
582
+ const steps = saveSteps(f.source).filter((s) => !opts.only || opts.only.kind === 'flaky' || opts.only.kind === 'phone' || s.line === opts.only.line);
583
+ const cost = rounds + (opts.only && opts.only.kind !== 'double' && opts.only.kind !== 'race' ? 0 : steps.length);
584
+ if (planned + cost > budget) {
585
+ report.flows.push({ title: f.title, file: f.file, state: 'not tested', why: 'over the limit of ' + budget + ' runs' });
586
+ continue;
587
+ }
588
+ planned += cost;
589
+ chosen.push(Object.assign({}, f, { steps: steps, runs: [] }));
590
+ }
591
+ report.runsPlanned = planned;
592
+ if (!chosen.length) {
593
+ report.endedAt = new Date().toISOString();
594
+ return report;
595
+ }
596
+
597
+ const plainProbe = (f, round, n) => ({ id: probe.probeId(n, 'run ' + round + ' ' + f.title), source: plainSource(f.source, f.steps), flow: f, round: round });
598
+ const labelsOf = (list, word) => {
599
+ const out = {};
600
+ for (const p of list) out[p.id] = word(p);
601
+ return out;
602
+ };
603
+ let n = 0;
604
+ const keep = (results, list) => {
605
+ for (const p of list) {
606
+ const r = results[p.id] || null;
607
+ if (p.round) p.flow.runs[p.round - 1] = r;
608
+ for (const w of writesOf(r)) {
609
+ if (w.origin && !report.writeHosts.includes(w.origin)) report.writeHosts.push(w.origin);
610
+ if (leftBehind(w)) report.testData.push({ flow: p.flow.title, value: p.round ? 'run ' + p.round : p.what || 'double-click', request: words(w) });
611
+ }
612
+ }
613
+ };
614
+
615
+ // 1. The first run of every flow - which is also where the app is seen
616
+ // saving. Somewhere that is not this computer, and not said yes to, stops
617
+ // it here (the same rule as the input check).
618
+ say('Replaying ' + chosen.length + (chosen.length === 1 ? ' flow' : ' flows') + ' ' + rounds + (rounds === 1 ? ' time' : ' times') + ', one round at a time...');
619
+ const first = chosen.map((f) => plainProbe(f, 1, ++n));
620
+ keep(await runWith(first, labelsOf(first, (p) => p.flow.title + ' - run 1 of ' + rounds)), first);
621
+ const unconfirmed = probe.liveLooking(report.writeHosts).filter((h) => !(opts.confirmed || []).includes(h));
622
+ if (unconfirmed.length) {
623
+ for (const f of chosen) {
624
+ report.flows.push({ title: f.title, file: f.file, state: 'not tested',
625
+ why: 'your app saves to ' + unconfirmed.join(', ') + ', which is not this computer. Press again: you will be asked about that address first' });
626
+ }
627
+ report.stopped = stopped;
628
+ report.endedAt = new Date().toISOString();
629
+ return report;
630
+ }
631
+
632
+ // 2. The other rounds: every flow once, then every flow again, so time
633
+ // passes between two runs of the same flow.
634
+ for (let round = 2; round <= rounds && !stopped; round++) {
635
+ const list = chosen.map((f) => plainProbe(f, round, ++n));
636
+ keep(await runWith(list, labelsOf(list, (p) => p.flow.title + ' - run ' + round + ' of ' + rounds)), list);
637
+ }
638
+
639
+ // 3. Each saving step, clicked twice - against a run where it was clicked
640
+ // once and the flow worked.
641
+ const doubles = [];
642
+ for (const f of chosen) f.single = f.runs.find((r) => r && r.status === 'passed' && !limited(r)) || null;
643
+ if (!(opts.only && opts.only.kind !== 'double')) {
644
+ for (const f of chosen) {
645
+ const single = f.single;
646
+ for (const s of f.steps) {
647
+ const once = single && windowWrites(single, s.line);
648
+ const posts = once ? Object.keys(created(once)).length : 0;
649
+ const label = { flow: f.title, where: f.file + ':' + s.line, kind: 'double', step: s.name };
650
+ if (!single) report.notTested.push(Object.assign(label, { why: 'the flow never worked as recorded, so there is nothing to compare a double-click with' }));
651
+ else if (!once) report.notTested.push(Object.assign(label, { why: 'the replay did not reach this step' }));
652
+ else if (!posts) report.held.push(Object.assign(label, { why: singleClickWords(once) }));
653
+ else doubles.push({ id: probe.probeId(++n, 'double ' + s.name), source: doubleSource(f.source, f.steps, s), flow: f, step: s });
654
+ }
655
+ }
656
+ }
657
+ let doubled = {};
658
+ if (doubles.length && !stopped) {
659
+ say('Clicking ' + doubles.length + (doubles.length === 1 ? ' saving step' : ' saving steps') + ' twice, quickly...');
660
+ doubled = await runWith(doubles, labelsOf(doubles, (p) => p.flow.title + ' - "' + p.step.name + '" double-clicked'));
661
+ keep(doubled, doubles.map((p) => Object.assign({}, p, { round: 0 })));
662
+ }
663
+ const dupes = [];
664
+ for (const p of doubles) {
665
+ const v = judgeDouble(p.flow, p.step, p.flow.single, doubled[p.id]);
666
+ const label = { flow: p.flow.title, where: p.flow.file + ':' + p.step.line, kind: 'double', step: p.step.name };
667
+ if (v.outcome === 'duplicate') dupes.push({ probe: p, finding: v.finding });
668
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
669
+ else report.notTested.push(Object.assign(label, { why: !doubled[p.id] && stopped ? 'stopped before it finished' : v.why }));
670
+ }
671
+
672
+ // 4. A duplicate seen once is tried once more: two saves can also come
673
+ // from a slow app answering late, and one sighting is not a finding.
674
+ const again = dupes.slice(0, CONFIRM_RUNS);
675
+ if (again.length && !stopped && report.runsDone + again.length <= probe.MAX_RUNS) {
676
+ say('Double-clicking ' + again.length + ' of them again, to be sure...');
677
+ const list = again.map((d) => Object.assign({}, d.probe, { id: probe.probeId(++n, 'again ' + d.probe.step.name) }));
678
+ report.runsPlanned += list.length;
679
+ const res = await runWith(list, labelsOf(list, (p) => p.flow.title + ' - "' + p.step.name + '" double-clicked again'));
680
+ keep(res, list.map((p) => Object.assign({}, p, { round: 0 })));
681
+ again.forEach((d, i) => {
682
+ const second = judgeDouble(d.probe.flow, d.probe.step, d.probe.flow.single, res[list[i].id]);
683
+ if (second.outcome === 'duplicate') d.finding.proof.push('It happened again on a second try.');
684
+ else {
685
+ d.finding.status = 'verification required';
686
+ d.finding.severity = 'LOW';
687
+ d.finding.headline = d.finding.headline.replace(/\.$/, '') + ' - once, not on a second try.';
688
+ d.finding.proof.push(res[list[i].id] ? 'A second double-click saved it once.' : 'A second try did not finish, so it is not confirmed.');
689
+ }
690
+ });
691
+ } else {
692
+ for (const d of again) d.finding.proof.push('Not tried a second time (stopped, or no room left in the ' + probe.MAX_RUNS + ' runs).');
693
+ }
694
+ for (const d of dupes) report.findings.push(d.finding);
695
+
696
+ // 4a. Two at once: each save a single click makes, sent twice at the same
697
+ // moment - only to an app on this computer, and only where a double-click
698
+ // did not already show it saving twice. Within what is left of the cap.
699
+ if (!(opts.only && opts.only.kind !== 'race')) {
700
+ const doubledTwice = new Set(dupes.map((d) => d.probe.flow.file + ':' + d.probe.step.line));
701
+ const races = [];
702
+ for (const f of chosen) {
703
+ for (const s of f.steps) {
704
+ const where = f.file + ':' + s.line;
705
+ const label = { flow: f.title, where: where, kind: 'race', step: s.name };
706
+ const once = f.single && windowWrites(f.single, s.line);
707
+ const post = once && once.find((w) => w.method === 'POST' && ok(w) && !w.signIn);
708
+ if (!post || doubledTwice.has(where)) continue;
709
+ if (probe.liveLooking([post.origin || post.host]).length) {
710
+ report.notTested.push(Object.assign(label, { why: 'two saves at the same moment are only sent to an app on this computer, and this one saves to ' + (post.origin || post.host) }));
711
+ continue;
712
+ }
713
+ if (stopped || report.runsDone + races.length + 1 > budget) {
714
+ report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it ran' : 'over the limit of ' + budget + ' runs' }));
715
+ continue;
716
+ }
717
+ const pageHeld = report.held.some((h) => h.kind === 'double' && h.where === where);
718
+ races.push({ id: probe.probeId(++n, 'twice ' + s.name), source: twiceSource(f.source, s, { method: post.method, host: post.host, path: post.path }), flow: f, step: s, pageHeld: pageHeld });
719
+ }
720
+ }
721
+ if (races.length) {
722
+ report.runsPlanned += races.length;
723
+ say('Sending ' + races.length + (races.length === 1 ? ' save' : ' saves') + ' twice at the same moment...');
724
+ const raced = await runWith(races, labelsOf(races, (p) => p.flow.title + ' - "' + p.step.name + '" sent twice at once'));
725
+ keep(raced, races.map((p) => Object.assign({}, p, { round: 0, what: 'sent twice at once' })));
726
+ for (const p of races) {
727
+ const v = judgeTwice(p.flow, p.step, p.flow.single, raced[p.id], p.pageHeld);
728
+ const label = { flow: p.flow.title, where: p.flow.file + ':' + p.step.line, kind: 'race', step: p.step.name };
729
+ if (v.finding) report.findings.push(v.finding);
730
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
731
+ else report.notTested.push(Object.assign(label, { why: !raced[p.id] && stopped ? 'stopped before it finished' : v.why }));
732
+ }
733
+ }
734
+ }
735
+
736
+ // 4c. At phone size: each flow once more, every button clicked with every
737
+ // write blocked. Within what is left of the cap.
738
+ if (!(opts.only && opts.only.kind !== 'phone')) {
739
+ const phones = [];
740
+ for (const f of chosen) {
741
+ const label = { flow: f.title, where: f.file, kind: 'phone' };
742
+ if (stopped || report.runsDone + phones.length + 1 > budget) {
743
+ report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it ran' : 'over the limit of ' + budget + ' runs' }));
744
+ continue;
745
+ }
746
+ phones.push({ id: probe.probeId(++n, 'phone ' + f.title), source: phoneSource(f.source), flow: f });
747
+ }
748
+ if (phones.length) {
749
+ report.runsPlanned += phones.length;
750
+ say('Replaying ' + phones.length + (phones.length === 1 ? ' flow' : ' flows') + ' at phone size, clicking every button with saving blocked...');
751
+ const swept = await runWith(phones, labelsOf(phones, (p) => p.flow.title + ' - at phone size, every button'));
752
+ keep(swept, phones.map((p) => Object.assign({}, p, { round: 0, what: 'at phone size' })));
753
+ // The same button on the same page throwing the same error is one
754
+ // problem, however many flows pass by it (blind round 17: one menu
755
+ // button, said five times).
756
+ const samePhone = new Map();
757
+ for (const p of phones) {
758
+ const v = judgePhone(p.flow, swept[p.id]);
759
+ const label = { flow: p.flow.title, where: p.flow.file, kind: 'phone' };
760
+ const same = v.finding ? v.finding.headline + '|' + v.finding.proof[2] : null;
761
+ if (same && samePhone.has(same)) {
762
+ const first = samePhone.get(same);
763
+ first.alsoIn = (first.alsoIn || 0) + 1;
764
+ if (first.alsoIn <= 3) first.proof.splice(first.proof.length - 1, 0, 'The same in "' + p.flow.title + '" (' + p.flow.file + ').');
765
+ report.held.push(Object.assign(label, { why: 'at phone size, the same error as in "' + first.flow + '" - said once, there' }));
766
+ } else if (v.finding) {
767
+ if (same) samePhone.set(same, v.finding);
768
+ report.findings.push(v.finding);
769
+ }
770
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
771
+ else report.notTested.push(Object.assign(label, { why: !swept[p.id] && stopped ? 'stopped before it finished' : v.why }));
772
+ // What was left unclicked is its own line, so a re-check is not held
773
+ // up by it.
774
+ for (const why of v.unclicked || []) report.notTested.push(Object.assign({}, label, { kind: 'phone-unclicked', why: why }));
775
+ }
776
+ }
777
+ }
778
+
779
+ // 4b. What each save added, looked for on the page right after it - from
780
+ // the plain runs, so it costs no run of its own.
781
+ if (!(opts.only && opts.only.kind !== 'shown')) {
782
+ for (const f of chosen) {
783
+ for (const s of f.steps) {
784
+ if (!typedFor(f.source, saveSteps(f.source), s).length) continue;
785
+ const v = judgeShown(f, s, f.runs);
786
+ const label = { flow: f.title, where: f.file + ':' + s.line, kind: 'shown', step: s.name };
787
+ if (v.finding) report.findings.push(v.finding);
788
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
789
+ else report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it finished' : v.why }));
790
+ }
791
+ }
792
+ }
793
+
794
+ // 5. The three runs of each flow. A re-check of a double-click ran its
795
+ // flow once, only to have a single click to compare with - one run says
796
+ // nothing about flakiness, so the flow counts as tested when that run
797
+ // worked, and nothing is judged flaky from it.
798
+ for (const f of chosen) {
799
+ if (opts.only && opts.only.kind === 'phone') {
800
+ report.flows.push({ title: f.title, file: f.file, state: f.runs.some((r) => r) ? 'tested' : 'not tested', why: f.runs.some((r) => r) ? '' : 'the run did not finish' });
801
+ continue;
802
+ }
803
+ if (opts.only && (opts.only.kind === 'double' || opts.only.kind === 'race')) {
804
+ report.flows.push({ title: f.title, file: f.file, state: f.single ? 'tested' : 'not tested',
805
+ why: f.single ? '' : 'the flow did not work as recorded, so there was nothing to compare the double-click with' });
806
+ continue;
807
+ }
808
+ // A re-check of what a save shows needs the runs to have finished, not a
809
+ // verdict on flakiness.
810
+ if (opts.only && opts.only.kind === 'shown') {
811
+ const finished = f.runs.some((r) => r && !limited(r));
812
+ report.flows.push({ title: f.title, file: f.file, state: finished ? 'tested' : 'not tested', why: finished ? '' : 'none of the runs finished' });
813
+ continue;
814
+ }
815
+ const v = judgeFlaky(f, f.runs);
816
+ report.flows.push({ title: f.title, file: f.file, state: v.outcome === 'not tested' ? 'not tested' : 'tested', why: v.why || '',
817
+ runs: f.runs.map((r) => (!r ? 'did not finish' : limited(r) ? 'not counted (429)' : r.status)) });
818
+ const label = { flow: f.title, where: f.file, kind: 'flaky' };
819
+ if (v.finding) report.findings.push(v.finding);
820
+ else if (v.outcome === 'held') report.held.push(Object.assign(label, { why: v.why }));
821
+ else if (v.outcome === 'broken') report.notTested.push(Object.assign(label, { why: v.why }));
822
+ else if (v.outcome === 'not tested') report.notTested.push(Object.assign(label, { why: stopped ? 'stopped before it finished' : v.why }));
823
+ }
824
+
825
+ for (const f of report.findings) f.fixPrompt = fixPrompt(f);
826
+ const rank = { HIGH: 0, MEDIUM: 1, LOW: 2 };
827
+ report.findings.sort((a, b) => rank[a.severity] - rank[b.severity]);
828
+ report.stopped = stopped;
829
+ report.endedAt = new Date().toISOString();
830
+ return report;
831
+ }
832
+
833
+ // What a finding is about, for keeping it across presses: the flow for a
834
+ // flaky one, the step for a double-click.
835
+ const keyOf = (x) => (x.kind === 'flaky' ? String(x.where).replace(/:\d+$/, '') : x.where) + '|' + x.kind;
836
+
837
+ /**
838
+ * What a press did not try is not cleared by it: a finding from an earlier
839
+ * press whose flow or step was not tried this time is kept, and says so.
840
+ */
841
+ function carryOver(previous, report, root) {
842
+ if (!previous || !Array.isArray(previous.findings)) return report;
843
+ const tried = new Set(report.held.concat(report.findings).map(keyOf));
844
+ for (const f of previous.findings) {
845
+ if (tried.has(keyOf(f))) continue;
846
+ const file = String(f.where).replace(/:\d+$/, '');
847
+ if (root && !fs.existsSync(path.join(root, file))) continue;
848
+ const kept = Object.assign({}, f, { carried: f.carried || previous.endedAt || previous.startedAt });
849
+ if (!f.carried) kept.proof = (f.proof || []).concat(['Seen on an earlier run; not tried on the latest one.']);
850
+ report.findings.push(kept);
851
+ }
852
+ return report;
853
+ }
854
+
855
+ function load(root, env) {
856
+ return probe.loadReport(root, 'reliability', env);
857
+ }
858
+
859
+ function save(root, report, env, options) {
860
+ if (!(options && options.asIs)) carryOver(load(root, env), report, root);
861
+ const file = probe.reportFile(root, 'reliability', env);
862
+ fs.mkdirSync(path.dirname(file), { recursive: true });
863
+ fs.writeFileSync(file, JSON.stringify(report, null, 2) + '\n', 'utf8');
864
+ return file;
865
+ }
866
+
867
+ /**
868
+ * Re-checks one finding: a flaky flow is replayed five times and is fixed
869
+ * only when all five work; a double-click is tried again against one plain
870
+ * run. The same four verdicts as every re-check.
871
+ */
872
+ async function recheck(root, flows, finding, options) {
873
+ const opts = options || {};
874
+ const file = String(finding.where || '').replace(/:\d+$/, '');
875
+ const line = Number((/:(\d+)$/.exec(finding.where || '') || [])[1]);
876
+ const unsure = (why) => ({ verdict: 'could not confirm', why: why, newProblems: [], testData: [] });
877
+ const flow = flows.find((f) => f.file === file);
878
+ if (!flow) return unsure('the recording ' + file + ' is not there, or cannot be run now');
879
+ if (finding.kind !== 'flaky' && finding.kind !== 'phone' && !saveSteps(flow.source).some((s) => s.line === line)) return unsure('that step is no longer at line ' + line + ' of ' + file);
880
+ const report = await run(root, [flow], Object.assign({}, opts, {
881
+ only: finding.kind === 'flaky' || finding.kind === 'phone' ? { file: file, kind: finding.kind } : { file: file, kind: finding.kind, line: line },
882
+ }));
883
+ const before = load(root, opts.env);
884
+ if (before && report.writeHosts.some((h) => !(before.writeHosts || []).includes(h))) {
885
+ before.writeHosts = Array.from(new Set((before.writeHosts || []).concat(report.writeHosts)));
886
+ save(root, before, opts.env, { asIs: true });
887
+ }
888
+ if (report.stopped) return unsure('it was stopped before it finished');
889
+ const ran = report.flows.find((f) => f.file === file);
890
+ if (!ran || ran.state !== 'tested') return unsure(ran ? ran.why : 'the flow did not run');
891
+ const left = report.notTested.find((x) => keyOf(x) === keyOf(finding));
892
+ if (left) return unsure(left.why);
893
+ const same = report.findings.find((f) => keyOf(f) === keyOf(finding));
894
+ const others = report.findings.filter((f) => f !== same && f.status === 'confirmed');
895
+ const result = same ? { verdict: 'still open', why: same.proof.slice(1).join(' '), newProblems: others }
896
+ : { verdict: 'fixed', why: finding.kind === 'flaky' ? 'it worked all ' + RECHECK_ROUNDS + ' times'
897
+ : finding.kind === 'shown' || finding.kind === 'race' || finding.kind === 'phone' ? (report.held.find((x) => keyOf(x) === keyOf(finding)) || {}).why || 'it held'
898
+ : 'a double-click saved it once', newProblems: others };
899
+ const saved = load(root, opts.env);
900
+ if (saved && result.verdict === 'fixed') {
901
+ saved.findings = saved.findings.filter((f) => keyOf(f) !== keyOf(finding));
902
+ save(root, saved, opts.env, { asIs: true });
903
+ }
904
+ result.testData = report.testData;
905
+ return result;
906
+ }
907
+
908
+ /** What a held or not-tested line is about: the flow, a double-click, or what a save shows. */
909
+ function heldLabel(x) {
910
+ return x.kind === 'flaky' ? x.flow + ' - '
911
+ : x.kind === 'shown' ? 'After "' + x.step + '" (' + x.where + '), looked for on the page - '
912
+ : x.kind === 'race' ? '"' + x.step + '" (' + x.where + ') sent twice at once - '
913
+ : x.kind === 'phone' || x.kind === 'phone-unclicked' ? 'At phone size, ' + x.flow + ' - '
914
+ : '"' + x.step + '" (' + x.where + ') double-clicked - ';
915
+ }
916
+
917
+ /** What is said when it ends, in the terminal and on the page. */
918
+ function summaryLines(report) {
919
+ const lines = [''];
920
+ const problems = report.findings.filter((f) => f.status === 'confirmed');
921
+ const toCheck = report.findings.filter((f) => f.status !== 'confirmed');
922
+ const triedAny = report.held.length || report.findings.some((f) => !f.carried);
923
+ if (report.stopped) lines.push('Stopped after ' + report.runsDone + ' of ' + report.runsPlanned + ' runs. What did not run was NOT tested.');
924
+ lines.push(problems.length ? problems.length + (problems.length === 1 ? ' problem' : ' problems') + ' found:'
925
+ : triedAny ? 'No problem found in what was tried.' : 'Nothing was tested - see why below.');
926
+ for (const f of report.findings) {
927
+ lines.push('');
928
+ lines.push(' ' + f.severity + (f.status === 'confirmed' ? '' : ' (to check)') + ' ' + f.headline);
929
+ for (const p of f.proof) lines.push(' ' + p);
930
+ }
931
+ if (toCheck.length && !problems.length) lines.push('', toCheck.length + (toCheck.length === 1 ? ' thing' : ' things') + ' to check above - not proven problems.');
932
+ if (report.held.length) {
933
+ lines.push('', 'Held (' + report.held.length + '):');
934
+ for (const h of report.held) lines.push(' ' + heldLabel(h) + h.why);
935
+ }
936
+ const skipped = report.flows.filter((f) => f.state !== 'tested');
937
+ if (report.notTested.length || skipped.length) {
938
+ lines.push('', 'NOT tested:');
939
+ const said = new Set();
940
+ for (const f of skipped) {
941
+ said.add(f.file);
942
+ lines.push(' ' + f.title + ' - ' + f.why);
943
+ }
944
+ for (const x of report.notTested) {
945
+ if (x.kind === 'flaky' && said.has(x.where)) continue;
946
+ lines.push(' ' + heldLabel(x) + x.why);
947
+ }
948
+ }
949
+ if (report.testData.length) {
950
+ lines.push('', 'Test data left in your app: ' + report.testData.length + (report.testData.length === 1 ? ' write was' : ' writes were') + ' accepted.');
951
+ lines.push('These are your flows\' own saves, replayed. Kryptheon does not delete them. The writes:');
952
+ for (const t of report.testData.slice(0, 15)) lines.push(' ' + t.request + ' (' + t.flow + ', ' + t.value + ')');
953
+ if (report.testData.length > 15) lines.push(' ... and ' + (report.testData.length - 15) + ' more');
954
+ }
955
+ lines.push('');
956
+ return lines;
957
+ }
958
+
959
+ module.exports = {
960
+ ROUNDS: ROUNDS,
961
+ saveSteps: saveSteps,
962
+ plainSource: plainSource,
963
+ doubleSource: doubleSource,
964
+ windowWrites: windowWrites,
965
+ created: created,
966
+ thingOf: thingOf,
967
+ recordIds: recordIds,
968
+ judgeFlaky: judgeFlaky,
969
+ judgeDouble: judgeDouble,
970
+ judgeShown: judgeShown,
971
+ judgeTwice: judgeTwice,
972
+ judgePhone: judgePhone,
973
+ phoneSource: phoneSource,
974
+ twiceSource: twiceSource,
975
+ typedFor: typedFor,
976
+ fixPrompt: fixPrompt,
977
+ run: run,
978
+ recheck: recheck,
979
+ carryOver: carryOver,
980
+ save: save,
981
+ load: load,
982
+ summaryLines: summaryLines,
983
+ knownAddresses: probe.knownAddresses,
984
+ };