@graphty/visual-review 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +448 -144
- package/capture/capture.mjs +7 -0
- package/package.json +1 -1
- package/templates/visual-review.yml +4 -2
- package/trusted/cli.mjs +1 -0
- package/trusted/gate.mjs +246 -42
- package/trusted/lib/accept.mjs +474 -116
- package/trusted/lib/approval.mjs +300 -0
- package/trusted/lib/config.mjs +4 -3
- package/trusted/lib/github.mjs +176 -29
- package/trusted/lib/serve.mjs +1237 -154
- package/trusted/lib/thumbs.mjs +149 -0
- package/trusted/page/index.html +20 -6
- package/trusted/page/passkey.js +94 -0
- package/trusted/page/review.css +924 -92
- package/trusted/page/review.js +3238 -991
package/trusted/lib/serve.mjs
CHANGED
|
@@ -6,22 +6,69 @@
|
|
|
6
6
|
* served origin. Images are served only when their file is named by results.json and its bytes
|
|
7
7
|
* hash to the hash results.json gives, so the page shows exactly what CI compared. Decisions
|
|
8
8
|
* are kept in `<tmp>/state/<target>.json` until Finish, each with the hash of the image it was
|
|
9
|
-
* taken on, so a restart resumes them and a new CI run
|
|
9
|
+
* taken on, so a restart resumes them and a new CI run or attempt shows only those whose image is
|
|
10
|
+
* unchanged. The file is the only copy: every request reads it and every change rewrites it, so a
|
|
11
|
+
* failed write changes nothing, and a decision whose item is missing from this run (a project
|
|
12
|
+
* still capturing, a download that failed) is kept for when it comes back.
|
|
10
13
|
*
|
|
11
14
|
* Finish runs in the background: a large seed takes minutes, longer than a browser (Safari on an
|
|
12
15
|
* iPad) keeps one request open. POST /api/finish starts it and GET /api/finish-status reports its
|
|
13
|
-
* step, then its result or error, so a reload finds the running Finish.
|
|
16
|
+
* step, then its result or error, so a reload finds the running Finish. The job is also written
|
|
17
|
+
* to `<tmp>/state/finish.json`, so a server restarted during a Finish says it was interrupted.
|
|
18
|
+
*
|
|
19
|
+
* Passkeys: once any key is known (passkeys.json on the fetched default branch, or one this server
|
|
20
|
+
* registered whose pull request is not merged yet, `<tmp>/state/passkeys-pending.json`), Finish
|
|
21
|
+
* needs the owner's approval. A pending key approves only while the default branch holds none: once
|
|
22
|
+
* it does, only its keys count, as in the gate, so a key registered later (by anyone who can reach
|
|
23
|
+
* this page) approves nothing until its pull request merges. POST /api/finish-prepare builds the
|
|
24
|
+
* record and returns its hash as the WebAuthn challenge; POST /api/finish takes the approval,
|
|
25
|
+
* checks it, and commits exactly that record. The server decides whether approval is required, so
|
|
26
|
+
* leaving it out never skips it. The rpId is the host name of the page's own origin.
|
|
27
|
+
*
|
|
28
|
+
* The list of targets is cached: `GET /api/prs?cached=1` answers at once with the cache, its age,
|
|
29
|
+
* the progress of a refresh, why the last one failed and any network retry under way, and
|
|
30
|
+
* `?refresh=1` starts one in the background, so the page never waits on GitHub to change screens.
|
|
31
|
+
* The list is also kept in `<tmp>/state/list.json`, so a restarted server shows it at once and
|
|
32
|
+
* refreshes it behind. With `warm` the server refreshes at start and every two minutes, so the
|
|
33
|
+
* captures of a CI run that just finished are downloaded before the owner opens them. A target
|
|
34
|
+
* whose captures are still downloading is listed with `downloading: true` and its progress, and
|
|
35
|
+
* rebuilt in the cache when they land; `GET /api/pr/...` of a project still downloading answers
|
|
36
|
+
* 202 and moves its download to the front of the queue. What GitHub says about a finished run
|
|
37
|
+
* (its jobs and artifacts) is asked once and kept beside its downloads. Grid tiles load
|
|
38
|
+
* thumbnails (`GET /api/thumb/...`), scaled down in worker threads as soon as a capture lands and
|
|
39
|
+
* kept on disk.
|
|
14
40
|
*/
|
|
15
41
|
|
|
16
42
|
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
|
17
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
43
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
18
44
|
import { readFile } from "node:fs/promises";
|
|
19
|
-
import { dirname, join } from "node:path";
|
|
45
|
+
import { basename, dirname, join } from "node:path";
|
|
20
46
|
import { fileURLToPath } from "node:url";
|
|
21
47
|
|
|
22
|
-
import {
|
|
23
|
-
|
|
48
|
+
import {
|
|
49
|
+
AcceptError,
|
|
50
|
+
behindMaster,
|
|
51
|
+
cleanReason,
|
|
52
|
+
commitStatus,
|
|
53
|
+
decisionProblem,
|
|
54
|
+
finish,
|
|
55
|
+
prepareRecord,
|
|
56
|
+
proposeKey,
|
|
57
|
+
} from "./accept.mjs";
|
|
58
|
+
import { PASSKEYS_FILE, parsePasskeys, recordHash, verifyApproval, verifyRegistration } from "./approval.mjs";
|
|
59
|
+
import {
|
|
60
|
+
downloadCaptures,
|
|
61
|
+
exec,
|
|
62
|
+
getRun,
|
|
63
|
+
hurry,
|
|
64
|
+
newestCiRun,
|
|
65
|
+
openPullRequests,
|
|
66
|
+
retrying,
|
|
67
|
+
visualJobs,
|
|
68
|
+
} from "./github.mjs";
|
|
69
|
+
import { CONFIG_FILE } from "./config.mjs";
|
|
24
70
|
import { validateResults } from "./results.mjs";
|
|
71
|
+
import { scaled } from "./thumbs.mjs";
|
|
25
72
|
|
|
26
73
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
27
74
|
const STATIC = {
|
|
@@ -29,6 +76,7 @@ const STATIC = {
|
|
|
29
76
|
"/review.js": ["../page/review.js", "text/javascript; charset=utf-8"],
|
|
30
77
|
"/review.css": ["../page/review.css", "text/css; charset=utf-8"],
|
|
31
78
|
"/pixelmatch.mjs": ["../vendor/pixelmatch.mjs", "text/javascript; charset=utf-8"],
|
|
79
|
+
"/passkey.js": ["../page/passkey.js", "text/javascript; charset=utf-8"],
|
|
32
80
|
};
|
|
33
81
|
const HEADERS = {
|
|
34
82
|
"content-security-policy":
|
|
@@ -45,7 +93,25 @@ const HEADERS = {
|
|
|
45
93
|
const componentOf = (id) => id.split("--")[0];
|
|
46
94
|
|
|
47
95
|
const REVIEWABLE = new Set(["changed", "moved", "new", "unseeded", "removed", "unstable", "failed"]);
|
|
48
|
-
const WRITES = new Set(["decide", "accept-all", "finish"]);
|
|
96
|
+
const WRITES = new Set(["decide", "accept-all", "finish", "finish-prepare", "passkey-challenge", "register"]);
|
|
97
|
+
// A registration challenge is good for one use within this long; a prepared approval for this long.
|
|
98
|
+
const CHALLENGE_MS = 5 * 60000;
|
|
99
|
+
const APPROVAL_MS = 10 * 60000;
|
|
100
|
+
// How long a page load waits for a run's captures to download before it lists the target as
|
|
101
|
+
// downloading; the download goes on, and a reload picks it up.
|
|
102
|
+
const PATIENCE = 1000;
|
|
103
|
+
// An unknown target id refreshes from GitHub at most this often: a closed pull request's open
|
|
104
|
+
// grid asks for dozens of images at once.
|
|
105
|
+
const UNKNOWN_REFRESH = 60000;
|
|
106
|
+
// With `warm`, the list is refreshed this often, so a CI run that finished meanwhile is downloaded
|
|
107
|
+
// before the owner opens it.
|
|
108
|
+
const POLL = 2 * 60000;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The newest gh call waiting to retry after a network failure, for the page.
|
|
112
|
+
* @returns {{ error: string, attempt: number, of: number, until: number } | null} it, or null
|
|
113
|
+
*/
|
|
114
|
+
const networkTrouble = () => [...retrying].at(-1) ?? null;
|
|
49
115
|
|
|
50
116
|
/**
|
|
51
117
|
* The image a decision was taken on: the capture, or for a removed item its baseline.
|
|
@@ -124,13 +190,27 @@ async function signingIdentity(repo) {
|
|
|
124
190
|
*/
|
|
125
191
|
export function sessionToken(stateDir) {
|
|
126
192
|
const file = join(stateDir, "token");
|
|
127
|
-
|
|
193
|
+
// An empty file (a full disk on the first start) would be an empty token: write a new one.
|
|
194
|
+
if (!existsSync(file) || readFileSync(file, "utf8").trim() === "") {
|
|
128
195
|
mkdirSync(stateDir, { recursive: true });
|
|
129
196
|
writeFileSync(file, randomBytes(32).toString("base64url"), { mode: 0o600 });
|
|
130
197
|
}
|
|
131
198
|
return readFileSync(file, "utf8").trim();
|
|
132
199
|
}
|
|
133
200
|
|
|
201
|
+
/**
|
|
202
|
+
* Writes a JSON file through a sibling renamed into place, so a kill or a full disk mid-write
|
|
203
|
+
* never leaves half a file.
|
|
204
|
+
* @param {string} file the file
|
|
205
|
+
* @param {unknown} value what to write
|
|
206
|
+
*/
|
|
207
|
+
function writeJson(file, value) {
|
|
208
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
209
|
+
const part = `${file}.tmp`;
|
|
210
|
+
writeFileSync(part, JSON.stringify(value, null, 2));
|
|
211
|
+
renameSync(part, file);
|
|
212
|
+
}
|
|
213
|
+
|
|
134
214
|
/**
|
|
135
215
|
* Reads and validates one project's results.json.
|
|
136
216
|
* @param {string} dir the capture directory
|
|
@@ -167,30 +247,125 @@ async function loadResults(dir) {
|
|
|
167
247
|
* a preview to look at, with no decisions and no Finish
|
|
168
248
|
* @param {string} [options.startCommand] the shell command that starts this server, shown so the
|
|
169
249
|
* owner can restart it from their own shell and sign Finish with their own key
|
|
250
|
+
* @param {boolean} [options.warm] start downloading every target's captures right away
|
|
170
251
|
* @returns {(req: import("node:http").IncomingMessage, res: import("node:http").ServerResponse) => void}
|
|
171
252
|
* the handler, for node:https in the CLI and node:http in the tests
|
|
172
253
|
*/
|
|
173
|
-
export function createApp({ repo, gh, config, tmp, token, origin, masterRun, results, startCommand = null }) {
|
|
254
|
+
export function createApp({ repo, gh, config, tmp, token, origin, masterRun, results, startCommand = null, warm }) {
|
|
174
255
|
const stateDir = join(tmp, "state");
|
|
175
256
|
const { projects, defaultBranch } = config;
|
|
176
257
|
const names = Object.keys(projects);
|
|
177
258
|
/** @type {Map<string, object>} targets by id: a pull request number, or "master" */
|
|
178
259
|
let targets = new Map();
|
|
179
|
-
/**
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
* posted?: true }>>}
|
|
184
|
-
*/
|
|
185
|
-
const decisions = new Map();
|
|
260
|
+
/** What the last refresh could not read from GitHub (the pull request list, a git fetch). */
|
|
261
|
+
let listWarnings = [];
|
|
262
|
+
/** Why a target's saved decisions were set aside, by target id. */
|
|
263
|
+
const stateProblems = new Map();
|
|
186
264
|
let finishing = false;
|
|
265
|
+
const jobFile = join(stateDir, "finish.json");
|
|
187
266
|
/**
|
|
188
267
|
* The newest Finish, kept after it ends so a reload still shows its result.
|
|
189
|
-
* @type {{ id: number, target: string, pr: number | null,
|
|
190
|
-
* result: object | null, error: string | null
|
|
268
|
+
* @type {{ id: number, target: string, pr: number | null, branch: string | null, running: boolean,
|
|
269
|
+
* step: string | null, result: object | null, error: string | null, warnings: string[],
|
|
270
|
+
* interrupted?: true } | null}
|
|
191
271
|
*/
|
|
192
272
|
let job = null;
|
|
273
|
+
try {
|
|
274
|
+
const saved = JSON.parse(readFileSync(jobFile, "utf8"));
|
|
275
|
+
if (saved?.running) {
|
|
276
|
+
const where = saved.branch ?? "the visual/seed-* branch";
|
|
277
|
+
job = {
|
|
278
|
+
...saved,
|
|
279
|
+
running: false,
|
|
280
|
+
step: null,
|
|
281
|
+
interrupted: true,
|
|
282
|
+
warnings: saved.warnings ?? [],
|
|
283
|
+
error:
|
|
284
|
+
`the server stopped while this Finish was at "${saved.step}", so it never ended: check ` +
|
|
285
|
+
`whether ${where} on origin has its commit before pressing Finish again`,
|
|
286
|
+
};
|
|
287
|
+
console.error(`visual-review: ${job.error}`);
|
|
288
|
+
}
|
|
289
|
+
} catch {
|
|
290
|
+
// No Finish ran yet.
|
|
291
|
+
}
|
|
292
|
+
const persist = (j) => {
|
|
293
|
+
try {
|
|
294
|
+
writeJson(jobFile, j);
|
|
295
|
+
} catch (err) {
|
|
296
|
+
const warning = `could not save the Finish job (a restart would not report it): ${err.message}`;
|
|
297
|
+
if (!j.warnings.includes(warning)) {
|
|
298
|
+
j.warnings.push(warning);
|
|
299
|
+
console.error(`visual-review: ${warning}`);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
};
|
|
193
303
|
let signer = null;
|
|
304
|
+
const rpId = new URL(origin).hostname;
|
|
305
|
+
const pendingFile = join(stateDir, "passkeys-pending.json");
|
|
306
|
+
/** Registration challenges (base64url) and when each expires. */
|
|
307
|
+
const challenges = new Map();
|
|
308
|
+
/**
|
|
309
|
+
* The record each target's Finish was prepared with, waiting for its approval.
|
|
310
|
+
* @type {Map<string, { challenge: string, record: object, work: object, digest: string,
|
|
311
|
+
* expires: number }>}
|
|
312
|
+
*/
|
|
313
|
+
const approvals = new Map();
|
|
314
|
+
// Fails closed: only a missing file means no pending key; any other failure throws.
|
|
315
|
+
const readPending = () => {
|
|
316
|
+
let text;
|
|
317
|
+
try {
|
|
318
|
+
text = readFileSync(pendingFile, "utf8");
|
|
319
|
+
} catch (err) {
|
|
320
|
+
if (err.code === "ENOENT") {
|
|
321
|
+
return [];
|
|
322
|
+
}
|
|
323
|
+
throw err;
|
|
324
|
+
}
|
|
325
|
+
try {
|
|
326
|
+
return parsePasskeys(text);
|
|
327
|
+
} catch (err) {
|
|
328
|
+
throw new Error(`${pendingFile} is invalid: ${err.message}`);
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
/**
|
|
332
|
+
* The keys an approval may come from: the fetched default branch's passkeys.json, and the
|
|
333
|
+
* keys this server registered that it does not hold yet. A git failure or an invalid file
|
|
334
|
+
* throws: approval is never skipped because a key could not be read. `trusted` is what may
|
|
335
|
+
* approve: the default branch's keys, or the pending ones while it has none.
|
|
336
|
+
* @returns {Promise<{ main: object[], pending: object[], trusted: object[] }>} the keys
|
|
337
|
+
*/
|
|
338
|
+
async function knownKeys() {
|
|
339
|
+
const ref = `refs/remotes/origin/${defaultBranch}`;
|
|
340
|
+
const git = (args) => exec("git", args, { cwd: repo });
|
|
341
|
+
const listed = await git(["ls-tree", "--name-only", ref, "--", PASSKEYS_FILE]);
|
|
342
|
+
const text = listed === "" ? null : await git(["show", `${ref}:${PASSKEYS_FILE}`]);
|
|
343
|
+
let main = [];
|
|
344
|
+
try {
|
|
345
|
+
main = text === null ? [] : parsePasskeys(text);
|
|
346
|
+
} catch (err) {
|
|
347
|
+
throw new Error(`${PASSKEYS_FILE} on ${defaultBranch} is invalid: ${err.message}`);
|
|
348
|
+
}
|
|
349
|
+
const ids = new Set(main.map((k) => k.id));
|
|
350
|
+
const pending = readPending().filter((k) => !ids.has(k.id));
|
|
351
|
+
return { main, pending, trusted: main.length > 0 ? main : pending };
|
|
352
|
+
}
|
|
353
|
+
/** The passkey state the targets screen shows, read when the target list refreshes. */
|
|
354
|
+
let passkeyState = { main: [], pending: [], problem: null };
|
|
355
|
+
const readPasskeys = async () => {
|
|
356
|
+
if (results) {
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
try {
|
|
360
|
+
passkeyState = { ...(await knownKeys()), problem: null };
|
|
361
|
+
} catch (err) {
|
|
362
|
+
passkeyState = { main: [], pending: [], problem: err.message };
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
/** An open pull request that registers a passkey, waiting for the owner to merge it. */
|
|
366
|
+
let passkeyPr = null;
|
|
367
|
+
/** The passkey pull request this server opened: listed until GitHub's list or the file has it. */
|
|
368
|
+
let passkeyOpened = null;
|
|
194
369
|
const busy = (t) => finishing && job?.target === t.id;
|
|
195
370
|
const BUSY = "a Finish is running on this target: wait for it to end";
|
|
196
371
|
|
|
@@ -200,62 +375,285 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
200
375
|
return p?.results?.items.find((i) => i.file === key.slice(at + 1));
|
|
201
376
|
};
|
|
202
377
|
const stateFile = (t) => join(stateDir, `${t.id}.json`);
|
|
378
|
+
const warned = new Set();
|
|
379
|
+
const warnOnce = (message) => {
|
|
380
|
+
if (!warned.has(message)) {
|
|
381
|
+
warned.add(message);
|
|
382
|
+
console.error(`visual-review: ${message}`);
|
|
383
|
+
}
|
|
384
|
+
};
|
|
203
385
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
386
|
+
/**
|
|
387
|
+
* A target's saved decisions by `<project>/<file>`, each with the hash of the image it was
|
|
388
|
+
* taken on; `bulk` marks an accept from Accept all that was never opened one by one, `posted`
|
|
389
|
+
* a reject an earlier Finish already commented. A file that is not a decisions file is moved
|
|
390
|
+
* aside, never overwritten, so the owner can still recover what it held.
|
|
391
|
+
* @param {object} t the target
|
|
392
|
+
* @returns {Record<string, { decision: string, reason: string | null, hash: string | null,
|
|
393
|
+
* bulk?: true, posted?: true }>} the decisions
|
|
394
|
+
*/
|
|
395
|
+
const readState = (t) => {
|
|
396
|
+
const file = stateFile(t);
|
|
397
|
+
let saved;
|
|
398
|
+
try {
|
|
399
|
+
saved = JSON.parse(readFileSync(file, "utf8"));
|
|
400
|
+
} catch (err) {
|
|
401
|
+
if (err.code === "ENOENT") {
|
|
402
|
+
return {};
|
|
403
|
+
}
|
|
404
|
+
if (!(err instanceof SyntaxError)) {
|
|
405
|
+
throw err;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
if (typeof saved !== "object" || saved === null || Array.isArray(saved)) {
|
|
409
|
+
const aside = `${file}.unreadable-${Date.now()}`;
|
|
410
|
+
renameSync(file, aside);
|
|
411
|
+
stateProblems.set(t.id, `the saved decisions were unreadable; the file was moved to ${aside}`);
|
|
412
|
+
console.error(`visual-review: ${file} is not a decisions file: moved it to ${aside}`);
|
|
413
|
+
return {};
|
|
414
|
+
}
|
|
415
|
+
for (const [k, d] of Object.entries(saved)) {
|
|
416
|
+
if (typeof d !== "object" || d === null || typeof d.decision !== "string") {
|
|
417
|
+
warnOnce(`${file}: ignoring the malformed entry ${JSON.stringify(k)}`);
|
|
418
|
+
delete saved[k];
|
|
219
419
|
}
|
|
220
|
-
decisions.set(key, mine);
|
|
221
420
|
}
|
|
222
|
-
return
|
|
421
|
+
return saved;
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Changes a target's saved decisions. The file is read, changed and written in one go, so a
|
|
426
|
+
* failed write changes nothing and a second server on the same directory is not overwritten.
|
|
427
|
+
* ponytail: no lock between processes; two writes in the same millisecond can still race.
|
|
428
|
+
* @param {object} t the target
|
|
429
|
+
* @param {(saved: Record<string, object>) => void} change edits the decisions in place
|
|
430
|
+
*/
|
|
431
|
+
const update = (t, change) => {
|
|
432
|
+
const saved = readState(t);
|
|
433
|
+
change(saved);
|
|
434
|
+
writeJson(stateFile(t), saved);
|
|
223
435
|
};
|
|
224
436
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
437
|
+
/**
|
|
438
|
+
* The decisions that apply to this run: those whose item is in it, with the image the decision
|
|
439
|
+
* was taken on, and still decidable that way. The others stay in the file.
|
|
440
|
+
* @param {object} t the target
|
|
441
|
+
* @returns {Map<string, { decision: string, reason: string | null, bulk?: true, posted?: true }>}
|
|
442
|
+
* by `<project>/<file>`
|
|
443
|
+
*/
|
|
444
|
+
const decisionsOf = (t) => {
|
|
445
|
+
const mine = new Map();
|
|
446
|
+
for (const [k, { hash, ...d }] of Object.entries(readState(t))) {
|
|
447
|
+
const item = itemOf(t, k);
|
|
448
|
+
const project = k.slice(0, k.indexOf("/"));
|
|
449
|
+
if (
|
|
450
|
+
item &&
|
|
451
|
+
imageHash(item) === hash &&
|
|
452
|
+
!decisionProblem(item, d.decision, d.reason ?? null) &&
|
|
453
|
+
(d.decision === "reject" || acceptable(t, project))
|
|
454
|
+
) {
|
|
455
|
+
mine.set(k, d);
|
|
456
|
+
}
|
|
229
457
|
}
|
|
230
|
-
|
|
231
|
-
writeFileSync(stateFile(t), JSON.stringify(out, null, 2));
|
|
458
|
+
return mine;
|
|
232
459
|
};
|
|
233
460
|
|
|
234
461
|
// `problem` is what CI said (the job failed, or no artifact); results.json can add its own.
|
|
235
462
|
async function project(name, dir, problem) {
|
|
236
463
|
const loaded = dir ? await loadResults(dir) : { results: null, problem: null };
|
|
464
|
+
if (loaded.results) {
|
|
465
|
+
prewarm(dir, loaded.results);
|
|
466
|
+
}
|
|
237
467
|
return { project: name, dir, results: loaded.results, problem: problem ?? loaded.problem, logUrl: null };
|
|
238
468
|
}
|
|
239
469
|
|
|
470
|
+
/**
|
|
471
|
+
* The gh runner for one finished run: what GitHub says about it (its jobs, its artifacts) no
|
|
472
|
+
* longer changes, so each answer is asked once and kept on disk beside the run's downloads,
|
|
473
|
+
* and pruned with them. A refresh then asks GitHub only for the pull requests and their newest
|
|
474
|
+
* runs. ponytail: an artifact that expires later is still listed as there; its download fails
|
|
475
|
+
* and says so, which only matters for a capture never downloaded in its 90 days.
|
|
476
|
+
* @param {{ id: number, attempt: number }} run the finished run
|
|
477
|
+
* @returns {Function} the gh runner
|
|
478
|
+
*/
|
|
479
|
+
const kept = (run) => async (args, input) => {
|
|
480
|
+
if (args[0] !== "api") {
|
|
481
|
+
return gh(args, input);
|
|
482
|
+
}
|
|
483
|
+
const name = createHash("sha256").update(args.join(" ")).digest("hex").slice(0, 16);
|
|
484
|
+
const file = join(tmp, `${run.id}-${run.attempt}`, "gh", `${name}.json`);
|
|
485
|
+
const saved = await readFile(file, "utf8").catch(() => null);
|
|
486
|
+
if (saved !== null) {
|
|
487
|
+
return saved;
|
|
488
|
+
}
|
|
489
|
+
const answer = await gh(args, input);
|
|
490
|
+
try {
|
|
491
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
492
|
+
writeFileSync(`${file}.tmp`, answer);
|
|
493
|
+
renameSync(`${file}.tmp`, file);
|
|
494
|
+
} catch (err) {
|
|
495
|
+
warnOnce(`could not keep GitHub's answers in ${dirname(file)}: ${err.message}`);
|
|
496
|
+
}
|
|
497
|
+
return answer;
|
|
498
|
+
};
|
|
499
|
+
|
|
240
500
|
async function build(info, run) {
|
|
241
|
-
const
|
|
242
|
-
const
|
|
501
|
+
const g = run.status === "completed" ? kept(run) : gh;
|
|
502
|
+
const jobs = await visualJobs(g, run, run.attempt, names);
|
|
503
|
+
const others = [];
|
|
504
|
+
// Each project's capture as its download lands; `partial` is the target listed while the
|
|
505
|
+
// rest are still downloading, whose rows fill in one by one. `download` counts the
|
|
506
|
+
// artifacts and their bytes, for the page's progress.
|
|
507
|
+
const landed = new Map();
|
|
508
|
+
const download = { done: 0, total: names.length, bytesDone: 0, bytes: null, startedAt: Date.now() };
|
|
509
|
+
let sizes = {};
|
|
510
|
+
let partial = null;
|
|
511
|
+
const fill = async (name) => {
|
|
512
|
+
const p = await project(name, landed.get(name)?.dir, problemOf(name, landed.get(name), run, jobs));
|
|
513
|
+
p.logUrl = jobs[name]?.url ?? run.url;
|
|
514
|
+
p.bytes = sizes[name] ?? null;
|
|
515
|
+
partial.projects[names.indexOf(name)] = p;
|
|
516
|
+
await decorate(partial);
|
|
517
|
+
};
|
|
518
|
+
const downloads = downloadCaptures(
|
|
519
|
+
g,
|
|
520
|
+
run,
|
|
521
|
+
names,
|
|
522
|
+
tmp,
|
|
523
|
+
others,
|
|
524
|
+
(name, got) => {
|
|
525
|
+
landed.set(name, got);
|
|
526
|
+
if (got) {
|
|
527
|
+
download.done++;
|
|
528
|
+
download.bytesDone += got.bytes;
|
|
529
|
+
}
|
|
530
|
+
if (partial) {
|
|
531
|
+
fill(name).catch((err) => console.error(`visual-review: ${name} of ${info.id}: ${err.message}`));
|
|
532
|
+
}
|
|
533
|
+
},
|
|
534
|
+
(planned) => {
|
|
535
|
+
sizes = planned;
|
|
536
|
+
download.total = Object.keys(planned).length;
|
|
537
|
+
download.bytes = Object.values(planned).reduce((a, b) => a + b, 0);
|
|
538
|
+
},
|
|
539
|
+
);
|
|
540
|
+
downloads.catch(() => {}); // A download still running after PATIENCE fails on a later refresh.
|
|
541
|
+
const downloaded = await Promise.race([
|
|
542
|
+
downloads,
|
|
543
|
+
new Promise((resolve) => setTimeout(resolve, PATIENCE, null).unref()),
|
|
544
|
+
]);
|
|
545
|
+
if (!downloaded) {
|
|
546
|
+
// Listed as downloading, and rebuilt in the cache when the download lands, so the page
|
|
547
|
+
// (which asks again every few seconds) fills the rows in without a refresh.
|
|
548
|
+
downloads
|
|
549
|
+
.then(async () => {
|
|
550
|
+
await refreshing?.catch(() => {});
|
|
551
|
+
const shown = targets.get(info.id);
|
|
552
|
+
if (shown?.downloading && shown.runId === run.id && shown.runAttempt === run.attempt) {
|
|
553
|
+
const built = await build(info, run);
|
|
554
|
+
await decorate(built);
|
|
555
|
+
if (targets.get(info.id) === shown) {
|
|
556
|
+
targets.set(info.id, built);
|
|
557
|
+
save();
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
})
|
|
561
|
+
.catch((err) => console.error(`visual-review: rebuilding ${info.id} failed: ${err.message}`));
|
|
562
|
+
partial = blank(info, null, run);
|
|
563
|
+
partial.downloading = true;
|
|
564
|
+
partial.download = download;
|
|
565
|
+
for (const p of partial.projects) {
|
|
566
|
+
p.downloading = true;
|
|
567
|
+
p.bytes = sizes[p.project] ?? null;
|
|
568
|
+
}
|
|
569
|
+
for (const name of landed.keys()) {
|
|
570
|
+
await fill(name);
|
|
571
|
+
}
|
|
572
|
+
return partial;
|
|
573
|
+
}
|
|
243
574
|
const list = [];
|
|
244
575
|
for (const name of names) {
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
const p = await project(
|
|
248
|
-
name,
|
|
249
|
-
downloaded[name]?.dir,
|
|
250
|
-
failed ? (job ? "capture failed" : "no capture") : null,
|
|
251
|
-
);
|
|
252
|
-
p.logUrl = job?.url ?? run.url;
|
|
576
|
+
const p = await project(name, downloaded[name]?.dir, problemOf(name, downloaded[name], run, jobs));
|
|
577
|
+
p.logUrl = jobs[name]?.url ?? run.url;
|
|
253
578
|
list.push(p);
|
|
254
579
|
}
|
|
255
|
-
|
|
580
|
+
const warnings = others.map(
|
|
581
|
+
(p) =>
|
|
582
|
+
`the run captured ${p}, which this server's ${CONFIG_FILE} does not list: serve from a checkout that has it`,
|
|
583
|
+
);
|
|
584
|
+
return { ...info, runId: run.id, runAttempt: run.attempt, runUrl: run.url, projects: list, warnings };
|
|
256
585
|
}
|
|
257
586
|
|
|
258
|
-
|
|
587
|
+
// Why a project's capture is missing, from its download and its CI job; null when it is there.
|
|
588
|
+
function problemOf(name, got, run, jobs) {
|
|
589
|
+
const job = jobs[name];
|
|
590
|
+
if (got?.error) {
|
|
591
|
+
return retryLater("download failed", got.error);
|
|
592
|
+
}
|
|
593
|
+
if (got?.expired) {
|
|
594
|
+
return "artifact expired: re-run the visual job";
|
|
595
|
+
}
|
|
596
|
+
if (!got && run.status !== "completed" && job?.conclusion !== "failure") {
|
|
597
|
+
return "CI still running; reload when it finishes";
|
|
598
|
+
}
|
|
599
|
+
if (!got || job?.conclusion === "failure") {
|
|
600
|
+
return job ? "capture failed" : "no capture";
|
|
601
|
+
}
|
|
602
|
+
return null;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const retryLater = (what, message) => `${what}: ${message.split("\n")[0]}; reload the page to retry`;
|
|
606
|
+
|
|
607
|
+
// A target with no capture to show: GitHub would not give it to us (after gh's retries), CI has
|
|
608
|
+
// not run yet, or its download is still going. The other targets still load.
|
|
609
|
+
const blank = (info, problem, run = null) => ({
|
|
610
|
+
...info,
|
|
611
|
+
runId: run?.id ?? null,
|
|
612
|
+
runAttempt: run?.attempt ?? null,
|
|
613
|
+
runUrl: run?.url ?? null,
|
|
614
|
+
projects: names.map((n) => ({ project: n, dir: null, results: null, problem, logUrl: run?.url ?? null })),
|
|
615
|
+
warnings: [],
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
// A target that loaded before keeps what it showed when a later refresh of it fails.
|
|
619
|
+
const keptOr = (info, err) => {
|
|
620
|
+
const old = targets.get(info.id);
|
|
621
|
+
return old?.runId
|
|
622
|
+
? { ...old, warnings: [retryLater("could not refresh", err.message)] }
|
|
623
|
+
: blank(info, retryLater("failed to load", err.message));
|
|
624
|
+
};
|
|
625
|
+
|
|
626
|
+
// Concurrent callers (the startup refresh, the page's first request) share one refresh.
|
|
627
|
+
let refreshing = null;
|
|
628
|
+
let refreshedAt = 0;
|
|
629
|
+
let loadedOnce = false;
|
|
630
|
+
/** Why the last refresh failed as a whole (no list at all), for the page; null once one works. */
|
|
631
|
+
let failure = null;
|
|
632
|
+
/** What the running refresh is doing, for the page: a step, a count, and when it began. */
|
|
633
|
+
let progress = null;
|
|
634
|
+
const step = (name, done = null, total = null) => {
|
|
635
|
+
if (progress) {
|
|
636
|
+
Object.assign(progress, { step: name, done, total });
|
|
637
|
+
}
|
|
638
|
+
};
|
|
639
|
+
const refresh = () =>
|
|
640
|
+
(refreshing ??= (async () => {
|
|
641
|
+
progress = { step: "starting", done: null, total: null, startedAt: Date.now() };
|
|
642
|
+
try {
|
|
643
|
+
await load();
|
|
644
|
+
loadedOnce = true;
|
|
645
|
+
failure = null;
|
|
646
|
+
} catch (err) {
|
|
647
|
+
failure = err.message.split("\n")[0];
|
|
648
|
+
throw err;
|
|
649
|
+
} finally {
|
|
650
|
+
refreshing = null;
|
|
651
|
+
refreshedAt = Date.now();
|
|
652
|
+
progress = null;
|
|
653
|
+
}
|
|
654
|
+
})());
|
|
655
|
+
|
|
656
|
+
async function load() {
|
|
259
657
|
const next = new Map();
|
|
260
658
|
if (results) {
|
|
261
659
|
const list = await Promise.all(
|
|
@@ -282,58 +680,228 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
282
680
|
});
|
|
283
681
|
}
|
|
284
682
|
} else {
|
|
285
|
-
|
|
683
|
+
// When the list of pull requests cannot be read, the ones listed before stay, and the
|
|
684
|
+
// master seed still loads.
|
|
685
|
+
listWarnings = [];
|
|
686
|
+
const kept = [...targets.values()].filter((t) => t.pr !== null);
|
|
687
|
+
step("listing pull requests");
|
|
688
|
+
const prs = await openPullRequests(gh).catch((err) => {
|
|
689
|
+
if (!masterRun && kept.length === 0) {
|
|
690
|
+
throw err;
|
|
691
|
+
}
|
|
692
|
+
console.error(`visual-review: pull requests not listed: ${err.message}`);
|
|
693
|
+
const warning = retryLater("could not list the pull requests", err.message);
|
|
694
|
+
listWarnings.push(warning);
|
|
695
|
+
for (const t of kept) {
|
|
696
|
+
next.set(t.id, { ...t, warnings: [warning] });
|
|
697
|
+
}
|
|
698
|
+
return [];
|
|
699
|
+
});
|
|
286
700
|
// Best effort: the default branch and the pull requests' branches, so the badge below
|
|
287
|
-
// sees what accept will see.
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
701
|
+
// sees what accept will see. Skipped while a Finish runs, which fetches and pushes too.
|
|
702
|
+
const fetch = (refs) =>
|
|
703
|
+
exec("git", ["fetch", "-q", "origin", ...refs], {
|
|
704
|
+
cwd: repo,
|
|
705
|
+
env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
|
|
706
|
+
});
|
|
707
|
+
const ref = (b) => `+refs/heads/${b}:refs/remotes/origin/${b}`;
|
|
708
|
+
const logFetch = (what) => (err) => {
|
|
709
|
+
console.error(`visual-review: git fetch of ${what} failed: ${err.message}`);
|
|
710
|
+
listWarnings.push(`git fetch of ${what} failed, so "merge master first" may be wrong: ${err.message}`);
|
|
711
|
+
};
|
|
712
|
+
const fetched = (async () => {
|
|
713
|
+
if (finishing) {
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
await fetch([ref(defaultBranch)]).catch(logFetch(defaultBranch));
|
|
717
|
+
// One fetch for every branch; one missing on origin (a fork's, a deleted one) fails
|
|
718
|
+
// them all, so then one at a time.
|
|
719
|
+
if (prs.length > 0) {
|
|
720
|
+
await fetch(prs.map((p) => ref(p.branch))).catch(async () => {
|
|
721
|
+
for (const p of prs) {
|
|
722
|
+
await fetch([ref(p.branch)]).catch(logFetch(`${p.branch} (#${p.number})`));
|
|
723
|
+
}
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
})();
|
|
293
727
|
// Every pull request at once: one after another took about 40 s for 18 of them.
|
|
728
|
+
let found = 0;
|
|
729
|
+
step("finding CI runs", 0, prs.length);
|
|
294
730
|
const built = await Promise.all(
|
|
295
731
|
prs.map(async (pr) => {
|
|
296
|
-
const
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
:
|
|
732
|
+
const info = {
|
|
733
|
+
id: String(pr.number),
|
|
734
|
+
pr: pr.number,
|
|
735
|
+
title: pr.title,
|
|
736
|
+
url: pr.url,
|
|
737
|
+
branch: pr.branch,
|
|
738
|
+
};
|
|
739
|
+
try {
|
|
740
|
+
const run = await newestCiRun(gh, pr.headSha, config);
|
|
741
|
+
return run
|
|
742
|
+
? await build(info, run)
|
|
743
|
+
: blank(info, `waiting for CI on ${pr.headSha.slice(0, 10)}`);
|
|
744
|
+
} catch (err) {
|
|
745
|
+
return keptOr(info, err);
|
|
746
|
+
} finally {
|
|
747
|
+
step("finding CI runs", ++found, prs.length);
|
|
748
|
+
}
|
|
301
749
|
}),
|
|
302
750
|
);
|
|
751
|
+
step("fetching branches");
|
|
303
752
|
await fetched;
|
|
304
|
-
|
|
753
|
+
await readPasskeys();
|
|
754
|
+
// GitHub's list can lag a pull request just opened: keep it a while, until its key is merged.
|
|
755
|
+
const opened =
|
|
756
|
+
passkeyOpened &&
|
|
757
|
+
Date.now() - passkeyOpened.at < 5 * 60000 &&
|
|
758
|
+
!passkeyState.main.some((k) => k.id === passkeyOpened.id)
|
|
759
|
+
? passkeyOpened.number
|
|
760
|
+
: null;
|
|
761
|
+
passkeyPr = prs.find((p) => p.branch.startsWith("visual/passkey-"))?.number ?? opened;
|
|
762
|
+
for (const t of built) next.set(t.id, t);
|
|
305
763
|
if (masterRun) {
|
|
306
|
-
|
|
764
|
+
step(`reading ${defaultBranch}'s CI run`);
|
|
765
|
+
const info = { id: "master", pr: null, title: defaultBranch, url: null, branch: null };
|
|
307
766
|
next.set(
|
|
308
767
|
"master",
|
|
309
|
-
await
|
|
768
|
+
await getRun(gh, masterRun)
|
|
769
|
+
.then((run) => build(info, run))
|
|
770
|
+
.catch((err) => keptOr(info, err)),
|
|
310
771
|
);
|
|
311
772
|
}
|
|
312
773
|
}
|
|
774
|
+
step("checking the baselines", 0, next.size);
|
|
775
|
+
let checked = 0;
|
|
313
776
|
for (const t of next.values()) {
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
t.headSha = first?.headSha ?? null;
|
|
317
|
-
const base = t.pr === null ? t.commit : t.headSha;
|
|
318
|
-
t.earlier = await earlierAccepts(repo, t.headSha, t.pr, config.baselines);
|
|
319
|
-
t.mergeMasterFirst = false;
|
|
320
|
-
for (const p of t.projects) {
|
|
321
|
-
if (
|
|
322
|
-
!t.local &&
|
|
323
|
-
p.results &&
|
|
324
|
-
base &&
|
|
325
|
-
(await behindMaster(repo, base, p.project, config).catch(() => true))
|
|
326
|
-
) {
|
|
327
|
-
t.mergeMasterFirst = true;
|
|
328
|
-
}
|
|
329
|
-
}
|
|
777
|
+
await decorate(t);
|
|
778
|
+
step("checking the baselines", ++checked, next.size);
|
|
330
779
|
}
|
|
331
780
|
signer = await signingIdentity(repo);
|
|
332
781
|
targets = next;
|
|
782
|
+
save();
|
|
783
|
+
prune();
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const listFile = join(stateDir, "list.json");
|
|
787
|
+
/**
|
|
788
|
+
* Keeps the list of targets on disk, without the captures' results (results.json is read again
|
|
789
|
+
* from each capture's directory), so a restarted server shows it before GitHub answers.
|
|
790
|
+
*/
|
|
791
|
+
function save() {
|
|
792
|
+
if (results) {
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
try {
|
|
796
|
+
writeJson(listFile, {
|
|
797
|
+
at: Date.now(),
|
|
798
|
+
targets: [...targets.values()].map((t) => ({
|
|
799
|
+
...t,
|
|
800
|
+
earlier: [...(t.earlier ?? [])],
|
|
801
|
+
projects: t.projects.map((p) => ({ ...p, results: undefined })),
|
|
802
|
+
})),
|
|
803
|
+
});
|
|
804
|
+
} catch (err) {
|
|
805
|
+
const warning = `could not keep the list of targets (a restart would wait for GitHub): ${err.message}`;
|
|
806
|
+
console.error(`visual-review: ${warning}`);
|
|
807
|
+
if (!listWarnings.includes(warning)) {
|
|
808
|
+
listWarnings.push(warning);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
/**
|
|
814
|
+
* The list a previous run of this server kept, shown until the first refresh replaces it. A
|
|
815
|
+
* capture whose directory is gone is listed as downloading: that refresh downloads it again.
|
|
816
|
+
*/
|
|
817
|
+
async function restore() {
|
|
818
|
+
let saved;
|
|
819
|
+
try {
|
|
820
|
+
saved = JSON.parse(readFileSync(listFile, "utf8"));
|
|
821
|
+
} catch {
|
|
822
|
+
return; // Never saved, or unreadable: the first refresh builds the list.
|
|
823
|
+
}
|
|
824
|
+
try {
|
|
825
|
+
const next = new Map();
|
|
826
|
+
for (const t of saved.targets) {
|
|
827
|
+
const list = [];
|
|
828
|
+
for (const p of t.projects) {
|
|
829
|
+
const there = p.dir && existsSync(join(p.dir, "results.json"));
|
|
830
|
+
const loaded = there ? await project(p.project, p.dir, p.problem) : null;
|
|
831
|
+
list.push(
|
|
832
|
+
p.dir && !loaded?.results
|
|
833
|
+
? { ...p, dir: null, results: null, problem: null, downloading: true }
|
|
834
|
+
: { ...p, results: loaded?.results ?? null },
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
const downloading = t.downloading === true || list.some((p) => p.downloading);
|
|
838
|
+
next.set(t.id, { ...t, earlier: new Map(t.earlier), projects: list, downloading });
|
|
839
|
+
}
|
|
840
|
+
signer = await signingIdentity(repo);
|
|
841
|
+
await readPasskeys();
|
|
842
|
+
if (!loadedOnce) {
|
|
843
|
+
targets = next;
|
|
844
|
+
loadedOnce = true;
|
|
845
|
+
refreshedAt = saved.at;
|
|
846
|
+
}
|
|
847
|
+
} catch (err) {
|
|
848
|
+
console.error(`visual-review: ignoring the kept list in ${listFile}: ${err.message}`);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// A target's captured commits, its earlier accepts and whether it is behind the default branch.
|
|
853
|
+
async function decorate(t) {
|
|
854
|
+
const first = t.projects.find((p) => p.results)?.results;
|
|
855
|
+
t.commit = first?.commit ?? null;
|
|
856
|
+
t.headSha = first?.headSha ?? null;
|
|
857
|
+
const base = t.pr === null ? t.commit : t.headSha;
|
|
858
|
+
t.earlier = await earlierAccepts(repo, t.headSha, t.pr, config.baselines);
|
|
859
|
+
// null: unknown, when the captured head was never fetched or git fails.
|
|
860
|
+
const known =
|
|
861
|
+
base !== null &&
|
|
862
|
+
(await exec("git", ["cat-file", "-e", `${base}^{commit}`], { cwd: repo }).then(
|
|
863
|
+
() => true,
|
|
864
|
+
() => false,
|
|
865
|
+
));
|
|
866
|
+
t.mergeMasterFirst = false;
|
|
867
|
+
for (const p of t.projects) {
|
|
868
|
+
if (!t.local && p.results && base) {
|
|
869
|
+
const behind = known ? await behindMaster(repo, base, p.project, config).catch(() => null) : null;
|
|
870
|
+
if (behind !== false && t.mergeMasterFirst !== true) {
|
|
871
|
+
t.mergeMasterFirst = behind;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
// Deletes the downloads of runs no target shows any more (never state/). Not while a Finish
|
|
878
|
+
// reads its captures, nor after a refresh that could not load some target.
|
|
879
|
+
function prune() {
|
|
880
|
+
const runs = new Set([...targets.values()].map((t) => t.runId));
|
|
881
|
+
if (results || finishing || runs.has(null) || !existsSync(tmp)) {
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
for (const d of readdirSync(tmp)) {
|
|
885
|
+
const run = /^(\d+)-\d+$/.exec(d)?.[1];
|
|
886
|
+
if (run && !runs.has(Number(run))) {
|
|
887
|
+
rmSync(join(tmp, d), { recursive: true, force: true });
|
|
888
|
+
}
|
|
889
|
+
}
|
|
333
890
|
}
|
|
334
891
|
|
|
335
892
|
const summary = (t) => {
|
|
336
|
-
const
|
|
893
|
+
const warnings = [...(t.warnings ?? [])];
|
|
894
|
+
let decided = new Map();
|
|
895
|
+
try {
|
|
896
|
+
decided = decisionsOf(t);
|
|
897
|
+
} catch (err) {
|
|
898
|
+
// One unreadable state file never fails the whole list.
|
|
899
|
+
console.error(`visual-review: ${stateFile(t)}: ${err.message}`);
|
|
900
|
+
warnings.push(`could not read the saved decisions: ${err.message}`);
|
|
901
|
+
}
|
|
902
|
+
if (stateProblems.has(t.id)) {
|
|
903
|
+
warnings.push(stateProblems.get(t.id));
|
|
904
|
+
}
|
|
337
905
|
return {
|
|
338
906
|
id: t.id,
|
|
339
907
|
pr: t.pr,
|
|
@@ -347,8 +915,40 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
347
915
|
commit: t.commit,
|
|
348
916
|
headSha: t.headSha,
|
|
349
917
|
mergeMasterFirst: t.mergeMasterFirst,
|
|
918
|
+
downloading: t.downloading === true,
|
|
919
|
+
// While downloading: how many artifacts and bytes have landed of how many, and since when
|
|
920
|
+
// (server time). `bytes` is null until GitHub has listed the artifacts.
|
|
921
|
+
download: t.downloading
|
|
922
|
+
? { done: 0, total: names.length, bytesDone: 0, bytes: null, startedAt: null, ...t.download }
|
|
923
|
+
: null,
|
|
924
|
+
defaultBranch,
|
|
925
|
+
// Decisions the next Finish would publish (a reject an earlier Finish posted is not).
|
|
926
|
+
unpublished: [...decided.values()].filter((d) => !d.posted).length,
|
|
927
|
+
warnings,
|
|
350
928
|
signer,
|
|
351
929
|
startCommand,
|
|
930
|
+
// What the targets screen and Finish's header say about the passkey. The Finish routes
|
|
931
|
+
// read the keys again themselves: this is for display only.
|
|
932
|
+
passkey: {
|
|
933
|
+
rpId,
|
|
934
|
+
required: passkeyState.problem !== null || passkeyState.main.length + passkeyState.pending.length > 0,
|
|
935
|
+
keys: [
|
|
936
|
+
...passkeyState.main.map(({ id, label, rpId: host }) => ({
|
|
937
|
+
id,
|
|
938
|
+
label,
|
|
939
|
+
rpId: host,
|
|
940
|
+
pending: false,
|
|
941
|
+
})),
|
|
942
|
+
...passkeyState.pending.map(({ id, label, rpId: host }) => ({
|
|
943
|
+
id,
|
|
944
|
+
label,
|
|
945
|
+
rpId: host,
|
|
946
|
+
pending: true,
|
|
947
|
+
})),
|
|
948
|
+
],
|
|
949
|
+
problem: passkeyState.problem,
|
|
950
|
+
waiting: passkeyPr,
|
|
951
|
+
},
|
|
352
952
|
projects: t.projects.map((p) => {
|
|
353
953
|
const counts = {};
|
|
354
954
|
for (const item of p.results?.items ?? []) {
|
|
@@ -360,11 +960,16 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
360
960
|
return {
|
|
361
961
|
project: p.project,
|
|
362
962
|
problem: p.problem,
|
|
963
|
+
downloading: p.downloading === true,
|
|
964
|
+
// The size of the project's artifact, while it downloads.
|
|
965
|
+
bytes: p.downloading ? (p.bytes ?? null) : null,
|
|
363
966
|
logUrl: p.logUrl,
|
|
364
967
|
counts,
|
|
365
968
|
reviewable,
|
|
366
969
|
decided: mine.length,
|
|
367
970
|
undecided: reviewable - mine.length,
|
|
971
|
+
// Accepts and exclusions an earlier Finish pushed, waiting for a new CI run.
|
|
972
|
+
finished: mine.filter(([, d]) => d.posted && d.decision !== "reject").length,
|
|
368
973
|
notOpened: mine.filter(([, d]) => d.bulk).length,
|
|
369
974
|
acceptable: acceptable(t, p.project),
|
|
370
975
|
local: p.results?.local ?? null,
|
|
@@ -382,7 +987,7 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
382
987
|
const LOCAL = "is a local preview: nothing is decided on it; only CI captures of a pushed commit are";
|
|
383
988
|
|
|
384
989
|
async function targetOf(id) {
|
|
385
|
-
if (!targets.has(id)) {
|
|
990
|
+
if (!targets.has(id) && (refreshing || Date.now() - refreshedAt > UNKNOWN_REFRESH)) {
|
|
386
991
|
await refresh();
|
|
387
992
|
}
|
|
388
993
|
return targets.get(id);
|
|
@@ -391,21 +996,262 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
391
996
|
async function projectOf(id, name) {
|
|
392
997
|
const t = await targetOf(id);
|
|
393
998
|
const p = t?.projects.find((x) => x.project === name);
|
|
394
|
-
return p?.results ?
|
|
999
|
+
return { t, p: p?.results ? p : undefined };
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
const gone = (id) => [
|
|
1003
|
+
404,
|
|
1004
|
+
{
|
|
1005
|
+
error: `${id === "master" ? defaultBranch : `#${id}`} is not listed any more (closed, or it could not be loaded): reload the list`,
|
|
1006
|
+
},
|
|
1007
|
+
];
|
|
1008
|
+
|
|
1009
|
+
// Whether a write is about the capture the page shows: `hash` (the image), or `runId` and
|
|
1010
|
+
// `runAttempt`. A page that sends neither is not checked.
|
|
1011
|
+
const sameCapture = (t, body, item) =>
|
|
1012
|
+
body.hash !== undefined
|
|
1013
|
+
? body.hash === imageHash(item)
|
|
1014
|
+
: body.runId === undefined || (body.runId === t.runId && body.runAttempt === t.runAttempt);
|
|
1015
|
+
const CHANGED = "the capture changed since the page loaded it (a new CI run or attempt): reload the page";
|
|
1016
|
+
|
|
1017
|
+
/**
|
|
1018
|
+
* The decisions a Finish of `t` would apply now: every one but the rejects an earlier Finish
|
|
1019
|
+
* already posted, in a fixed order, and a digest of them that changes when any of them does.
|
|
1020
|
+
* @param {object} t the target
|
|
1021
|
+
* @returns {{ list: { project: string, file: string, decision: string, reason: string | null }[],
|
|
1022
|
+
* digest: string }} the decisions and their digest
|
|
1023
|
+
*/
|
|
1024
|
+
const finishList = (t) => {
|
|
1025
|
+
const list = [...decisionsOf(t)]
|
|
1026
|
+
.filter(([, v]) => !v.posted)
|
|
1027
|
+
.sort(([a], [b]) => (a < b ? -1 : 1))
|
|
1028
|
+
.map(([k, v]) => {
|
|
1029
|
+
const at = k.indexOf("/");
|
|
1030
|
+
return { project: k.slice(0, at), file: k.slice(at + 1), decision: v.decision, reason: v.reason };
|
|
1031
|
+
});
|
|
1032
|
+
return { list, digest: createHash("sha256").update(JSON.stringify(list)).digest("hex") };
|
|
1033
|
+
};
|
|
1034
|
+
const CHANGED_SINCE = "Decisions changed since this sheet opened: check the summary again.";
|
|
1035
|
+
|
|
1036
|
+
// What Finish's sheet states before it runs: what it commits and posts, and the status it sets.
|
|
1037
|
+
// The passkey's challenge comes from POST /api/finish-prepare.
|
|
1038
|
+
const finishPreview = (t) => {
|
|
1039
|
+
const { list, digest } = finishList(t);
|
|
1040
|
+
const s = summary(t);
|
|
1041
|
+
const count = (d) => list.filter((x) => x.decision === d).length;
|
|
1042
|
+
const unloaded = t.projects.filter((p) => !p.results).map((p) => p.project);
|
|
1043
|
+
const undecided = s.projects.reduce((n, p) => n + p.undecided, 0);
|
|
1044
|
+
const notes = list
|
|
1045
|
+
.filter((x) => x.reason !== null && x.decision !== "exclude")
|
|
1046
|
+
.map((x) => ({ decision: x.decision, project: x.project, file: x.file, note: x.reason }));
|
|
1047
|
+
return {
|
|
1048
|
+
accepts: count("accept"),
|
|
1049
|
+
excludes: count("exclude"),
|
|
1050
|
+
rejects: count("reject"),
|
|
1051
|
+
acceptNotes: notes.filter((n) => n.decision === "accept").length,
|
|
1052
|
+
notes,
|
|
1053
|
+
notOpened: s.projects.reduce((n, p) => n + p.notOpened, 0),
|
|
1054
|
+
undecided: s.projects
|
|
1055
|
+
.filter((p) => p.undecided > 0)
|
|
1056
|
+
.map((p) => ({ project: p.project, undecided: p.undecided })),
|
|
1057
|
+
unloaded,
|
|
1058
|
+
status: commitStatus({
|
|
1059
|
+
accepted: count("accept"),
|
|
1060
|
+
rejected: count("reject"),
|
|
1061
|
+
excluded: count("exclude"),
|
|
1062
|
+
undecided,
|
|
1063
|
+
unloaded,
|
|
1064
|
+
}),
|
|
1065
|
+
digest,
|
|
1066
|
+
};
|
|
1067
|
+
};
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* What a Finish of this target applies: its decisions (a reject an earlier Finish posted stays
|
|
1071
|
+
* shown as rejected, never posted again), their digest, the captures, and what is left
|
|
1072
|
+
* undecided or unloaded.
|
|
1073
|
+
* @param {object} t the target
|
|
1074
|
+
* @returns {{ list: object[], digest: string, captures: object, undecided: number,
|
|
1075
|
+
* unloaded: string[] }} the work
|
|
1076
|
+
*/
|
|
1077
|
+
const finishWork = (t) => ({
|
|
1078
|
+
...finishList(t),
|
|
1079
|
+
captures: capturesOf(t),
|
|
1080
|
+
undecided: summary(t).projects.reduce((n, p) => n + p.undecided, 0),
|
|
1081
|
+
unloaded: t.projects.filter((p) => !p.results).map((p) => p.project),
|
|
1082
|
+
});
|
|
1083
|
+
|
|
1084
|
+
const capturesOf = (t) =>
|
|
1085
|
+
Object.fromEntries(
|
|
1086
|
+
t.projects.filter((p) => p.results).map((p) => [p.project, { dir: p.dir, results: p.results }]),
|
|
1087
|
+
);
|
|
1088
|
+
|
|
1089
|
+
/** Thumbnails being made, by the file they are kept in, and whether a tile is waiting for one. */
|
|
1090
|
+
const making = new Map();
|
|
1091
|
+
|
|
1092
|
+
/**
|
|
1093
|
+
* An image's thumbnail, made in a worker thread unless it is on disk already. A tile waiting
|
|
1094
|
+
* for one (`urgent`) goes ahead of those made in advance, even of its own made in advance.
|
|
1095
|
+
* @param {{ path: string, hash: string, file: string, dir: string }} where the image
|
|
1096
|
+
* @param {boolean} urgent a tile is waiting for it
|
|
1097
|
+
* @returns {Promise<Array<unknown>>} the answer: the PNG, or why not
|
|
1098
|
+
*/
|
|
1099
|
+
function thumbOf(where, urgent) {
|
|
1100
|
+
const kept = join(tmp, "thumbs", `${where.hash}.png`);
|
|
1101
|
+
const was = making.get(kept);
|
|
1102
|
+
if (was && (was.urgent || !urgent)) {
|
|
1103
|
+
return was.done;
|
|
1104
|
+
}
|
|
1105
|
+
const done = (async () => {
|
|
1106
|
+
// Made in advance, it was just found missing: reading here, for a thousand items at once,
|
|
1107
|
+
// would fill Node's file thread pool and stall every other request's file reads.
|
|
1108
|
+
const cached = urgent ? await readFile(kept).catch(() => null) : null;
|
|
1109
|
+
if (cached) {
|
|
1110
|
+
return [200, cached, "image/png"];
|
|
1111
|
+
}
|
|
1112
|
+
let small;
|
|
1113
|
+
try {
|
|
1114
|
+
small = await scaled(async () => {
|
|
1115
|
+
const full = await readImage(where);
|
|
1116
|
+
if (full[0] !== 200) {
|
|
1117
|
+
throw Object.assign(new Error(full[1].error), { answer: full });
|
|
1118
|
+
}
|
|
1119
|
+
return full[1];
|
|
1120
|
+
}, urgent);
|
|
1121
|
+
} catch (err) {
|
|
1122
|
+
if (err.answer) {
|
|
1123
|
+
return err.answer;
|
|
1124
|
+
}
|
|
1125
|
+
throw err;
|
|
1126
|
+
}
|
|
1127
|
+
// Its own temporary name: the advance copy and a tile's may be written at once.
|
|
1128
|
+
const part = `${kept}.${urgent ? "tile" : "ahead"}.tmp`;
|
|
1129
|
+
try {
|
|
1130
|
+
mkdirSync(dirname(kept), { recursive: true });
|
|
1131
|
+
writeFileSync(part, small);
|
|
1132
|
+
renameSync(part, kept);
|
|
1133
|
+
} catch (err) {
|
|
1134
|
+
warnOnce(`could not keep thumbnails in ${dirname(kept)}: ${err.message}`);
|
|
1135
|
+
}
|
|
1136
|
+
return [200, small, "image/png"];
|
|
1137
|
+
})().finally(() => {
|
|
1138
|
+
if (making.get(kept)?.done === done) {
|
|
1139
|
+
making.delete(kept);
|
|
1140
|
+
}
|
|
1141
|
+
});
|
|
1142
|
+
making.set(kept, { urgent, done });
|
|
1143
|
+
return done;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
/**
|
|
1147
|
+
* Makes, in the background, the grid thumbnail of every item of a capture that needs a
|
|
1148
|
+
* decision, so a grid opens with its tiles ready.
|
|
1149
|
+
* @param {string} dir the capture's directory
|
|
1150
|
+
* @param {{ items: object[] }} r its results.json
|
|
1151
|
+
*/
|
|
1152
|
+
function prewarm(dir, r) {
|
|
1153
|
+
for (const item of r.items) {
|
|
1154
|
+
const kind = item.capture ? "capture" : "baseline";
|
|
1155
|
+
const hash = item[kind];
|
|
1156
|
+
if (REVIEWABLE.has(item.status) && hash && !existsSync(join(tmp, "thumbs", `${hash}.png`))) {
|
|
1157
|
+
const own = kind === "capture" || item.baseline === item.capture;
|
|
1158
|
+
const path = own ? join(dir, item.file) : join(dir, "baselines", item.file);
|
|
1159
|
+
thumbOf({ path, hash, file: item.file, dir }, false).catch((err) =>
|
|
1160
|
+
warnOnce(`could not make the thumbnail of ${path}: ${err.message}`),
|
|
1161
|
+
);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// The unpublished count after a write, so the page's Finish button stays current.
|
|
1167
|
+
const unpublishedOf = (t) => finishList(t).list.length;
|
|
1168
|
+
|
|
1169
|
+
/**
|
|
1170
|
+
* Where a target's image is on this disk, by results.json.
|
|
1171
|
+
* @param {string} id the target
|
|
1172
|
+
* @param {string} name the project
|
|
1173
|
+
* @param {string} kind "capture" or "baseline"
|
|
1174
|
+
* @param {string} file the image's file name
|
|
1175
|
+
* @returns {Promise<Array<unknown> | { path: string, hash: string, file: string, dir: string }>}
|
|
1176
|
+
* where the image is, or the answer to send instead
|
|
1177
|
+
*/
|
|
1178
|
+
async function imageOf(id, name, kind, file) {
|
|
1179
|
+
const { t, p } = await projectOf(id, name);
|
|
1180
|
+
if (!t) {
|
|
1181
|
+
return gone(id);
|
|
1182
|
+
}
|
|
1183
|
+
const item = p?.results.items.find((i) => i.file === file);
|
|
1184
|
+
const hash = item && { capture: item.capture, baseline: item.baseline }[kind];
|
|
1185
|
+
if (!hash) {
|
|
1186
|
+
return [404, { error: "no such image" }];
|
|
1187
|
+
}
|
|
1188
|
+
// A moved item's baseline is its capture's bytes, so the artifact holds only the capture.
|
|
1189
|
+
const own = kind === "capture" || item.baseline === item.capture;
|
|
1190
|
+
const path = own ? join(p.dir, file) : join(p.dir, "baselines", file);
|
|
1191
|
+
return { path, hash, file, dir: p.dir };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
async function readImage({ path, hash, file, dir }) {
|
|
1195
|
+
const bytes = await readFile(path).catch(() => null);
|
|
1196
|
+
if (!bytes || createHash("sha256").update(bytes).digest("hex") !== hash) {
|
|
1197
|
+
// CI hashed the bytes it uploaded, so the copy on this disk is damaged: drop it,
|
|
1198
|
+
// and the next reload downloads it again.
|
|
1199
|
+
console.error(
|
|
1200
|
+
`visual-review: ${path} does not match results.json: downloading it again on the next reload`,
|
|
1201
|
+
);
|
|
1202
|
+
rmSync(join(dir, "results.json"), { force: true });
|
|
1203
|
+
return [409, { error: `${file}: the downloaded copy is damaged; reload the page to download it again` }];
|
|
1204
|
+
}
|
|
1205
|
+
return [200, bytes, "image/png"];
|
|
395
1206
|
}
|
|
396
1207
|
|
|
397
1208
|
const routes = {
|
|
398
|
-
"GET /api/prs": async () => {
|
|
399
|
-
|
|
400
|
-
|
|
1209
|
+
"GET /api/prs": async (_, __, query) => {
|
|
1210
|
+
// ?cached=1 answers at once (starting the first load if there is none); ?refresh=1
|
|
1211
|
+
// also starts a refresh. Neither waits for GitHub. After a first load that failed, only
|
|
1212
|
+
// ?refresh=1 (the page's Retry) tries again.
|
|
1213
|
+
await restored;
|
|
1214
|
+
const cachedOnly = query?.get("cached") === "1" || query?.get("refresh") === "1";
|
|
1215
|
+
if (cachedOnly) {
|
|
1216
|
+
if (query.get("refresh") === "1" || (!loadedOnce && failure === null)) {
|
|
1217
|
+
refresh().catch((err) => console.error(`visual-review: refresh failed: ${err.message}`));
|
|
1218
|
+
}
|
|
1219
|
+
} else {
|
|
1220
|
+
await refresh();
|
|
1221
|
+
}
|
|
1222
|
+
return [
|
|
1223
|
+
200,
|
|
1224
|
+
{
|
|
1225
|
+
targets: loadedOnce || !cachedOnly ? [...targets.values()].map(summary) : null,
|
|
1226
|
+
warning: listWarnings.join("\n") || null,
|
|
1227
|
+
defaultBranch,
|
|
1228
|
+
updatedAt: refreshedAt || null,
|
|
1229
|
+
now: Date.now(),
|
|
1230
|
+
refreshing: progress,
|
|
1231
|
+
// Why the last refresh got no list at all; a gh call waiting to retry.
|
|
1232
|
+
error: failure,
|
|
1233
|
+
network: networkTrouble(),
|
|
1234
|
+
},
|
|
1235
|
+
];
|
|
401
1236
|
},
|
|
402
|
-
// One target's counts without refetching from GitHub,
|
|
403
|
-
"GET /api/target": async ([id]) => {
|
|
1237
|
+
// One target's counts without refetching from GitHub; with ?finish=1, what Finish would do.
|
|
1238
|
+
"GET /api/target": async ([id], _, query) => {
|
|
404
1239
|
const t = await targetOf(id);
|
|
405
|
-
|
|
1240
|
+
if (!t) {
|
|
1241
|
+
return gone(id);
|
|
1242
|
+
}
|
|
1243
|
+
return [200, query?.get("finish") === "1" ? { ...summary(t), finish: finishPreview(t) } : summary(t)];
|
|
406
1244
|
},
|
|
407
1245
|
"GET /api/pr": async ([id, name]) => {
|
|
408
1246
|
const { t, p } = await projectOf(id, name);
|
|
1247
|
+
if (!t) {
|
|
1248
|
+
return gone(id);
|
|
1249
|
+
}
|
|
1250
|
+
if (t.projects.find((x) => x.project === name)?.downloading) {
|
|
1251
|
+
// The reviewer is waiting for this one: it downloads next. The page asks again.
|
|
1252
|
+
hurry((dir) => basename(dir) === name && dir.startsWith(join(tmp, `${t.runId}-`)));
|
|
1253
|
+
return [202, { downloading: true, target: summary(t), network: networkTrouble(), now: Date.now() }];
|
|
1254
|
+
}
|
|
409
1255
|
if (!p) {
|
|
410
1256
|
return [404, { error: "no such capture" }];
|
|
411
1257
|
}
|
|
@@ -428,22 +1274,22 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
428
1274
|
];
|
|
429
1275
|
},
|
|
430
1276
|
"GET /api/img": async ([id, name, kind, file]) => {
|
|
431
|
-
const
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
return [409, { error: `${file} does not match results.json` }];
|
|
442
|
-
}
|
|
443
|
-
return [200, bytes, "image/png"];
|
|
1277
|
+
const where = await imageOf(id, name, kind, file);
|
|
1278
|
+
return Array.isArray(where) ? where : readImage(where);
|
|
1279
|
+
},
|
|
1280
|
+
// The same image scaled down for a grid tile, made once and kept by the image's hash.
|
|
1281
|
+
"GET /api/thumb": async ([id, name, kind, file]) => {
|
|
1282
|
+
const where = await imageOf(id, name, kind, file);
|
|
1283
|
+
if (Array.isArray(where)) {
|
|
1284
|
+
return where;
|
|
1285
|
+
}
|
|
1286
|
+
return thumbOf(where, true);
|
|
444
1287
|
},
|
|
445
1288
|
"POST /api/decide": async (_, body) => {
|
|
446
1289
|
const { p, t } = await projectOf(String(body.id), body.project);
|
|
1290
|
+
if (!t) {
|
|
1291
|
+
return gone(String(body.id));
|
|
1292
|
+
}
|
|
447
1293
|
const item = p?.results.items.find((i) => i.file === body.file);
|
|
448
1294
|
if (!item) {
|
|
449
1295
|
return [404, { error: "no such item" }];
|
|
@@ -456,8 +1302,22 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
456
1302
|
return [403, { error: `${body.project} ${LOCAL}` }];
|
|
457
1303
|
}
|
|
458
1304
|
if (body.decision === null) {
|
|
459
|
-
|
|
460
|
-
|
|
1305
|
+
update(t, (saved) => {
|
|
1306
|
+
delete saved[key];
|
|
1307
|
+
});
|
|
1308
|
+
return [200, { ok: true, unpublished: unpublishedOf(t) }];
|
|
1309
|
+
}
|
|
1310
|
+
if (!sameCapture(t, body, item)) {
|
|
1311
|
+
return [409, { error: `${body.file}: ${CHANGED}` }];
|
|
1312
|
+
}
|
|
1313
|
+
if (body.opened === true) {
|
|
1314
|
+
// Opening an item Accept all decided marks it opened. It never decides anything,
|
|
1315
|
+
// so a stale page cannot bring back a decision undone or finished meanwhile.
|
|
1316
|
+
if (decisionsOf(t).get(key)?.bulk) {
|
|
1317
|
+
update(t, (saved) => {
|
|
1318
|
+
delete saved[key].bulk;
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
461
1321
|
return [200, { ok: true }];
|
|
462
1322
|
}
|
|
463
1323
|
if (body.decision !== "reject" && !acceptable(t, body.project)) {
|
|
@@ -480,15 +1340,22 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
480
1340
|
const done = { accept: "accepted", reject: "rejected", exclude: "excluded" }[before.decision];
|
|
481
1341
|
return [409, { error: `${body.file} is already ${done}: Undo it first to change it` }];
|
|
482
1342
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
1343
|
+
update(t, (saved) => {
|
|
1344
|
+
saved[key] = { decision: body.decision, reason, hash: imageHash(item) };
|
|
1345
|
+
});
|
|
1346
|
+
return [200, { ok: true, unpublished: unpublishedOf(t) }];
|
|
486
1347
|
},
|
|
487
1348
|
"POST /api/accept-all": async (_, body) => {
|
|
488
1349
|
const { p, t } = await projectOf(String(body.id), body.project);
|
|
1350
|
+
if (!t) {
|
|
1351
|
+
return gone(String(body.id));
|
|
1352
|
+
}
|
|
489
1353
|
if (!p) {
|
|
490
1354
|
return [404, { error: "no such capture" }];
|
|
491
1355
|
}
|
|
1356
|
+
if (!sameCapture(t, { runId: body.runId, runAttempt: body.runAttempt }, null)) {
|
|
1357
|
+
return [409, { error: CHANGED }];
|
|
1358
|
+
}
|
|
492
1359
|
if (busy(t)) {
|
|
493
1360
|
return [409, { error: BUSY }];
|
|
494
1361
|
}
|
|
@@ -504,20 +1371,99 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
504
1371
|
const inScope = (item) => typeof body.component !== "string" || componentOf(item.id) === body.component;
|
|
505
1372
|
const mine = decisionsOf(t);
|
|
506
1373
|
let accepted = 0;
|
|
507
|
-
|
|
508
|
-
const
|
|
509
|
-
|
|
510
|
-
mine.
|
|
511
|
-
|
|
1374
|
+
update(t, (saved) => {
|
|
1375
|
+
for (const item of p.results.items) {
|
|
1376
|
+
const key = `${body.project}/${item.file}`;
|
|
1377
|
+
if (inScope(item) && !mine.has(key) && !decisionProblem(item, "accept", null)) {
|
|
1378
|
+
saved[key] = { decision: "accept", reason: null, bulk: true, hash: imageHash(item) };
|
|
1379
|
+
accepted++;
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
});
|
|
1383
|
+
return [200, { accepted, unpublished: unpublishedOf(t) }];
|
|
1384
|
+
},
|
|
1385
|
+
"GET /api/passkeys": async () => {
|
|
1386
|
+
const { main, pending } = await knownKeys();
|
|
1387
|
+
const show = (k, isPending) => ({ id: k.id, rpId: k.rpId, label: k.label ?? null, pending: isPending });
|
|
1388
|
+
return [
|
|
1389
|
+
200,
|
|
1390
|
+
{
|
|
1391
|
+
rpId,
|
|
1392
|
+
keys: [...main.map((k) => show(k, false)), ...pending.map((k) => show(k, true))],
|
|
1393
|
+
required: main.length + pending.length > 0,
|
|
1394
|
+
},
|
|
1395
|
+
];
|
|
1396
|
+
},
|
|
1397
|
+
"POST /api/passkey-challenge": async () => {
|
|
1398
|
+
if (results) {
|
|
1399
|
+
return [403, { error: "a local preview registers no passkey" }];
|
|
1400
|
+
}
|
|
1401
|
+
const now = Date.now();
|
|
1402
|
+
for (const [c, expires] of challenges) {
|
|
1403
|
+
if (expires < now) {
|
|
1404
|
+
challenges.delete(c);
|
|
512
1405
|
}
|
|
513
1406
|
}
|
|
514
|
-
|
|
515
|
-
|
|
1407
|
+
const challenge = randomBytes(32).toString("base64url");
|
|
1408
|
+
challenges.set(challenge, now + CHALLENGE_MS);
|
|
1409
|
+
const { main, pending } = await knownKeys();
|
|
1410
|
+
return [
|
|
1411
|
+
200,
|
|
1412
|
+
{
|
|
1413
|
+
challenge,
|
|
1414
|
+
userId: randomBytes(16).toString("base64url"),
|
|
1415
|
+
rpId,
|
|
1416
|
+
known: [...main, ...pending].map((k) => k.id),
|
|
1417
|
+
},
|
|
1418
|
+
];
|
|
516
1419
|
},
|
|
517
|
-
"POST /api/
|
|
1420
|
+
"POST /api/register": async (_, body) => {
|
|
1421
|
+
const expires = challenges.get(body.challenge);
|
|
1422
|
+
challenges.delete(body.challenge);
|
|
1423
|
+
if (!expires || expires < Date.now()) {
|
|
1424
|
+
return [409, { error: "the registration expired or was already used: press Register passkey again" }];
|
|
1425
|
+
}
|
|
1426
|
+
const why = verifyRegistration(body, { challenge: body.challenge, origin, rpId });
|
|
1427
|
+
if (why) {
|
|
1428
|
+
return [400, { error: `the passkey was not accepted: ${why}` }];
|
|
1429
|
+
}
|
|
1430
|
+
const { main, pending } = await knownKeys();
|
|
1431
|
+
if ([...main, ...pending].some((k) => k.id === body.credentialId)) {
|
|
1432
|
+
return [409, { error: "this passkey is already registered" }];
|
|
1433
|
+
}
|
|
1434
|
+
if (finishing) {
|
|
1435
|
+
return [409, { error: "a Finish is running: register when it ends" }];
|
|
1436
|
+
}
|
|
1437
|
+
const label = typeof body.label === "string" ? body.label.replace(/\s+/g, " ").trim().slice(0, 60) : "";
|
|
1438
|
+
const entry = {
|
|
1439
|
+
id: body.credentialId,
|
|
1440
|
+
publicKey: body.publicKey,
|
|
1441
|
+
rpId,
|
|
1442
|
+
label: label || "passkey",
|
|
1443
|
+
registeredAt: new Date().toISOString(),
|
|
1444
|
+
};
|
|
1445
|
+
let proposed;
|
|
1446
|
+
try {
|
|
1447
|
+
proposed = await proposeKey({ repo, gh, entry, config });
|
|
1448
|
+
} catch (err) {
|
|
1449
|
+
if (err instanceof AcceptError) {
|
|
1450
|
+
return [409, { error: err.message }];
|
|
1451
|
+
}
|
|
1452
|
+
throw err;
|
|
1453
|
+
}
|
|
1454
|
+
writeJson(pendingFile, { version: 1, keys: [...readPending(), entry] });
|
|
1455
|
+
passkeyPr = Number(proposed.pullRequest.split("/").pop()) || passkeyPr;
|
|
1456
|
+
passkeyOpened = { number: passkeyPr, id: entry.id, at: Date.now() };
|
|
1457
|
+
await readPasskeys();
|
|
1458
|
+
console.log(
|
|
1459
|
+
`visual-review: registered passkey ${entry.id}; merge ${proposed.pullRequest} to enforce approvals`,
|
|
1460
|
+
);
|
|
1461
|
+
return [200, { entry, ...proposed }];
|
|
1462
|
+
},
|
|
1463
|
+
"POST /api/finish-prepare": async (_, body) => {
|
|
518
1464
|
const t = await targetOf(String(body.id));
|
|
519
1465
|
if (!t) {
|
|
520
|
-
return
|
|
1466
|
+
return gone(String(body.id));
|
|
521
1467
|
}
|
|
522
1468
|
if (t.local) {
|
|
523
1469
|
return [403, { error: "a local preview has no Finish" }];
|
|
@@ -525,36 +1471,129 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
525
1471
|
if (finishing) {
|
|
526
1472
|
return [409, { error: "a Finish is already running" }];
|
|
527
1473
|
}
|
|
528
|
-
const
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
1474
|
+
const { trusted: keys } = await knownKeys();
|
|
1475
|
+
if (keys.length === 0) {
|
|
1476
|
+
approvals.delete(t.id);
|
|
1477
|
+
return [200, { required: false }];
|
|
1478
|
+
}
|
|
1479
|
+
const here = keys.filter((k) => k.rpId === rpId);
|
|
1480
|
+
if (here.length === 0) {
|
|
1481
|
+
const hosts = [...new Set(keys.map((k) => k.rpId))].join(", ");
|
|
1482
|
+
return [
|
|
1483
|
+
409,
|
|
1484
|
+
{
|
|
1485
|
+
error: `no passkey is registered for ${rpId}: serve this page from the host your passkey is for (${hosts}), or register one here`,
|
|
1486
|
+
},
|
|
1487
|
+
];
|
|
1488
|
+
}
|
|
1489
|
+
const work = finishWork(t);
|
|
1490
|
+
let prepared;
|
|
1491
|
+
try {
|
|
1492
|
+
prepared = await prepareRecord({
|
|
1493
|
+
repo,
|
|
1494
|
+
target: { pr: t.pr, branch: t.branch },
|
|
1495
|
+
projects: work.captures,
|
|
1496
|
+
decisions: work.list,
|
|
1497
|
+
now: new Date(),
|
|
1498
|
+
config,
|
|
535
1499
|
});
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
1500
|
+
} catch (err) {
|
|
1501
|
+
if (err instanceof AcceptError) {
|
|
1502
|
+
return [409, { error: err.message }];
|
|
1503
|
+
}
|
|
1504
|
+
throw err;
|
|
1505
|
+
}
|
|
1506
|
+
const challenge = recordHash(prepared.record).toString("base64url");
|
|
1507
|
+
approvals.set(t.id, {
|
|
1508
|
+
challenge,
|
|
1509
|
+
record: prepared.record,
|
|
1510
|
+
work,
|
|
1511
|
+
digest: work.digest,
|
|
1512
|
+
expires: Date.now() + APPROVAL_MS,
|
|
1513
|
+
});
|
|
1514
|
+
return [
|
|
1515
|
+
200,
|
|
1516
|
+
{
|
|
1517
|
+
required: true,
|
|
1518
|
+
challenge,
|
|
1519
|
+
rpId,
|
|
1520
|
+
allowCredentials: here.map((k) => k.id),
|
|
1521
|
+
record: prepared.record,
|
|
1522
|
+
accepts: prepared.accepts,
|
|
1523
|
+
excludes: prepared.excludes,
|
|
1524
|
+
rejects: prepared.rejects,
|
|
1525
|
+
},
|
|
1526
|
+
];
|
|
1527
|
+
},
|
|
1528
|
+
"POST /api/finish": async (_, body) => {
|
|
1529
|
+
const t = await targetOf(String(body.id));
|
|
1530
|
+
if (!t) {
|
|
1531
|
+
return gone(String(body.id));
|
|
1532
|
+
}
|
|
1533
|
+
if (t.local) {
|
|
1534
|
+
return [403, { error: "a local preview has no Finish" }];
|
|
1535
|
+
}
|
|
1536
|
+
if (finishing) {
|
|
1537
|
+
return [409, { error: "a Finish is already running" }];
|
|
1538
|
+
}
|
|
1539
|
+
let work = finishWork(t);
|
|
1540
|
+
// The page's sheet stated what Finish would do; a decision changed since (in another
|
|
1541
|
+
// tab, say) would make that statement wrong.
|
|
1542
|
+
if (body.digest !== undefined && body.digest !== work.digest) {
|
|
1543
|
+
return [409, { error: CHANGED_SINCE }];
|
|
1544
|
+
}
|
|
1545
|
+
let approval = null;
|
|
1546
|
+
const { main, trusted } = await knownKeys();
|
|
1547
|
+
if (trusted.length > 0) {
|
|
1548
|
+
if (!body.approval) {
|
|
1549
|
+
return [400, { error: "this Finish needs your passkey approval (Face ID): press Finish again" }];
|
|
1550
|
+
}
|
|
1551
|
+
const p = approvals.get(t.id);
|
|
1552
|
+
if (!p || p.challenge !== body.challenge || p.expires < Date.now() || p.digest !== work.digest) {
|
|
1553
|
+
return [
|
|
1554
|
+
409,
|
|
1555
|
+
{
|
|
1556
|
+
error: "the approval is stale (made on another tab, expired, or the decisions changed): press Finish again",
|
|
1557
|
+
},
|
|
1558
|
+
];
|
|
1559
|
+
}
|
|
1560
|
+
const record = { ...p.record, approval: body.approval };
|
|
1561
|
+
const why = verifyApproval(record, trusted, { origin });
|
|
1562
|
+
if (why) {
|
|
1563
|
+
return [403, { error: `the approval was refused: ${why}` }];
|
|
1564
|
+
}
|
|
1565
|
+
approvals.delete(t.id);
|
|
1566
|
+
// The decisions the approved record was built from, not a fresh read of the state.
|
|
1567
|
+
work = p.work;
|
|
1568
|
+
approval = { record, pendingKeys: main.length > 0 ? [] : trusted, origin };
|
|
1569
|
+
}
|
|
1570
|
+
const { list, captures, undecided } = work;
|
|
540
1571
|
finishing = true;
|
|
541
1572
|
job = {
|
|
542
1573
|
id: (job?.id ?? 0) + 1,
|
|
543
1574
|
target: t.id,
|
|
544
1575
|
pr: t.pr,
|
|
1576
|
+
branch: t.branch,
|
|
545
1577
|
running: true,
|
|
546
1578
|
step: "starting",
|
|
547
1579
|
result: null,
|
|
548
1580
|
error: null,
|
|
1581
|
+
warnings: [],
|
|
549
1582
|
};
|
|
550
|
-
|
|
1583
|
+
persist(job);
|
|
1584
|
+
console.log(
|
|
1585
|
+
`visual-review: Finish of ${finishLabel(t)} started: ${list.length} decisions, ${undecided} undecided`,
|
|
1586
|
+
);
|
|
1587
|
+
runFinish(job, t, list, {
|
|
551
1588
|
repo,
|
|
552
1589
|
gh,
|
|
553
1590
|
target: { pr: t.pr, branch: t.branch },
|
|
554
1591
|
projects: captures,
|
|
555
1592
|
decisions: list,
|
|
556
1593
|
undecided,
|
|
1594
|
+
unloaded: work.unloaded,
|
|
557
1595
|
config,
|
|
1596
|
+
...(approval && { approval, now: new Date(approval.record.reviewedAt) }),
|
|
558
1597
|
});
|
|
559
1598
|
return [202, { job }];
|
|
560
1599
|
},
|
|
@@ -562,47 +1601,74 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
562
1601
|
};
|
|
563
1602
|
|
|
564
1603
|
/**
|
|
565
|
-
* Runs one Finish to its end, recording its steps and outcome on `j
|
|
1604
|
+
* Runs one Finish to its end, recording its steps and outcome on `j` (and in the log). It
|
|
1605
|
+
* never throws: nothing awaits it.
|
|
566
1606
|
* @param {object} j the job
|
|
567
1607
|
* @param {object} t the target
|
|
568
|
-
* @param {
|
|
1608
|
+
* @param {{ project: string, file: string, decision: string }[]} sent the decisions it applies
|
|
569
1609
|
* @param {object} input finish's input
|
|
570
1610
|
*/
|
|
571
|
-
async function runFinish(j, t,
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
1611
|
+
async function runFinish(j, t, sent, input) {
|
|
1612
|
+
// Marks what Finish published, so no later Finish publishes it again. The accepts and
|
|
1613
|
+
// exclusions it pushed stay shown as finished until a new CI run replaces this capture (their
|
|
1614
|
+
// hash then no longer matches, or the item is unchanged); rejects stay keyed by image hash,
|
|
1615
|
+
// so an unchanged rejected capture on the next CI run still reads as rejected.
|
|
1616
|
+
const clear = (posted) =>
|
|
1617
|
+
update(t, (saved) => {
|
|
1618
|
+
for (const { project, file, decision } of sent) {
|
|
1619
|
+
const k = `${project}/${file}`;
|
|
1620
|
+
if (saved[k] && (decision !== "reject" || posted)) {
|
|
1621
|
+
saved[k].posted = true;
|
|
1622
|
+
}
|
|
581
1623
|
}
|
|
1624
|
+
});
|
|
1625
|
+
try {
|
|
1626
|
+
j.result = await finish({
|
|
1627
|
+
...input,
|
|
1628
|
+
progress: (step) => {
|
|
1629
|
+
j.step = step;
|
|
1630
|
+
persist(j);
|
|
1631
|
+
},
|
|
1632
|
+
});
|
|
1633
|
+
try {
|
|
1634
|
+
clear(true);
|
|
1635
|
+
} catch (err) {
|
|
1636
|
+
// What was pushed and posted stands; the decisions it applied are still saved.
|
|
1637
|
+
j.warnings.push(`pushed and posted, but the decisions could not be cleared: ${err.message}`);
|
|
1638
|
+
console.error(`visual-review: Finish of ${finishLabel(t)}: ${j.warnings.at(-1)}`);
|
|
582
1639
|
}
|
|
583
|
-
|
|
1640
|
+
const r = j.result;
|
|
1641
|
+
const status = r.statusError ? `; status not posted: ${r.statusError}` : "";
|
|
1642
|
+
console.log(
|
|
1643
|
+
`visual-review: Finish of ${finishLabel(t)} done: commit ${r.commit ?? "none"}, ${r.rejects} rejects${status}`,
|
|
1644
|
+
);
|
|
584
1645
|
} catch (err) {
|
|
585
|
-
if (err instanceof AcceptError && err.committed) {
|
|
1646
|
+
if (err instanceof AcceptError && err.committed && !err.pullRequestMissing) {
|
|
586
1647
|
// The accepts are on the branch; keep only the rejects, so Finish again only comments.
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}
|
|
1648
|
+
try {
|
|
1649
|
+
clear(false);
|
|
1650
|
+
} catch (e) {
|
|
1651
|
+
j.warnings.push(`the accepts pushed could not be cleared from the decisions: ${e.message}`);
|
|
1652
|
+
console.error(`visual-review: Finish of ${finishLabel(t)}: ${j.warnings.at(-1)}`);
|
|
591
1653
|
}
|
|
592
|
-
save(t);
|
|
593
1654
|
}
|
|
594
|
-
|
|
1655
|
+
// Refused before anything ran, the message says it all; later, it names the step.
|
|
1656
|
+
j.error = ["starting", "checking"].includes(j.step) ? err.message : `${j.step}: ${err.message}`;
|
|
1657
|
+
console.error(`visual-review: Finish of ${finishLabel(t)} failed: ${j.error}`);
|
|
595
1658
|
} finally {
|
|
596
1659
|
j.running = false;
|
|
597
1660
|
j.step = null;
|
|
598
1661
|
finishing = false;
|
|
1662
|
+
persist(j);
|
|
599
1663
|
}
|
|
600
1664
|
}
|
|
601
1665
|
|
|
1666
|
+
const finishLabel = (t) => (t.pr === null ? "the master seed" : `#${t.pr}`);
|
|
1667
|
+
|
|
602
1668
|
const tokenOk = (given) => {
|
|
603
1669
|
const a = Buffer.from(String(given ?? ""));
|
|
604
1670
|
const b = Buffer.from(token);
|
|
605
|
-
return a.length === b.length && timingSafeEqual(a, b);
|
|
1671
|
+
return b.length > 0 && a.length === b.length && timingSafeEqual(a, b);
|
|
606
1672
|
};
|
|
607
1673
|
|
|
608
1674
|
function send(res, status, body, type = "application/json") {
|
|
@@ -628,6 +1694,16 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
628
1694
|
return body;
|
|
629
1695
|
}
|
|
630
1696
|
|
|
1697
|
+
// The list a previous run of the server kept: read before any request for the list is answered.
|
|
1698
|
+
const restored = warm && !results ? restore() : Promise.resolve();
|
|
1699
|
+
if (warm) {
|
|
1700
|
+
// Download the captures now, and again every POLL so that a CI run that finished meanwhile
|
|
1701
|
+
// is on disk before the page first asks for it.
|
|
1702
|
+
const failed = (when) => (err) => console.error(`visual-review: ${when} refresh failed: ${err.message}`);
|
|
1703
|
+
restored.then(() => refresh().catch(failed("startup")));
|
|
1704
|
+
setInterval(() => refresh().catch(failed("background")), POLL).unref();
|
|
1705
|
+
}
|
|
1706
|
+
|
|
631
1707
|
return async (req, res) => {
|
|
632
1708
|
try {
|
|
633
1709
|
const url = new URL(req.url, origin);
|
|
@@ -651,12 +1727,19 @@ export function createApp({ repo, gh, config, tmp, token, origin, masterRun, res
|
|
|
651
1727
|
if (writes && req.headers.origin !== origin) {
|
|
652
1728
|
return send(res, 403, { error: "foreign origin" });
|
|
653
1729
|
}
|
|
654
|
-
const [status, body, type] = await handler(
|
|
1730
|
+
const [status, body, type] = await handler(
|
|
1731
|
+
args,
|
|
1732
|
+
writes ? await readBody(req) : undefined,
|
|
1733
|
+
url.searchParams,
|
|
1734
|
+
);
|
|
655
1735
|
return send(res, status, body, type);
|
|
656
1736
|
} catch (err) {
|
|
657
1737
|
// A malformed request (bad JSON, bad escape, oversized body) is the client's; the rest,
|
|
658
1738
|
// gh and git failures included, is ours.
|
|
659
1739
|
const client = err instanceof SyntaxError || err instanceof URIError || err.message.startsWith("body ");
|
|
1740
|
+
if (!client) {
|
|
1741
|
+
console.error(`visual-review: ${req.method} ${req.url} failed: ${err.message}`);
|
|
1742
|
+
}
|
|
660
1743
|
return send(res, client ? 400 : 500, { error: err.message });
|
|
661
1744
|
}
|
|
662
1745
|
};
|