@tobi-or-not/quorum 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/bench/fingerprint.d.ts +36 -0
- package/dist/bench/fingerprint.js +79 -0
- package/dist/bench/records.d.ts +62 -0
- package/dist/bench/records.js +161 -0
- package/dist/bench/report.d.ts +110 -0
- package/dist/bench/report.js +332 -0
- package/dist/bench/run-dir.d.ts +101 -0
- package/dist/bench/run-dir.js +379 -0
- package/dist/bench/runner.d.ts +80 -0
- package/dist/bench/runner.js +160 -0
- package/dist/bench/score.d.ts +30 -0
- package/dist/bench/score.js +38 -0
- package/dist/bench/stats.d.ts +51 -0
- package/dist/bench/stats.js +83 -0
- package/dist/bench/suite.d.ts +87 -0
- package/dist/bench/suite.js +286 -0
- package/dist/cli/bench.d.ts +32 -0
- package/dist/cli/bench.js +463 -0
- package/dist/cli/io.d.ts +45 -0
- package/dist/cli/io.js +45 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +17 -67
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +20 -11
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.js +47 -14
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +25 -0
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +8 -3
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +28 -4
- package/package.json +2 -2
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every read and write of a bench run directory, which holds `run.json` (written once, whole),
|
|
3
|
+
* `results.jsonl` (one line per review, appended one at a time), and the reports (`report.md` and
|
|
4
|
+
* `report.json`, rewritten whole). Both data files are read as untrusted, since another run or a person
|
|
5
|
+
* may have written them: strict schemas (see `records.ts`), bounded sizes, and no file that is not a
|
|
6
|
+
* regular file. No write follows a symbolic link: `results.jsonl` is opened with `O_NOFOLLOW`, and every
|
|
7
|
+
* other file is written to a new file that is renamed over the name, which replaces a link there.
|
|
8
|
+
*
|
|
9
|
+
* Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
|
|
10
|
+
* runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
|
|
11
|
+
* I/O failure is an `Error`.
|
|
12
|
+
*/
|
|
13
|
+
import { randomBytes } from 'node:crypto';
|
|
14
|
+
import { constants } from 'node:fs';
|
|
15
|
+
import { lstat, mkdir, open, opendir, rename, unlink } from 'node:fs/promises';
|
|
16
|
+
import { join, resolve } from 'node:path';
|
|
17
|
+
import { decode, readBounded } from "../config/load.js";
|
|
18
|
+
import { ConfigError } from "../core/errors.js";
|
|
19
|
+
import { inertJson } from "../core/sanitize.js";
|
|
20
|
+
import { MAX_RECORD_BYTES, MAX_RUN_FILE_BYTES, parseRecord, parseRunFile } from "./records.js";
|
|
21
|
+
export const RUN_FILE = 'run.json';
|
|
22
|
+
export const RESULTS_FILE = 'results.jsonl';
|
|
23
|
+
export const REPORT_MARKDOWN = 'report.md';
|
|
24
|
+
export const REPORT_JSON = 'report.json';
|
|
25
|
+
/** How the directory is named in an issue: by the flag that names it. */
|
|
26
|
+
const OUT = '--out';
|
|
27
|
+
/** How `runDifferences` words a configuration that was given no path. */
|
|
28
|
+
const DEFAULT_CONFIGURATION = '(default configuration)';
|
|
29
|
+
/** The bytes read from `results.jsonl` at a time. */
|
|
30
|
+
const CHUNK_BYTES = 64 * 1024;
|
|
31
|
+
/** The byte that ends a line; no multi-byte UTF-8 sequence holds it. */
|
|
32
|
+
const LINE_FEED = 0x0A;
|
|
33
|
+
/** An error code worth printing: `EACCES`, never text. */
|
|
34
|
+
const ERROR_CODE = /^E[A-Z]+$/;
|
|
35
|
+
/** Whether the platform can refuse a symbolic link at the open itself. */
|
|
36
|
+
const HAS_NOFOLLOW = typeof constants.O_NOFOLLOW === 'number';
|
|
37
|
+
const NO_FOLLOW = HAS_NOFOLLOW
|
|
38
|
+
? constants.O_NOFOLLOW
|
|
39
|
+
: 0;
|
|
40
|
+
/** Without blocking, so a FIFO is refused instead of waited for. */
|
|
41
|
+
const READ_FLAGS = constants.O_RDONLY | constants.O_NONBLOCK | NO_FOLLOW;
|
|
42
|
+
const APPEND_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | NO_FOLLOW;
|
|
43
|
+
const NEW_FILE_FLAGS = constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL;
|
|
44
|
+
function refusal(path, message) {
|
|
45
|
+
return new ConfigError([
|
|
46
|
+
{ path, message },
|
|
47
|
+
]);
|
|
48
|
+
}
|
|
49
|
+
/** The code of a failed system call, when it is one worth printing. */
|
|
50
|
+
function errorCode(error) {
|
|
51
|
+
const code = error?.code;
|
|
52
|
+
return typeof code === 'string' && ERROR_CODE.test(code)
|
|
53
|
+
? code
|
|
54
|
+
: undefined;
|
|
55
|
+
}
|
|
56
|
+
/** ` (EACCES)`, or nothing for an error without a code worth printing. */
|
|
57
|
+
function codeSuffix(error) {
|
|
58
|
+
const code = errorCode(error);
|
|
59
|
+
return code === undefined
|
|
60
|
+
? ''
|
|
61
|
+
: ` (${code})`;
|
|
62
|
+
}
|
|
63
|
+
function ioFailure(file, what, error) {
|
|
64
|
+
return new Error(`${file}: ${what}${codeSuffix(error)}`);
|
|
65
|
+
}
|
|
66
|
+
/** Whether `directory` holds an entry, of any kind: it reads the first and no more. */
|
|
67
|
+
async function holdsEntry(directory) {
|
|
68
|
+
try {
|
|
69
|
+
const entries = await opendir(directory);
|
|
70
|
+
try {
|
|
71
|
+
return await entries.read() !== null;
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
await entries.close().catch(() => undefined);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
throw ioFailure(OUT, 'cannot be read', error);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/** `empty` for a directory with no `run.json` and no entry; refused when it holds any. */
|
|
82
|
+
async function emptyDirectory(directory) {
|
|
83
|
+
if (await holdsEntry(directory))
|
|
84
|
+
throw refusal(OUT, `holds files but no ${RUN_FILE}, so it is not a run directory: name a new directory, or an empty one`);
|
|
85
|
+
return { kind: 'empty' };
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* What `directory`, the run directory a command was given, holds. A directory that does not exist is
|
|
89
|
+
* `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
|
|
90
|
+
* as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
|
|
91
|
+
* `run.json`, and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
|
|
92
|
+
* format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
|
|
93
|
+
* its parents may be links: they are the user's own path.
|
|
94
|
+
*
|
|
95
|
+
* @throws ConfigError as above; Error when the directory or `run.json` cannot be read
|
|
96
|
+
*/
|
|
97
|
+
export async function inspectRunDir(directory) {
|
|
98
|
+
// Resolved first: `lstat` follows a link when the path ends in a separator or a `.`.
|
|
99
|
+
const location = resolve(directory);
|
|
100
|
+
let stats;
|
|
101
|
+
try {
|
|
102
|
+
stats = await lstat(location);
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
if (errorCode(error) === 'ENOENT')
|
|
106
|
+
return { kind: 'none' };
|
|
107
|
+
throw ioFailure(OUT, 'cannot be read', error);
|
|
108
|
+
}
|
|
109
|
+
if (stats.isSymbolicLink())
|
|
110
|
+
throw refusal(OUT, 'is a symbolic link');
|
|
111
|
+
if (!stats.isDirectory())
|
|
112
|
+
throw refusal(OUT, 'is not a directory');
|
|
113
|
+
const bytes = await readBounded(join(location, RUN_FILE), MAX_RUN_FILE_BYTES, '1 MiB', { noFollow: true });
|
|
114
|
+
if ('problem' in bytes) {
|
|
115
|
+
if (bytes.problem === 'does not exist')
|
|
116
|
+
return emptyDirectory(location);
|
|
117
|
+
throw bytes.problem === 'cannot be read'
|
|
118
|
+
? new Error(`${RUN_FILE}: cannot be read`)
|
|
119
|
+
: refusal(RUN_FILE, bytes.problem);
|
|
120
|
+
}
|
|
121
|
+
const text = decode(bytes.value);
|
|
122
|
+
const run = 'problem' in text
|
|
123
|
+
? undefined
|
|
124
|
+
: parseRunFile(text.value);
|
|
125
|
+
if (run === undefined)
|
|
126
|
+
throw refusal(RUN_FILE, 'is not a run file this quorum writes');
|
|
127
|
+
return { kind: 'run', run };
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
|
|
131
|
+
* quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
|
|
132
|
+
* which holds the preset by name), then another suite by its name, else a suite whose content changed,
|
|
133
|
+
* then each current configuration the run does not hold, at `--config`, then each recorded one that is
|
|
134
|
+
* not given again. A current and a recorded configuration that share a path (`null`, the usual
|
|
135
|
+
* resolution, included) but not a fingerprint are one configuration whose settings changed, and give one
|
|
136
|
+
* issue in place of the two. The same configurations in another order are the same run. A configuration
|
|
137
|
+
* is named by its path as written, or `(default configuration)`: the recorded paths are text from
|
|
138
|
+
* `run.json`, to make inert where they are printed.
|
|
139
|
+
*/
|
|
140
|
+
export function runDifferences(recorded, current) {
|
|
141
|
+
const issues = [];
|
|
142
|
+
if (recorded.quorum !== current.quorum) {
|
|
143
|
+
issues.push({ path: RUN_FILE, message: `was written by quorum ${recorded.quorum}, and this is quorum ${current.quorum}: a run belongs to one build, so name a new ${OUT}` });
|
|
144
|
+
}
|
|
145
|
+
if (recorded.suite.name !== current.suite.name) {
|
|
146
|
+
issues.push({ path: RUN_FILE, message: `is a run of the suite ${recorded.suite.name}, not ${current.suite.name}: name a new ${OUT}` });
|
|
147
|
+
}
|
|
148
|
+
else if (recorded.suite.hash !== current.suite.hash) {
|
|
149
|
+
issues.push({ path: RUN_FILE, message: `the suite's suite.yaml or patches changed since the run began: name a new ${OUT}` });
|
|
150
|
+
}
|
|
151
|
+
const recordedPrints = new Set(recorded.configurations.map(({ fingerprint }) => fingerprint));
|
|
152
|
+
const currentPrints = new Set(current.configurations.map(({ fingerprint }) => fingerprint));
|
|
153
|
+
const changed = new Set();
|
|
154
|
+
for (const { path, fingerprint } of current.configurations) {
|
|
155
|
+
if (recordedPrints.has(fingerprint))
|
|
156
|
+
continue;
|
|
157
|
+
const name = path ?? DEFAULT_CONFIGURATION;
|
|
158
|
+
const before = recorded.configurations.find(entry => entry.path === path && !currentPrints.has(entry.fingerprint) && !changed.has(entry.fingerprint));
|
|
159
|
+
if (before === undefined) {
|
|
160
|
+
issues.push({ path: '--config', message: `${name} is not one of the run's configurations` });
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
changed.add(before.fingerprint);
|
|
164
|
+
issues.push({ path: '--config', message: `the settings of ${name} changed since the run began: name a new ${OUT}` });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
for (const { path, fingerprint } of recorded.configurations) {
|
|
168
|
+
if (!currentPrints.has(fingerprint) && !changed.has(fingerprint))
|
|
169
|
+
issues.push({ path: RUN_FILE, message: `its configuration ${path ?? DEFAULT_CONFIGURATION} is not given` });
|
|
170
|
+
}
|
|
171
|
+
return issues;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Make `directory` and its parents, then write `run.json` into it whole, through a new file and a rename.
|
|
175
|
+
*
|
|
176
|
+
* @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
|
|
177
|
+
*/
|
|
178
|
+
export async function createRun(directory, run) {
|
|
179
|
+
try {
|
|
180
|
+
await mkdir(directory, { recursive: true });
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
throw ioFailure(directory, 'cannot be created', error);
|
|
184
|
+
}
|
|
185
|
+
await writeAtomic(directory, RUN_FILE, `${inertJson(run)}\n`);
|
|
186
|
+
}
|
|
187
|
+
/** `results.jsonl` opened to be read, or undefined when there is none; refused when it is a link or not a file. */
|
|
188
|
+
async function openResults(path) {
|
|
189
|
+
let stats;
|
|
190
|
+
try {
|
|
191
|
+
stats = await lstat(path);
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
if (errorCode(error) === 'ENOENT')
|
|
195
|
+
return undefined;
|
|
196
|
+
throw ioFailure(RESULTS_FILE, 'cannot be read', error);
|
|
197
|
+
}
|
|
198
|
+
if (stats.isSymbolicLink())
|
|
199
|
+
throw refusal(RESULTS_FILE, 'is a symbolic link');
|
|
200
|
+
if (!stats.isFile())
|
|
201
|
+
throw refusal(RESULTS_FILE, 'is not a file');
|
|
202
|
+
try {
|
|
203
|
+
return await open(path, READ_FLAGS);
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
throw errorCode(error) === 'ELOOP'
|
|
207
|
+
? refusal(RESULTS_FILE, 'is a symbolic link')
|
|
208
|
+
: ioFailure(RESULTS_FILE, 'cannot be read', error);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/** The line `bytes` holds, read as a record and handed to `visit`; a line that is none, or that `visit` refuses, is an issue. */
|
|
212
|
+
function visitLine(bytes, line, visit) {
|
|
213
|
+
const text = decode(bytes);
|
|
214
|
+
const record = 'problem' in text
|
|
215
|
+
? undefined
|
|
216
|
+
: parseRecord(text.value);
|
|
217
|
+
if (record === undefined)
|
|
218
|
+
throw refusal(RESULTS_FILE, `line ${line} is not a record of this run`);
|
|
219
|
+
const problem = visit(record, line);
|
|
220
|
+
if (problem !== undefined)
|
|
221
|
+
throw refusal(RESULTS_FILE, `line ${line} ${problem}`);
|
|
222
|
+
}
|
|
223
|
+
function tooLarge(line) {
|
|
224
|
+
return refusal(RESULTS_FILE, `line ${line} is over 16 MiB`);
|
|
225
|
+
}
|
|
226
|
+
/** Read into `chunk` from `position`; the number of bytes read, 0 at the end of the file. */
|
|
227
|
+
async function readInto(handle, chunk, position) {
|
|
228
|
+
try {
|
|
229
|
+
return (await handle.read(chunk, 0, chunk.length, position)).bytesRead;
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
throw ioFailure(RESULTS_FILE, 'cannot be read', error);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Read `results.jsonl` line by line and hand each record, with its line number from 1, to `visit`, which
|
|
237
|
+
* returns the problem with a record it refuses, or undefined. A file with no `results.jsonl` holds
|
|
238
|
+
* nothing. Bytes after the last line break are a line a run killed while it appended cut short: it is
|
|
239
|
+
* reported as `torn`, never visited, and the next append must drop it (see `openRecords`). Any other line
|
|
240
|
+
* that is no record, a line over 16 MiB (complete or not), a record `visit` refuses, a link and a file
|
|
241
|
+
* that is none are refused. The file is read in chunks of 64 KiB and split on the line feed byte, and a
|
|
242
|
+
* line over the limit is refused as soon as it is seen to pass it, so no line is read whole before it is
|
|
243
|
+
* known to fit.
|
|
244
|
+
*
|
|
245
|
+
* @throws ConfigError as above; Error `results.jsonl: cannot be read (<code>)` when a read fails
|
|
246
|
+
*/
|
|
247
|
+
export async function readRecords(directory, visit) {
|
|
248
|
+
const handle = await openResults(join(directory, RESULTS_FILE));
|
|
249
|
+
if (handle === undefined)
|
|
250
|
+
return { lines: 0, validBytes: 0, torn: false };
|
|
251
|
+
try {
|
|
252
|
+
const chunk = Buffer.allocUnsafe(CHUNK_BYTES);
|
|
253
|
+
/** The copies of the bytes of the line in progress that earlier chunks held. */
|
|
254
|
+
let pending = [];
|
|
255
|
+
let pendingBytes = 0;
|
|
256
|
+
let lines = 0;
|
|
257
|
+
let validBytes = 0;
|
|
258
|
+
let position = 0;
|
|
259
|
+
for (;;) {
|
|
260
|
+
const bytesRead = await readInto(handle, chunk, position);
|
|
261
|
+
if (bytesRead === 0)
|
|
262
|
+
break;
|
|
263
|
+
const view = chunk.subarray(0, bytesRead);
|
|
264
|
+
let start = 0;
|
|
265
|
+
for (let end = view.indexOf(LINE_FEED); end !== -1; end = view.indexOf(LINE_FEED, start)) {
|
|
266
|
+
lines += 1;
|
|
267
|
+
const piece = view.subarray(start, end);
|
|
268
|
+
const length = pendingBytes + piece.length;
|
|
269
|
+
if (length > MAX_RECORD_BYTES)
|
|
270
|
+
throw tooLarge(lines);
|
|
271
|
+
visitLine(pending.length === 0
|
|
272
|
+
? piece
|
|
273
|
+
: Buffer.concat([...pending, piece], length), lines, visit);
|
|
274
|
+
pending = [];
|
|
275
|
+
pendingBytes = 0;
|
|
276
|
+
validBytes = position + end + 1;
|
|
277
|
+
start = end + 1;
|
|
278
|
+
}
|
|
279
|
+
pendingBytes += view.length - start;
|
|
280
|
+
if (pendingBytes > MAX_RECORD_BYTES)
|
|
281
|
+
throw tooLarge(lines + 1);
|
|
282
|
+
if (start < view.length)
|
|
283
|
+
pending.push(Buffer.from(view.subarray(start)));
|
|
284
|
+
position += bytesRead;
|
|
285
|
+
}
|
|
286
|
+
return { lines, validBytes, torn: pendingBytes > 0 };
|
|
287
|
+
}
|
|
288
|
+
finally {
|
|
289
|
+
await handle.close().catch(() => undefined);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function writeFailure(file, error) {
|
|
293
|
+
return ioFailure(file, 'cannot be written', error);
|
|
294
|
+
}
|
|
295
|
+
/** `results.jsonl` opened to be appended to and cut to `validBytes`; never through a symbolic link. */
|
|
296
|
+
async function openAppend(path, validBytes) {
|
|
297
|
+
if (!HAS_NOFOLLOW && await lstat(path).then(stats => stats.isSymbolicLink(), () => false))
|
|
298
|
+
throw new Error(`${RESULTS_FILE}: cannot be written (ELOOP)`);
|
|
299
|
+
let handle;
|
|
300
|
+
try {
|
|
301
|
+
handle = await open(path, APPEND_FLAGS);
|
|
302
|
+
if ((await handle.stat()).size > validBytes)
|
|
303
|
+
await handle.truncate(validBytes);
|
|
304
|
+
return handle;
|
|
305
|
+
}
|
|
306
|
+
catch (error) {
|
|
307
|
+
await handle?.close().catch(() => undefined);
|
|
308
|
+
throw writeFailure(RESULTS_FILE, error);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Open `results.jsonl` to append to, making it when it is not there, and cut it to `validBytes` (what
|
|
313
|
+
* `readRecords` found valid), which drops a line a killed run left unfinished. `O_APPEND` puts each write at
|
|
314
|
+
* the end, and `O_NOFOLLOW` refuses a symbolic link. `append` writes one whole line, each call after the
|
|
315
|
+
* one before, so lines never interleave; once a write has failed, none after it is written, so a line
|
|
316
|
+
* is never joined to the fragment of a failed one. `close` waits for the lines asked for and closes.
|
|
317
|
+
*
|
|
318
|
+
* @throws Error `results.jsonl: cannot be written (<code>)` for any failure to open, cut, write or close
|
|
319
|
+
*/
|
|
320
|
+
export async function openRecords(directory, validBytes) {
|
|
321
|
+
const handle = await openAppend(join(directory, RESULTS_FILE), validBytes);
|
|
322
|
+
let failed;
|
|
323
|
+
let tail = Promise.resolve();
|
|
324
|
+
return {
|
|
325
|
+
append: (line) => {
|
|
326
|
+
const written = tail.then(async () => {
|
|
327
|
+
if (failed !== undefined)
|
|
328
|
+
throw failed;
|
|
329
|
+
try {
|
|
330
|
+
await handle.appendFile(line);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
failed = writeFailure(RESULTS_FILE, error);
|
|
334
|
+
throw failed;
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
tail = written.catch(() => undefined);
|
|
338
|
+
return written;
|
|
339
|
+
},
|
|
340
|
+
close: async () => {
|
|
341
|
+
await tail;
|
|
342
|
+
try {
|
|
343
|
+
await handle.close();
|
|
344
|
+
}
|
|
345
|
+
catch (error) {
|
|
346
|
+
throw writeFailure(RESULTS_FILE, error);
|
|
347
|
+
}
|
|
348
|
+
},
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
|
|
353
|
+
* made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
|
|
354
|
+
* The rename replaces a symbolic link at `name`, and never writes through it. A failure removes the
|
|
355
|
+
* temporary file it made.
|
|
356
|
+
*
|
|
357
|
+
* @throws Error `<name>: cannot be written (<code>)`
|
|
358
|
+
*/
|
|
359
|
+
export async function writeAtomic(directory, name, text) {
|
|
360
|
+
const temporary = join(directory, `.${name}.${randomBytes(4).toString('hex')}.tmp`);
|
|
361
|
+
let made = false;
|
|
362
|
+
try {
|
|
363
|
+
const handle = await open(temporary, NEW_FILE_FLAGS);
|
|
364
|
+
made = true;
|
|
365
|
+
try {
|
|
366
|
+
await handle.writeFile(text);
|
|
367
|
+
await handle.sync();
|
|
368
|
+
}
|
|
369
|
+
finally {
|
|
370
|
+
await handle.close();
|
|
371
|
+
}
|
|
372
|
+
await rename(temporary, join(directory, name));
|
|
373
|
+
}
|
|
374
|
+
catch (error) {
|
|
375
|
+
if (made)
|
|
376
|
+
await unlink(temporary).catch(() => undefined);
|
|
377
|
+
throw writeFailure(name, error);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
|
|
3
|
+
* `jobs` at once, and hands every review that resolved to the caller as a record to write. It knows
|
|
4
|
+
* nothing of the run directory, the report or the terminal: the caller gives it a way to write a line
|
|
5
|
+
* and a callback to report to.
|
|
6
|
+
*/
|
|
7
|
+
import type { Env } from '../config/env.ts';
|
|
8
|
+
import type { ResolvedConfig } from '../config/resolve.ts';
|
|
9
|
+
import type { ReviewRecord } from './records.ts';
|
|
10
|
+
import type { Suite } from './suite.ts';
|
|
11
|
+
/** One planned review: a trial of a case with a configuration. */
|
|
12
|
+
export interface BenchReview {
|
|
13
|
+
readonly trial: number;
|
|
14
|
+
readonly caseId: string;
|
|
15
|
+
/** The configuration's index in `configurations`, from 0. */
|
|
16
|
+
readonly configuration: number;
|
|
17
|
+
}
|
|
18
|
+
/** A configuration to run: the resolved config, and the fingerprint its records are keyed with. */
|
|
19
|
+
export interface RunnerConfiguration {
|
|
20
|
+
readonly config: ResolvedConfig;
|
|
21
|
+
readonly fingerprint: string;
|
|
22
|
+
}
|
|
23
|
+
/** What the runner reports: a review that resolved and was written, or one that rejected. */
|
|
24
|
+
export type RunnerEvent = {
|
|
25
|
+
readonly type: 'recorded';
|
|
26
|
+
readonly review: BenchReview;
|
|
27
|
+
readonly record: ReviewRecord;
|
|
28
|
+
} | {
|
|
29
|
+
readonly type: 'failed';
|
|
30
|
+
readonly review: BenchReview;
|
|
31
|
+
readonly error: unknown;
|
|
32
|
+
};
|
|
33
|
+
export interface RunReviewsOptions {
|
|
34
|
+
readonly suite: Suite;
|
|
35
|
+
readonly configurations: readonly RunnerConfiguration[];
|
|
36
|
+
readonly reviews: readonly BenchReview[];
|
|
37
|
+
/** At most this many reviews run at once. */
|
|
38
|
+
readonly jobs: number;
|
|
39
|
+
/** What a `claude-code` provider's CLI inherits. */
|
|
40
|
+
readonly env: Env;
|
|
41
|
+
readonly signal?: AbortSignal;
|
|
42
|
+
/** Writes one line of `results.jsonl`; it throws when the write fails. */
|
|
43
|
+
readonly append: (line: string) => Promise<void>;
|
|
44
|
+
readonly events: (event: RunnerEvent) => void;
|
|
45
|
+
/** The most a line may hold, its line break not counted, in bytes; `MAX_RECORD_BYTES` when left out. */
|
|
46
|
+
readonly maxRecordBytes?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface RunReviewsOutcome {
|
|
49
|
+
/** The reviews written to the record. */
|
|
50
|
+
readonly recorded: number;
|
|
51
|
+
/** The reviews that rejected, or whose record was too large to write. */
|
|
52
|
+
readonly failed: number;
|
|
53
|
+
/** Whether the caller's signal was aborted by the time the run ended. */
|
|
54
|
+
readonly interrupted: boolean;
|
|
55
|
+
}
|
|
56
|
+
/** The reviews of a bench in the order they run: by trial from 1, then by case, then by configuration. */
|
|
57
|
+
export declare function planReviews(options: {
|
|
58
|
+
readonly trials: number;
|
|
59
|
+
readonly cases: readonly string[];
|
|
60
|
+
readonly configurations: number;
|
|
61
|
+
}): BenchReview[];
|
|
62
|
+
/**
|
|
63
|
+
* Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
|
|
64
|
+
* that resolved as a record: `append` writes its line, then `events` reports it as `recorded`.
|
|
65
|
+
*
|
|
66
|
+
* Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
|
|
67
|
+
* counts only its own calls. A review that rejects is reported as `failed` and not recorded, and the
|
|
68
|
+
* others run. One `AbortController` of the runner's, joined to the caller's `signal`, cuts the reviews
|
|
69
|
+
* in flight when the caller is interrupted or a write fails, and a review that has not started when it is
|
|
70
|
+
* aborted does not start. A review that rejects while the run is aborted is neither recorded nor a
|
|
71
|
+
* failure: an interrupt is not a verdict on the review. A review that resolved is recorded, even when the
|
|
72
|
+
* interrupt came while its record was written. A line over `maxRecordBytes` is a failed review and is
|
|
73
|
+
* never written, since the reader of `results.jsonl` refuses a line that long, and with it the whole
|
|
74
|
+
* directory. A write that fails stops the run: it reports nothing for that review, cuts the others, and
|
|
75
|
+
* the run rejects with the error of the write once every review has settled. A callback of `events` that
|
|
76
|
+
* throws does the same.
|
|
77
|
+
*
|
|
78
|
+
* @throws the error of the first `append` (or `events`) that threw
|
|
79
|
+
*/
|
|
80
|
+
export declare function runReviews(options: RunReviewsOptions): Promise<RunReviewsOutcome>;
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
|
|
3
|
+
* `jobs` at once, and hands every review that resolved to the caller as a record to write. It knows
|
|
4
|
+
* nothing of the run directory, the report or the terminal: the caller gives it a way to write a line
|
|
5
|
+
* and a callback to report to.
|
|
6
|
+
*/
|
|
7
|
+
import { fromConfig } from "../config/from-config.js";
|
|
8
|
+
import { createLimiter } from "../core/concurrency.js";
|
|
9
|
+
import { MAX_RECORD_BYTES, recordLine } from "./records.js";
|
|
10
|
+
const OVERSIZED_RECORD = 'its record is over the size a line of results.jsonl holds';
|
|
11
|
+
const UNKNOWN_REVIEW = 'its case or its configuration is not in the run';
|
|
12
|
+
/** The reviews of a bench in the order they run: by trial from 1, then by case, then by configuration. */
|
|
13
|
+
export function planReviews(options) {
|
|
14
|
+
const reviews = [];
|
|
15
|
+
for (let trial = 1; trial <= options.trials; trial++) {
|
|
16
|
+
for (const caseId of options.cases) {
|
|
17
|
+
for (let configuration = 0; configuration < options.configurations; configuration++)
|
|
18
|
+
reviews.push({ trial, caseId, configuration });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return reviews;
|
|
22
|
+
}
|
|
23
|
+
/** What a review's `call` events add up to, and the sink that counts them. */
|
|
24
|
+
function tallyCalls() {
|
|
25
|
+
let calls = 0;
|
|
26
|
+
let failedCalls = 0;
|
|
27
|
+
let costed = 0;
|
|
28
|
+
let measured = false;
|
|
29
|
+
const total = { inputTokens: 0, outputTokens: 0 };
|
|
30
|
+
return {
|
|
31
|
+
sink: (event) => {
|
|
32
|
+
if (event.type !== 'call')
|
|
33
|
+
return;
|
|
34
|
+
calls += 1;
|
|
35
|
+
if (!event.ok)
|
|
36
|
+
failedCalls += 1;
|
|
37
|
+
if (event.usage === undefined)
|
|
38
|
+
return;
|
|
39
|
+
measured = true;
|
|
40
|
+
total.inputTokens += event.usage.inputTokens;
|
|
41
|
+
total.outputTokens += event.usage.outputTokens;
|
|
42
|
+
if (event.usage.costUsd === undefined)
|
|
43
|
+
return;
|
|
44
|
+
costed += 1;
|
|
45
|
+
total.costUsd = (total.costUsd ?? 0) + event.usage.costUsd;
|
|
46
|
+
},
|
|
47
|
+
counts: () => ({
|
|
48
|
+
calls,
|
|
49
|
+
failedCalls,
|
|
50
|
+
...(measured
|
|
51
|
+
? {
|
|
52
|
+
usage: { ...total },
|
|
53
|
+
}
|
|
54
|
+
: {}),
|
|
55
|
+
costComplete: costed === calls,
|
|
56
|
+
}),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
|
|
61
|
+
* that resolved as a record: `append` writes its line, then `events` reports it as `recorded`.
|
|
62
|
+
*
|
|
63
|
+
* Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
|
|
64
|
+
* counts only its own calls. A review that rejects is reported as `failed` and not recorded, and the
|
|
65
|
+
* others run. One `AbortController` of the runner's, joined to the caller's `signal`, cuts the reviews
|
|
66
|
+
* in flight when the caller is interrupted or a write fails, and a review that has not started when it is
|
|
67
|
+
* aborted does not start. A review that rejects while the run is aborted is neither recorded nor a
|
|
68
|
+
* failure: an interrupt is not a verdict on the review. A review that resolved is recorded, even when the
|
|
69
|
+
* interrupt came while its record was written. A line over `maxRecordBytes` is a failed review and is
|
|
70
|
+
* never written, since the reader of `results.jsonl` refuses a line that long, and with it the whole
|
|
71
|
+
* directory. A write that fails stops the run: it reports nothing for that review, cuts the others, and
|
|
72
|
+
* the run rejects with the error of the write once every review has settled. A callback of `events` that
|
|
73
|
+
* throws does the same.
|
|
74
|
+
*
|
|
75
|
+
* @throws the error of the first `append` (or `events`) that threw
|
|
76
|
+
*/
|
|
77
|
+
export async function runReviews(options) {
|
|
78
|
+
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
|
|
79
|
+
const own = new AbortController();
|
|
80
|
+
const signal = options.signal === undefined
|
|
81
|
+
? own.signal
|
|
82
|
+
: AbortSignal.any([options.signal, own.signal]);
|
|
83
|
+
const patches = new Map(suite.cases.map(({ id, patch }) => [
|
|
84
|
+
id,
|
|
85
|
+
suite.patches.get(patch),
|
|
86
|
+
]));
|
|
87
|
+
let recorded = 0;
|
|
88
|
+
let failed = 0;
|
|
89
|
+
let stopped;
|
|
90
|
+
const stop = (error) => {
|
|
91
|
+
stopped ??= { error };
|
|
92
|
+
own.abort();
|
|
93
|
+
};
|
|
94
|
+
const report = (event) => {
|
|
95
|
+
try {
|
|
96
|
+
events(event);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
stop(error);
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
const fail = (review, error) => {
|
|
103
|
+
failed += 1;
|
|
104
|
+
report({ type: 'failed', review, error });
|
|
105
|
+
};
|
|
106
|
+
async function recordOf(review) {
|
|
107
|
+
const configuration = configurations[review.configuration];
|
|
108
|
+
const diff = patches.get(review.caseId);
|
|
109
|
+
if (configuration === undefined || diff === undefined)
|
|
110
|
+
throw new Error(UNKNOWN_REVIEW);
|
|
111
|
+
const tally = tallyCalls();
|
|
112
|
+
const quorum = await fromConfig(configuration.config, { env, events: tally.sink });
|
|
113
|
+
const started = performance.now();
|
|
114
|
+
const result = await quorum.review({ diff }, { signal });
|
|
115
|
+
const latencyMs = Math.round(performance.now() - started);
|
|
116
|
+
return {
|
|
117
|
+
fingerprint: configuration.fingerprint,
|
|
118
|
+
case: review.caseId,
|
|
119
|
+
trial: review.trial,
|
|
120
|
+
latencyMs,
|
|
121
|
+
...tally.counts(),
|
|
122
|
+
result,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
async function runOne(review) {
|
|
126
|
+
if (signal.aborted)
|
|
127
|
+
return;
|
|
128
|
+
let record;
|
|
129
|
+
let line;
|
|
130
|
+
try {
|
|
131
|
+
record = await recordOf(review);
|
|
132
|
+
line = recordLine(record);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (!signal.aborted)
|
|
136
|
+
fail(review, error);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
if (stopped !== undefined)
|
|
140
|
+
return;
|
|
141
|
+
if (Buffer.byteLength(line) - 1 > maxRecordBytes) {
|
|
142
|
+
fail(review, new Error(OVERSIZED_RECORD));
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
await append(line);
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
stop(error);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
recorded += 1;
|
|
153
|
+
report({ type: 'recorded', review, record });
|
|
154
|
+
}
|
|
155
|
+
const limit = createLimiter(jobs);
|
|
156
|
+
await Promise.all(reviews.map(review => limit(() => runOne(review))));
|
|
157
|
+
if (stopped !== undefined)
|
|
158
|
+
throw stopped.error;
|
|
159
|
+
return { recorded, failed, interrupted: options.signal?.aborted === true };
|
|
160
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring a review against a suite case: whether its findings catch the case's bug, and how many kept
|
|
3
|
+
* findings catch nothing. Pure code over a result and a case, so a record can be scored again later.
|
|
4
|
+
*/
|
|
5
|
+
import type { Finding, QuorumResult } from '../core/types.ts';
|
|
6
|
+
import type { Location, SuiteCase } from './suite.ts';
|
|
7
|
+
/** What one review earned against one case. */
|
|
8
|
+
export interface ReviewOutcome {
|
|
9
|
+
/** A kept finding, at any confidence, catches one of the bug's locations; always false for a control. */
|
|
10
|
+
readonly keptCatch: boolean;
|
|
11
|
+
/** A finding the finder proposed catches one: a kept one, or one the panel discarded or left unreviewed. */
|
|
12
|
+
readonly proposedCatch: boolean;
|
|
13
|
+
/** The kept findings that catch no location: for a control, every kept finding. */
|
|
14
|
+
readonly unmatchedKept: number;
|
|
15
|
+
/** The review has a gap or an unreviewed finding, in any profile. */
|
|
16
|
+
readonly incomplete: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
|
|
20
|
+
* line from `window` lines before the location's start to `window` lines after its end. The window is
|
|
21
|
+
* taken as given, 0 included.
|
|
22
|
+
*/
|
|
23
|
+
export declare function catches(finding: Pick<Finding, 'file' | 'line'>, location: Location, window: number): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Score a review against a case. A bug is caught by a kept finding (`keptCatch`) or by any finding the
|
|
26
|
+
* finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window` lines of one of its
|
|
27
|
+
* locations. A control has nothing to catch, so every kept finding is unmatched. Every profile of the
|
|
28
|
+
* review counts, for the catches and for the gaps alike.
|
|
29
|
+
*/
|
|
30
|
+
export declare function scoreReview(result: QuorumResult, suiteCase: SuiteCase, window: number): ReviewOutcome;
|