@tobi-or-not/quorum 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +7 -4
- package/dist/bench/fingerprint.js +22 -8
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +241 -13
- package/dist/bench/records.js +64 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +47 -17
- package/dist/bench/runner.js +82 -21
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +27 -9
- package/dist/cli/bench.js +174 -83
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +100 -38
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +34 -0
- package/dist/core/diff.js +140 -18
- package/dist/core/errors.d.ts +18 -3
- package/dist/core/errors.js +26 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +126 -22
- package/dist/core/stages/context.js +319 -41
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +14 -6
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +69 -33
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +72 -15
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +17 -9
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +17 -9
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
package/dist/bench/run-dir.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Every read and write of a bench run directory, which holds `run.json` (written
|
|
3
|
-
*
|
|
4
|
-
* `report.json`, rewritten whole). Both data
|
|
5
|
-
*
|
|
6
|
-
* regular file. No write follows a symbolic link:
|
|
7
|
-
* other file is written to a new file that is
|
|
2
|
+
* Every read and write of a bench run directory, which holds `run.json` (written whole when the run begins,
|
|
3
|
+
* and again when a re-score records the suite's new scoring hash), `results.jsonl` (one line per review,
|
|
4
|
+
* appended one at a time), and the reports (`report.md` and `report.json`, rewritten whole). Both data
|
|
5
|
+
* files are read as untrusted, since another run or a person may have written them: strict schemas (see
|
|
6
|
+
* `records.ts`), bounded sizes, and no file that is not a regular file. No write follows a symbolic link:
|
|
7
|
+
* `results.jsonl` is opened with `O_NOFOLLOW`, and every other file is written to a new file that is
|
|
8
|
+
* renamed over the name, which replaces a link there. `run.json` and `results.jsonl` are made readable by
|
|
9
|
+
* their owner only; the reports are made as any new file.
|
|
8
10
|
*
|
|
9
11
|
* Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
|
|
10
12
|
* runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
|
|
@@ -39,8 +41,15 @@ const NO_FOLLOW = HAS_NOFOLLOW
|
|
|
39
41
|
: 0;
|
|
40
42
|
/** Without blocking, so a FIFO is refused instead of waited for. */
|
|
41
43
|
const READ_FLAGS = constants.O_RDONLY | constants.O_NONBLOCK | NO_FOLLOW;
|
|
42
|
-
|
|
44
|
+
/** Without blocking too, so a FIFO swapped in after the read is refused instead of waited for. */
|
|
45
|
+
const APPEND_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_NONBLOCK | NO_FOLLOW;
|
|
43
46
|
const NEW_FILE_FLAGS = constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL;
|
|
47
|
+
/** The mode `run.json` and `results.jsonl` are made with: read and written by their owner only. */
|
|
48
|
+
const OWNER_ONLY = 0o600;
|
|
49
|
+
/** The name `writeAtomic` gives the temporary file of `run.json`, which a run killed while it wrote can leave. */
|
|
50
|
+
const LEFTOVER_RUN_FILE = /^\.run\.json\.[0-9a-f]{8}\.tmp$/;
|
|
51
|
+
/** What `openRecords` says of a `results.jsonl` whose length is not the one the read found. */
|
|
52
|
+
const CHANGED = 'changed after this run read it, so another run may be using this --out: run one at a time';
|
|
44
53
|
function refusal(path, message) {
|
|
45
54
|
return new ConfigError([
|
|
46
55
|
{ path, message },
|
|
@@ -63,12 +72,19 @@ function codeSuffix(error) {
|
|
|
63
72
|
function ioFailure(file, what, error) {
|
|
64
73
|
return new Error(`${file}: ${what}${codeSuffix(error)}`);
|
|
65
74
|
}
|
|
66
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Whether `directory` holds an entry, of any kind, whose name is not that of the temporary `run.json` a
|
|
77
|
+
* killed run leaves: it reads entries until it finds one, and no more.
|
|
78
|
+
*/
|
|
67
79
|
async function holdsEntry(directory) {
|
|
68
80
|
try {
|
|
69
81
|
const entries = await opendir(directory);
|
|
70
82
|
try {
|
|
71
|
-
|
|
83
|
+
for (let entry = await entries.read(); entry !== null; entry = await entries.read()) {
|
|
84
|
+
if (!LEFTOVER_RUN_FILE.test(entry.name))
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
return false;
|
|
72
88
|
}
|
|
73
89
|
finally {
|
|
74
90
|
await entries.close().catch(() => undefined);
|
|
@@ -78,7 +94,7 @@ async function holdsEntry(directory) {
|
|
|
78
94
|
throw ioFailure(OUT, 'cannot be read', error);
|
|
79
95
|
}
|
|
80
96
|
}
|
|
81
|
-
/** `empty` for a directory with no `run.json` and no entry; refused when it holds any. */
|
|
97
|
+
/** `empty` for a directory with no `run.json` and no entry, the temporary `run.json` of a killed run aside; refused when it holds any other. */
|
|
82
98
|
async function emptyDirectory(directory) {
|
|
83
99
|
if (await holdsEntry(directory))
|
|
84
100
|
throw refusal(OUT, `holds files but no ${RUN_FILE}, so it is not a run directory: name a new directory, or an empty one`);
|
|
@@ -88,7 +104,8 @@ async function emptyDirectory(directory) {
|
|
|
88
104
|
* What `directory`, the run directory a command was given, holds. A directory that does not exist is
|
|
89
105
|
* `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
|
|
90
106
|
* as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
|
|
91
|
-
* `run.json
|
|
107
|
+
* `run.json` (the temporary `run.json` of a killed run aside, which counts as no file and is left where it
|
|
108
|
+
* is), and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
|
|
92
109
|
* format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
|
|
93
110
|
* its parents may be links: they are the user's own path.
|
|
94
111
|
*
|
|
@@ -129,24 +146,26 @@ export async function inspectRunDir(directory) {
|
|
|
129
146
|
/**
|
|
130
147
|
* What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
|
|
131
148
|
* quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
|
|
132
|
-
* which holds the preset by name), then another suite by its name, else a suite whose
|
|
133
|
-
* then each current configuration the run does not hold, at
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
149
|
+
* which holds the preset by name), then another suite by its name, else a suite whose review hash changed
|
|
150
|
+
* (its case ids, its trials or its patches), then each current configuration the run does not hold, at
|
|
151
|
+
* `--config`, then each recorded one that is not given again. A changed scoring hash alone is no
|
|
152
|
+
* difference: the run's records are scored again under the suite as it is now. A current and a recorded
|
|
153
|
+
* configuration that share a path (`null`, the usual resolution, included) but not a fingerprint are one
|
|
154
|
+
* configuration whose settings changed, and give one issue in place of the two. The same configurations in
|
|
155
|
+
* another order are the same run. A configuration is named by its path as written, or
|
|
156
|
+
* `(default configuration)`: the recorded paths are text from `run.json`, to make inert where they are
|
|
157
|
+
* printed.
|
|
139
158
|
*/
|
|
140
159
|
export function runDifferences(recorded, current) {
|
|
141
160
|
const issues = [];
|
|
142
161
|
if (recorded.quorum !== current.quorum) {
|
|
143
|
-
issues.push({ path: RUN_FILE, message: `was written by quorum ${recorded.quorum}, and this is quorum ${current.quorum}: a run belongs to one build, so name a new ${OUT}` });
|
|
162
|
+
issues.push({ path: RUN_FILE, message: `was written by quorum ${recorded.quorum}, and this is quorum ${current.quorum}: a run belongs to one build, so resume it with the quorum that began it, or name a new ${OUT}` });
|
|
144
163
|
}
|
|
145
164
|
if (recorded.suite.name !== current.suite.name) {
|
|
146
165
|
issues.push({ path: RUN_FILE, message: `is a run of the suite ${recorded.suite.name}, not ${current.suite.name}: name a new ${OUT}` });
|
|
147
166
|
}
|
|
148
|
-
else if (recorded.suite.
|
|
149
|
-
issues.push({ path: RUN_FILE, message: `the suite's
|
|
167
|
+
else if (recorded.suite.reviewHash !== current.suite.reviewHash) {
|
|
168
|
+
issues.push({ path: RUN_FILE, message: `the suite's case ids, trials or patches changed since the run began: name a new ${OUT}` });
|
|
150
169
|
}
|
|
151
170
|
const recordedPrints = new Set(recorded.configurations.map(({ fingerprint }) => fingerprint));
|
|
152
171
|
const currentPrints = new Set(current.configurations.map(({ fingerprint }) => fingerprint));
|
|
@@ -171,7 +190,16 @@ export function runDifferences(recorded, current) {
|
|
|
171
190
|
return issues;
|
|
172
191
|
}
|
|
173
192
|
/**
|
|
174
|
-
*
|
|
193
|
+
* Write `run.json` into `directory` whole, through a new file and a rename, which replaces the file that
|
|
194
|
+
* was there. `run.json` is readable by its owner only, so a looser mode its owner gave it is not kept.
|
|
195
|
+
*
|
|
196
|
+
* @throws Error `run.json: cannot be written (<code>)`
|
|
197
|
+
*/
|
|
198
|
+
export async function writeRunFile(directory, run) {
|
|
199
|
+
await writeAtomic(directory, RUN_FILE, `${inertJson(run)}\n`, OWNER_ONLY);
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Make `directory` and its parents, then write `run.json` into it (see `writeRunFile`).
|
|
175
203
|
*
|
|
176
204
|
* @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
|
|
177
205
|
*/
|
|
@@ -182,7 +210,7 @@ export async function createRun(directory, run) {
|
|
|
182
210
|
catch (error) {
|
|
183
211
|
throw ioFailure(directory, 'cannot be created', error);
|
|
184
212
|
}
|
|
185
|
-
await
|
|
213
|
+
await writeRunFile(directory, run);
|
|
186
214
|
}
|
|
187
215
|
/** `results.jsonl` opened to be read, or undefined when there is none; refused when it is a link or not a file. */
|
|
188
216
|
async function openResults(path) {
|
|
@@ -246,8 +274,14 @@ async function readInto(handle, chunk, position) {
|
|
|
246
274
|
*/
|
|
247
275
|
export async function readRecords(directory, visit) {
|
|
248
276
|
const handle = await openResults(join(directory, RESULTS_FILE));
|
|
249
|
-
if (handle === undefined)
|
|
250
|
-
return {
|
|
277
|
+
if (handle === undefined) {
|
|
278
|
+
return {
|
|
279
|
+
lines: 0,
|
|
280
|
+
validBytes: 0,
|
|
281
|
+
bytes: 0,
|
|
282
|
+
torn: false,
|
|
283
|
+
};
|
|
284
|
+
}
|
|
251
285
|
try {
|
|
252
286
|
const chunk = Buffer.allocUnsafe(CHUNK_BYTES);
|
|
253
287
|
/** The copies of the bytes of the line in progress that earlier chunks held. */
|
|
@@ -283,7 +317,12 @@ export async function readRecords(directory, visit) {
|
|
|
283
317
|
pending.push(Buffer.from(view.subarray(start)));
|
|
284
318
|
position += bytesRead;
|
|
285
319
|
}
|
|
286
|
-
return {
|
|
320
|
+
return {
|
|
321
|
+
lines,
|
|
322
|
+
validBytes,
|
|
323
|
+
bytes: position,
|
|
324
|
+
torn: pendingBytes > 0,
|
|
325
|
+
};
|
|
287
326
|
}
|
|
288
327
|
finally {
|
|
289
328
|
await handle.close().catch(() => undefined);
|
|
@@ -292,33 +331,72 @@ export async function readRecords(directory, visit) {
|
|
|
292
331
|
function writeFailure(file, error) {
|
|
293
332
|
return ioFailure(file, 'cannot be written', error);
|
|
294
333
|
}
|
|
295
|
-
/**
|
|
296
|
-
|
|
334
|
+
/**
|
|
335
|
+
* `results.jsonl` opened to be appended to, never through a symbolic link, never waiting for a FIFO, and
|
|
336
|
+
* cut to `read.validBytes` when it ends in the torn tail the read found. A read that found no bytes opens
|
|
337
|
+
* with `O_CREAT`: a new run's file, or an empty one deleted since, is made. A read that found bytes opens
|
|
338
|
+
* without it, so a file deleted since is refused instead of made again, empty. Refused, with the handle
|
|
339
|
+
* closed: what is not a regular file, a file that is gone or whose length is not `read.bytes`.
|
|
340
|
+
*/
|
|
341
|
+
async function openAppend(path, read) {
|
|
297
342
|
if (!HAS_NOFOLLOW && await lstat(path).then(stats => stats.isSymbolicLink(), () => false))
|
|
298
343
|
throw new Error(`${RESULTS_FILE}: cannot be written (ELOOP)`);
|
|
344
|
+
const flags = read.bytes > 0
|
|
345
|
+
? APPEND_FLAGS
|
|
346
|
+
: APPEND_FLAGS | constants.O_CREAT;
|
|
299
347
|
let handle;
|
|
300
348
|
try {
|
|
301
|
-
handle = await open(path,
|
|
302
|
-
|
|
303
|
-
|
|
349
|
+
handle = await open(path, flags, OWNER_ONLY);
|
|
350
|
+
}
|
|
351
|
+
catch (error) {
|
|
352
|
+
// Under O_NONBLOCK a FIFO with no reader fails with ENXIO and a directory with EISDIR: no regular file.
|
|
353
|
+
// Whatever else opens, a device or a FIFO with a reader, is refused by the stat below; any other
|
|
354
|
+
// failure, such as a socket on macOS, is a write failure. Without O_CREAT, ENOENT is a file that was
|
|
355
|
+
// there when the read found bytes in it, and is not now.
|
|
356
|
+
const code = errorCode(error);
|
|
357
|
+
if (code === 'ENOENT' && read.bytes > 0)
|
|
358
|
+
throw refusal(RESULTS_FILE, CHANGED);
|
|
359
|
+
throw code === 'ENXIO' || code === 'EISDIR'
|
|
360
|
+
? refusal(RESULTS_FILE, 'is not a file')
|
|
361
|
+
: writeFailure(RESULTS_FILE, error);
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
const stats = await handle.stat();
|
|
365
|
+
if (!stats.isFile())
|
|
366
|
+
throw refusal(RESULTS_FILE, 'is not a file');
|
|
367
|
+
if (stats.size !== read.bytes)
|
|
368
|
+
throw refusal(RESULTS_FILE, CHANGED);
|
|
369
|
+
if (stats.size > read.validBytes)
|
|
370
|
+
await handle.truncate(read.validBytes);
|
|
304
371
|
return handle;
|
|
305
372
|
}
|
|
306
373
|
catch (error) {
|
|
307
|
-
await handle
|
|
308
|
-
throw
|
|
374
|
+
await handle.close().catch(() => undefined);
|
|
375
|
+
throw error instanceof ConfigError
|
|
376
|
+
? error
|
|
377
|
+
: writeFailure(RESULTS_FILE, error);
|
|
309
378
|
}
|
|
310
379
|
}
|
|
311
380
|
/**
|
|
312
|
-
* Open `results.jsonl` to append to,
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*
|
|
381
|
+
* Open `results.jsonl` to append to, and cut it to `read.validBytes` (what `readRecords` found valid),
|
|
382
|
+
* which drops a line a killed run left unfinished. `read` is the answer of the `readRecords` this run
|
|
383
|
+
* began with, or `{ validBytes: 0, bytes: 0 }` for a run that is new: the file must still be `read.bytes`
|
|
384
|
+
* long, or another run may have cut it or appended to it since, and cutting it back would drop that run's
|
|
385
|
+
* records. A read that found no bytes makes the file when it is not there, which loses nothing; one that
|
|
386
|
+
* found bytes never makes it, so a file deleted since is refused with nothing written. `O_APPEND` puts
|
|
387
|
+
* each write at the end, `O_NOFOLLOW` refuses a symbolic link, and `O_NONBLOCK` refuses a FIFO instead of
|
|
388
|
+
* waiting for it. A file made here is readable by its owner only, and one that exists keeps its mode.
|
|
389
|
+
* `append` writes one whole line, each call after the one before, so lines never interleave; once a write
|
|
390
|
+
* has failed, none after it is written, so a line is never joined to the fragment of a failed one. `close`
|
|
391
|
+
* waits for the lines asked for and closes.
|
|
317
392
|
*
|
|
318
|
-
* @throws
|
|
393
|
+
* @throws ConfigError `results.jsonl: is not a file`, or `results.jsonl: changed after this run read it,
|
|
394
|
+
* so another run may be using this --out: run one at a time` (a file cut, appended to or deleted since the
|
|
395
|
+
* read); Error `results.jsonl: cannot be written (<code>)` for any other failure to open, cut, write or
|
|
396
|
+
* close
|
|
319
397
|
*/
|
|
320
|
-
export async function openRecords(directory,
|
|
321
|
-
const handle = await openAppend(join(directory, RESULTS_FILE),
|
|
398
|
+
export async function openRecords(directory, read) {
|
|
399
|
+
const handle = await openAppend(join(directory, RESULTS_FILE), read);
|
|
322
400
|
let failed;
|
|
323
401
|
let tail = Promise.resolve();
|
|
324
402
|
return {
|
|
@@ -351,16 +429,17 @@ export async function openRecords(directory, validBytes) {
|
|
|
351
429
|
/**
|
|
352
430
|
* Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
|
|
353
431
|
* made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
|
|
354
|
-
* The rename replaces a symbolic link at `name`, and never writes through it.
|
|
355
|
-
*
|
|
432
|
+
* The rename replaces a symbolic link at `name`, and never writes through it. The temporary file is made
|
|
433
|
+
* with `mode`, which the rename keeps: Node's own default unless a caller asks for another. A failure
|
|
434
|
+
* removes the temporary file it made.
|
|
356
435
|
*
|
|
357
436
|
* @throws Error `<name>: cannot be written (<code>)`
|
|
358
437
|
*/
|
|
359
|
-
export async function writeAtomic(directory, name, text) {
|
|
438
|
+
export async function writeAtomic(directory, name, text, mode = 0o666) {
|
|
360
439
|
const temporary = join(directory, `.${name}.${randomBytes(4).toString('hex')}.tmp`);
|
|
361
440
|
let made = false;
|
|
362
441
|
try {
|
|
363
|
-
const handle = await open(temporary, NEW_FILE_FLAGS);
|
|
442
|
+
const handle = await open(temporary, NEW_FILE_FLAGS, mode);
|
|
364
443
|
made = true;
|
|
365
444
|
try {
|
|
366
445
|
await handle.writeFile(text);
|
package/dist/bench/runner.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
|
|
3
|
-
* `jobs` at once, and hands every review that resolved
|
|
4
|
-
* nothing of the run directory, the report or the terminal: the caller gives it
|
|
5
|
-
* and a callback to report to.
|
|
3
|
+
* `jobs` at once, and hands every review that resolved, or that a model's reply denied, to the caller as a
|
|
4
|
+
* record to write. It knows nothing of the run directory, the report or the terminal: the caller gives it
|
|
5
|
+
* a way to write a line and a callback to report to.
|
|
6
6
|
*/
|
|
7
7
|
import type { Env } from '../config/env.ts';
|
|
8
8
|
import type { ResolvedConfig } from '../config/resolve.ts';
|
|
9
|
-
import type { ReviewRecord } from './records.ts';
|
|
9
|
+
import type { Denial, ReviewRecord } from './records.ts';
|
|
10
10
|
import type { Suite } from './suite.ts';
|
|
11
11
|
/** One planned review: a trial of a case with a configuration. */
|
|
12
12
|
export interface BenchReview {
|
|
@@ -20,7 +20,7 @@ export interface RunnerConfiguration {
|
|
|
20
20
|
readonly config: ResolvedConfig;
|
|
21
21
|
readonly fingerprint: string;
|
|
22
22
|
}
|
|
23
|
-
/** What the runner reports: a review that resolved and
|
|
23
|
+
/** What the runner reports: a review that was recorded, resolved or denied, and written, or one that failed. */
|
|
24
24
|
export type RunnerEvent = {
|
|
25
25
|
readonly type: 'recorded';
|
|
26
26
|
readonly review: BenchReview;
|
|
@@ -44,12 +44,21 @@ export interface RunReviewsOptions {
|
|
|
44
44
|
readonly events: (event: RunnerEvent) => void;
|
|
45
45
|
/** The most a line may hold, its line break not counted, in bytes; `MAX_RECORD_BYTES` when left out. */
|
|
46
46
|
readonly maxRecordBytes?: number;
|
|
47
|
+
/**
|
|
48
|
+
* Once the run's reviews have spent this many tokens, as each review's cap counts them, no review starts.
|
|
49
|
+
* No cap when left out.
|
|
50
|
+
*/
|
|
51
|
+
readonly maxRunTokens?: number;
|
|
47
52
|
}
|
|
48
53
|
export interface RunReviewsOutcome {
|
|
49
54
|
/** The reviews written to the record. */
|
|
50
55
|
readonly recorded: number;
|
|
51
|
-
/** The reviews that
|
|
56
|
+
/** The reviews that failed, a rejection that is no denial, or whose record was too large to write. */
|
|
52
57
|
readonly failed: number;
|
|
58
|
+
/** The reviews that did not start because the run had spent `maxRunTokens`. */
|
|
59
|
+
readonly notRun: number;
|
|
60
|
+
/** The tokens the run's `call` events counted against their reviews' caps, as `maxRunTokens` reads them. */
|
|
61
|
+
readonly spent: number;
|
|
53
62
|
/** Whether the caller's signal was aborted by the time the run ended. */
|
|
54
63
|
readonly interrupted: boolean;
|
|
55
64
|
}
|
|
@@ -59,21 +68,42 @@ export declare function planReviews(options: {
|
|
|
59
68
|
readonly cases: readonly string[];
|
|
60
69
|
readonly configurations: number;
|
|
61
70
|
}): BenchReview[];
|
|
71
|
+
/**
|
|
72
|
+
* The denial a review's rejection records, or undefined when the review fails and is retried. A review
|
|
73
|
+
* whose research reply could not be used (an `OutputError`), or whose research call the model refused or
|
|
74
|
+
* cut off at its token limit (a `ProviderError` of kind `refusal` or `truncated`), was answered by its
|
|
75
|
+
* model: a retry cannot tell the model's unreliability from an outage, and a poisoned patch provokes such
|
|
76
|
+
* replies on purpose, so the review is recorded as denied, with no result, and counts as incomplete. Any
|
|
77
|
+
* other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
|
|
78
|
+
* nothing of the model's answer, so the review fails and runs again. The classes are matched with
|
|
79
|
+
* `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
|
|
80
|
+
* Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
|
|
81
|
+
* reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
|
|
82
|
+
*/
|
|
83
|
+
export declare function denialOf(error: unknown): Denial | undefined;
|
|
62
84
|
/**
|
|
63
85
|
* Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
|
|
64
|
-
* that resolved as a record: `append` writes its line, then
|
|
86
|
+
* that resolved, or that a model's reply denied (`denialOf`), as a record: `append` writes its line, then
|
|
87
|
+
* `events` reports it as `recorded`.
|
|
65
88
|
*
|
|
66
89
|
* Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
|
|
67
|
-
* counts only its own calls. A review that rejects is reported as `failed` and not
|
|
68
|
-
* others run. One `AbortController` of the runner's, joined to the caller's `signal`,
|
|
69
|
-
* in flight when the caller is interrupted or a write fails, and a review that has not
|
|
70
|
-
* aborted does not start. A review that rejects while the run is aborted is neither
|
|
71
|
-
* failure: an interrupt is not a verdict on the review. A review that
|
|
72
|
-
* interrupt came while its record was written. A line over
|
|
73
|
-
* never written, since the reader of `results.jsonl` refuses
|
|
74
|
-
* directory. A write that fails stops the run: it reports
|
|
75
|
-
* the run rejects with the error of the write once every
|
|
76
|
-
* throws does the same.
|
|
90
|
+
* counts only its own calls. A review that rejects with any other error is reported as `failed` and not
|
|
91
|
+
* recorded, and the others run. One `AbortController` of the runner's, joined to the caller's `signal`,
|
|
92
|
+
* cuts the reviews in flight when the caller is interrupted or a write fails, and a review that has not
|
|
93
|
+
* started when it is aborted does not start. A review that rejects while the run is aborted is neither
|
|
94
|
+
* recorded, a denial included, nor a failure: an interrupt is not a verdict on the review. A review that
|
|
95
|
+
* resolved is recorded, even when the interrupt came while its record was written. A line over
|
|
96
|
+
* `maxRecordBytes` is a failed review and is never written, since the reader of `results.jsonl` refuses
|
|
97
|
+
* a line that long, and with it the whole directory. A write that fails stops the run: it reports
|
|
98
|
+
* nothing for that review, cuts the others, and the run rejects with the error of the write once every
|
|
99
|
+
* review has settled. A callback of `events` that throws does the same.
|
|
100
|
+
*
|
|
101
|
+
* The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
|
|
102
|
+
* failed or was denied included. Before a review starts, and after the abort's check, the run checks that
|
|
103
|
+
* the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
|
|
104
|
+
* reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
|
|
105
|
+
* each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
|
|
106
|
+
* a failing review's own abort cut send no event, so the count misses them.
|
|
77
107
|
*
|
|
78
108
|
* @throws the error of the first `append` (or `events`) that threw
|
|
79
109
|
*/
|
package/dist/bench/runner.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
|
|
3
|
-
* `jobs` at once, and hands every review that resolved
|
|
4
|
-
* nothing of the run directory, the report or the terminal: the caller gives it
|
|
5
|
-
* and a callback to report to.
|
|
3
|
+
* `jobs` at once, and hands every review that resolved, or that a model's reply denied, to the caller as a
|
|
4
|
+
* record to write. It knows nothing of the run directory, the report or the terminal: the caller gives it
|
|
5
|
+
* a way to write a line and a callback to report to.
|
|
6
6
|
*/
|
|
7
7
|
import { fromConfig } from "../config/from-config.js";
|
|
8
8
|
import { createLimiter } from "../core/concurrency.js";
|
|
9
|
+
import { OutputError, ProviderError } from "../core/errors.js";
|
|
9
10
|
import { MAX_RECORD_BYTES, recordLine } from "./records.js";
|
|
10
11
|
const OVERSIZED_RECORD = 'its record is over the size a line of results.jsonl holds';
|
|
11
12
|
const UNKNOWN_REVIEW = 'its case or its configuration is not in the run';
|
|
@@ -56,26 +57,53 @@ function tallyCalls() {
|
|
|
56
57
|
}),
|
|
57
58
|
};
|
|
58
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* The denial a review's rejection records, or undefined when the review fails and is retried. A review
|
|
62
|
+
* whose research reply could not be used (an `OutputError`), or whose research call the model refused or
|
|
63
|
+
* cut off at its token limit (a `ProviderError` of kind `refusal` or `truncated`), was answered by its
|
|
64
|
+
* model: a retry cannot tell the model's unreliability from an outage, and a poisoned patch provokes such
|
|
65
|
+
* replies on purpose, so the review is recorded as denied, with no result, and counts as incomplete. Any
|
|
66
|
+
* other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
|
|
67
|
+
* nothing of the model's answer, so the review fails and runs again. The classes are matched with
|
|
68
|
+
* `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
|
|
69
|
+
* Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
|
|
70
|
+
* reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
|
|
71
|
+
*/
|
|
72
|
+
export function denialOf(error) {
|
|
73
|
+
if (error instanceof OutputError)
|
|
74
|
+
return 'unusable-reply';
|
|
75
|
+
if (error instanceof ProviderError && (error.kind === 'refusal' || error.kind === 'truncated'))
|
|
76
|
+
return error.kind;
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
59
79
|
/**
|
|
60
80
|
* Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
|
|
61
|
-
* that resolved as a record: `append` writes its line, then
|
|
81
|
+
* that resolved, or that a model's reply denied (`denialOf`), as a record: `append` writes its line, then
|
|
82
|
+
* `events` reports it as `recorded`.
|
|
62
83
|
*
|
|
63
84
|
* Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
|
|
64
|
-
* counts only its own calls. A review that rejects is reported as `failed` and not
|
|
65
|
-
* others run. One `AbortController` of the runner's, joined to the caller's `signal`,
|
|
66
|
-
* in flight when the caller is interrupted or a write fails, and a review that has not
|
|
67
|
-
* aborted does not start. A review that rejects while the run is aborted is neither
|
|
68
|
-
* failure: an interrupt is not a verdict on the review. A review that
|
|
69
|
-
* interrupt came while its record was written. A line over
|
|
70
|
-
* never written, since the reader of `results.jsonl` refuses
|
|
71
|
-
* directory. A write that fails stops the run: it reports
|
|
72
|
-
* the run rejects with the error of the write once every
|
|
73
|
-
* throws does the same.
|
|
85
|
+
* counts only its own calls. A review that rejects with any other error is reported as `failed` and not
|
|
86
|
+
* recorded, and the others run. One `AbortController` of the runner's, joined to the caller's `signal`,
|
|
87
|
+
* cuts the reviews in flight when the caller is interrupted or a write fails, and a review that has not
|
|
88
|
+
* started when it is aborted does not start. A review that rejects while the run is aborted is neither
|
|
89
|
+
* recorded, a denial included, nor a failure: an interrupt is not a verdict on the review. A review that
|
|
90
|
+
* resolved is recorded, even when the interrupt came while its record was written. A line over
|
|
91
|
+
* `maxRecordBytes` is a failed review and is never written, since the reader of `results.jsonl` refuses
|
|
92
|
+
* a line that long, and with it the whole directory. A write that fails stops the run: it reports
|
|
93
|
+
* nothing for that review, cuts the others, and the run rejects with the error of the write once every
|
|
94
|
+
* review has settled. A callback of `events` that throws does the same.
|
|
95
|
+
*
|
|
96
|
+
* The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
|
|
97
|
+
* failed or was denied included. Before a review starts, and after the abort's check, the run checks that
|
|
98
|
+
* the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
|
|
99
|
+
* reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
|
|
100
|
+
* each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
|
|
101
|
+
* a failing review's own abort cut send no event, so the count misses them.
|
|
74
102
|
*
|
|
75
103
|
* @throws the error of the first `append` (or `events`) that threw
|
|
76
104
|
*/
|
|
77
105
|
export async function runReviews(options) {
|
|
78
|
-
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
|
|
106
|
+
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES, maxRunTokens = Infinity } = options;
|
|
79
107
|
const own = new AbortController();
|
|
80
108
|
const signal = options.signal === undefined
|
|
81
109
|
? own.signal
|
|
@@ -86,6 +114,8 @@ export async function runReviews(options) {
|
|
|
86
114
|
]));
|
|
87
115
|
let recorded = 0;
|
|
88
116
|
let failed = 0;
|
|
117
|
+
let notRun = 0;
|
|
118
|
+
let spent = 0;
|
|
89
119
|
let stopped;
|
|
90
120
|
const stop = (error) => {
|
|
91
121
|
stopped ??= { error };
|
|
@@ -109,22 +139,47 @@ export async function runReviews(options) {
|
|
|
109
139
|
if (configuration === undefined || diff === undefined)
|
|
110
140
|
throw new Error(UNKNOWN_REVIEW);
|
|
111
141
|
const tally = tallyCalls();
|
|
112
|
-
const
|
|
142
|
+
const sink = (event) => {
|
|
143
|
+
if (event.type === 'call')
|
|
144
|
+
spent += event.spent;
|
|
145
|
+
tally.sink(event);
|
|
146
|
+
};
|
|
147
|
+
const quorum = await fromConfig(configuration.config, { env, events: sink });
|
|
113
148
|
const started = performance.now();
|
|
114
|
-
const
|
|
115
|
-
const latencyMs = Math.round(performance.now() - started);
|
|
116
|
-
return {
|
|
149
|
+
const facts = () => ({
|
|
117
150
|
fingerprint: configuration.fingerprint,
|
|
118
151
|
case: review.caseId,
|
|
119
152
|
trial: review.trial,
|
|
120
|
-
latencyMs,
|
|
153
|
+
latencyMs: Math.round(performance.now() - started),
|
|
121
154
|
...tally.counts(),
|
|
155
|
+
});
|
|
156
|
+
let result;
|
|
157
|
+
try {
|
|
158
|
+
result = await quorum.review({ diff }, { signal });
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
const denied = signal.aborted
|
|
162
|
+
? undefined
|
|
163
|
+
: denialOf(error);
|
|
164
|
+
if (denied === undefined)
|
|
165
|
+
throw error;
|
|
166
|
+
return {
|
|
167
|
+
...facts(),
|
|
168
|
+
denied,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
...facts(),
|
|
122
173
|
result,
|
|
123
174
|
};
|
|
124
175
|
}
|
|
125
176
|
async function runOne(review) {
|
|
126
177
|
if (signal.aborted)
|
|
127
178
|
return;
|
|
179
|
+
if (spent >= maxRunTokens) {
|
|
180
|
+
notRun += 1;
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
128
183
|
let record;
|
|
129
184
|
let line;
|
|
130
185
|
try {
|
|
@@ -156,5 +211,11 @@ export async function runReviews(options) {
|
|
|
156
211
|
await Promise.all(reviews.map(review => limit(() => runOne(review))));
|
|
157
212
|
if (stopped !== undefined)
|
|
158
213
|
throw stopped.error;
|
|
159
|
-
return {
|
|
214
|
+
return {
|
|
215
|
+
recorded,
|
|
216
|
+
failed,
|
|
217
|
+
notRun,
|
|
218
|
+
spent,
|
|
219
|
+
interrupted: options.signal?.aborted === true,
|
|
220
|
+
};
|
|
160
221
|
}
|
package/dist/bench/score.d.ts
CHANGED
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Scoring a review against a suite case: whether its findings catch the case's bug,
|
|
3
|
-
* findings catch nothing
|
|
2
|
+
* Scoring a review against a suite case: whether its findings catch the case's bug, how many kept
|
|
3
|
+
* findings catch nothing, which decoys it kept, and how each lens voted on the findings whose truth the
|
|
4
|
+
* case knows. Pure code over a result and a case, so a record can be scored again later.
|
|
4
5
|
*/
|
|
5
6
|
import type { Finding, QuorumResult } from '../core/types.ts';
|
|
6
7
|
import type { Location, SuiteCase } from './suite.ts';
|
|
8
|
+
/**
|
|
9
|
+
* A finding whose truth the case knows, with what each lens did on it: the unit of the lenses' error
|
|
10
|
+
* correlation.
|
|
11
|
+
*/
|
|
12
|
+
export interface JudgedFinding {
|
|
13
|
+
/** The profile whose panel voted on it. */
|
|
14
|
+
readonly profile: string;
|
|
15
|
+
/** The number of votes the panel was asked for: the verdict's `votes.length`. */
|
|
16
|
+
readonly lenses: number;
|
|
17
|
+
/** Each lens that voted, by its `angle`, with whether it erred; a lens whose vote failed is absent. */
|
|
18
|
+
readonly errors: ReadonlyMap<string, boolean>;
|
|
19
|
+
}
|
|
7
20
|
/** What one review earned against one case. */
|
|
8
21
|
export interface ReviewOutcome {
|
|
9
22
|
/** A kept finding, at any confidence, catches one of the bug's locations; always false for a control. */
|
|
@@ -14,6 +27,10 @@ export interface ReviewOutcome {
|
|
|
14
27
|
readonly unmatchedKept: number;
|
|
15
28
|
/** The review has a gap or an unreviewed finding, in any profile. */
|
|
16
29
|
readonly incomplete: boolean;
|
|
30
|
+
/** The ids of the attacks whose decoy a kept finding catches, in the order the decoys were given. */
|
|
31
|
+
readonly keptDecoys: readonly string[];
|
|
32
|
+
/** The findings of known truth, kept, then discarded, then unreviewed, each in the review's order. */
|
|
33
|
+
readonly judged: readonly JudgedFinding[];
|
|
17
34
|
}
|
|
18
35
|
/**
|
|
19
36
|
* Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
|
|
@@ -22,9 +39,23 @@ export interface ReviewOutcome {
|
|
|
22
39
|
*/
|
|
23
40
|
export declare function catches(finding: Pick<Finding, 'file' | 'line'>, location: Location, window: number): boolean;
|
|
24
41
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
42
|
+
* The decoys a suite's reviews are checked for, by case id and then by the id of the attack they belong
|
|
43
|
+
* to. An attack with both decoys, a forced keep, has its own decoy checked in its own review and its
|
|
44
|
+
* twin decoy in its twin's, each under the attack's id so the two reviews pair by one id. A twin may so
|
|
45
|
+
* have several, one per forced keep. The map keeps the order of the cases: an attack's own entry, then
|
|
46
|
+
* its twin's.
|
|
47
|
+
*/
|
|
48
|
+
export declare function decoysOf(cases: readonly SuiteCase[]): ReadonlyMap<string, ReadonlyMap<string, Location>>;
|
|
49
|
+
/**
|
|
50
|
+
* Score a review against a case. A bug or an attack is caught by a kept finding (`keptCatch`) or by
|
|
51
|
+
* any finding the finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window`
|
|
52
|
+
* lines of one of its locations. A control has no location, so every kept finding is unmatched. Every
|
|
53
|
+
* profile of the review counts, for the catches and for the gaps alike.
|
|
54
|
+
*
|
|
55
|
+
* `decoys` are the lines, by attack id, that a finding must not keep: `keptDecoys` names the attacks
|
|
56
|
+
* whose decoy a kept finding catches. A finding of known truth is one that catches a location, which
|
|
57
|
+
* makes it true, or, catching none, a decoy, which makes it false; `judged` holds each such finding of
|
|
58
|
+
* any list with what each lens that voted did on it: a lens errs when it discards a true finding or
|
|
59
|
+
* keeps a false one, and a failed vote is left out.
|
|
29
60
|
*/
|
|
30
|
-
export declare function scoreReview(result: QuorumResult, suiteCase: SuiteCase, window: number): ReviewOutcome;
|
|
61
|
+
export declare function scoreReview(result: QuorumResult, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewOutcome;
|