@tobi-or-not/quorum 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +3 -2
- package/dist/bench/fingerprint.js +3 -2
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +171 -13
- package/dist/bench/records.js +34 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +29 -17
- package/dist/bench/runner.js +53 -18
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +20 -9
- package/dist/cli/bench.js +148 -74
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.js +2 -19
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +77 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +1 -1
package/dist/cli/bench.js
CHANGED
|
@@ -4,20 +4,21 @@
|
|
|
4
4
|
* caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
|
|
5
5
|
*/
|
|
6
6
|
import { resolve } from 'node:path';
|
|
7
|
-
import { parseArgs } from 'node:util';
|
|
8
7
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
8
|
+
import { flaggedCases } from "../bench/attacks.js";
|
|
9
9
|
import { fingerprint, newSalt } from "../bench/fingerprint.js";
|
|
10
10
|
import { MAX_CONFIGURATIONS, reviewKey, RUN_FORMAT } from "../bench/records.js";
|
|
11
|
-
import { buildReport, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
|
|
12
|
-
import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic } from "../bench/run-dir.js";
|
|
11
|
+
import { buildReport, kindCounts, kindsText, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
|
|
12
|
+
import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic, writeRunFile } from "../bench/run-dir.js";
|
|
13
13
|
import { planReviews, runReviews } from "../bench/runner.js";
|
|
14
|
+
import { decoysOf } from "../bench/score.js";
|
|
14
15
|
import { loadSuite, MAX_TRIALS, SUITE_ID } from "../bench/suite.js";
|
|
15
16
|
import { readEnv } from "../config/env.js";
|
|
16
17
|
import { fromConfig } from "../config/from-config.js";
|
|
17
18
|
import { loadConfig } from "../config/load.js";
|
|
18
19
|
import { ConfigError } from "../core/errors.js";
|
|
19
20
|
import { inertLine } from "../core/sanitize.js";
|
|
20
|
-
import { errorLines, errorMessage } from "./io.js";
|
|
21
|
+
import { errorLines, errorMessage, parseFlags } from "./io.js";
|
|
21
22
|
import { profileProviders } from "./render.js";
|
|
22
23
|
export const BENCH_USAGE = [
|
|
23
24
|
'Usage: quorum bench [options] <suite>',
|
|
@@ -32,13 +33,15 @@ export const BENCH_USAGE = [
|
|
|
32
33
|
' -j, --jobs <n> The reviews run at once (default 1)',
|
|
33
34
|
' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
|
|
34
35
|
' --retry-incomplete Run again the recorded reviews that came back incomplete',
|
|
36
|
+
' --report-only Write the reports of what is recorded, and run no review',
|
|
35
37
|
' --dry-run Check the suite and the configs, print the plan, and exit',
|
|
36
38
|
' -h, --help Print this help and exit',
|
|
37
39
|
'',
|
|
38
40
|
].join('\n');
|
|
39
41
|
/**
|
|
40
42
|
* The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
|
|
41
|
-
* not recorded, not that a review left a gap, and `stopped` means the
|
|
43
|
+
* not recorded, not that a review left a gap or that a model's reply denied it, and `stopped` means the
|
|
44
|
+
* run could not go on.
|
|
42
45
|
*/
|
|
43
46
|
const BENCH_EXIT = {
|
|
44
47
|
ok: 0,
|
|
@@ -49,7 +52,9 @@ const BENCH_EXIT = {
|
|
|
49
52
|
};
|
|
50
53
|
const MAX_JOBS = 16;
|
|
51
54
|
const DEFAULT_CONFIGURATION = '(default configuration)';
|
|
55
|
+
const UNPLANNED_CASE = 'a review names a case the plan does not hold';
|
|
52
56
|
const TORN_LINE = 'warning: results.jsonl: its last line was cut short, as a stopped run leaves it: that review runs again';
|
|
57
|
+
const RESCORED_LINE = 'Scoring: suite.yaml changed since the run began; its recorded reviews are scored again under it, with no call';
|
|
53
58
|
const OPTIONS = {
|
|
54
59
|
config: { type: 'string', short: 'c', multiple: true },
|
|
55
60
|
profile: { type: 'string', short: 'p', multiple: true },
|
|
@@ -58,25 +63,10 @@ const OPTIONS = {
|
|
|
58
63
|
jobs: { type: 'string', short: 'j' },
|
|
59
64
|
out: { type: 'string', short: 'o' },
|
|
60
65
|
'retry-incomplete': { type: 'boolean' },
|
|
66
|
+
'report-only': { type: 'boolean' },
|
|
61
67
|
'dry-run': { type: 'boolean' },
|
|
62
68
|
help: { type: 'boolean', short: 'h' },
|
|
63
69
|
};
|
|
64
|
-
/** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
65
|
-
function parseFlags(argv) {
|
|
66
|
-
try {
|
|
67
|
-
return parseArgs({
|
|
68
|
-
args: [...argv],
|
|
69
|
-
options: OPTIONS,
|
|
70
|
-
strict: true,
|
|
71
|
-
allowPositionals: true,
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
|
|
76
|
-
throw error;
|
|
77
|
-
throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
70
|
function usageIssue(path, message) {
|
|
81
71
|
return new ConfigError([
|
|
82
72
|
{ path, message },
|
|
@@ -90,8 +80,16 @@ function wholeNumber(flag, text, max) {
|
|
|
90
80
|
throw usageIssue(flag, `must be a whole number from 1 to ${max}`);
|
|
91
81
|
return Number(text);
|
|
92
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* The first of `configs`, as given, whose file a later one names again. A relative `--config` resolves
|
|
85
|
+
* against `cwd`, as `loadConfig` resolves it, so `a.yaml`, `./a.yaml` and its absolute path are one file.
|
|
86
|
+
*/
|
|
87
|
+
function repeatedConfig(configs, cwd) {
|
|
88
|
+
const paths = configs.map(config => resolve(cwd, config));
|
|
89
|
+
return configs.find((_, index) => paths.includes(paths[index], index + 1));
|
|
90
|
+
}
|
|
93
91
|
/** The options the flags of `argv` give, or the usage error they hold. */
|
|
94
|
-
function checkOptions(values, positionals) {
|
|
92
|
+
function checkOptions(values, positionals, cwd) {
|
|
95
93
|
const [suite, ...extra] = positionals;
|
|
96
94
|
if (suite === undefined)
|
|
97
95
|
throw new Error('no suite given');
|
|
@@ -104,6 +102,9 @@ function checkOptions(values, positionals) {
|
|
|
104
102
|
throw usageIssue('--config', 'must name a file');
|
|
105
103
|
if (configs.length > MAX_CONFIGURATIONS)
|
|
106
104
|
throw usageIssue('--config', `names more than ${MAX_CONFIGURATIONS} configurations`);
|
|
105
|
+
const repeated = repeatedConfig(configs, cwd);
|
|
106
|
+
if (repeated !== undefined)
|
|
107
|
+
throw usageIssue('--config', `names the file "${repeated}" more than once`);
|
|
107
108
|
if (values.out === '')
|
|
108
109
|
throw usageIssue('--out', 'must name a directory');
|
|
109
110
|
const cases = values.case ?? [];
|
|
@@ -111,6 +112,8 @@ function checkOptions(values, positionals) {
|
|
|
111
112
|
throw usageIssue('--case', `must be a case id matching ${SUITE_ID.source}`);
|
|
112
113
|
if (new Set(cases).size < cases.length)
|
|
113
114
|
throw usageIssue('--case', 'names a case more than once');
|
|
115
|
+
if (values['report-only'] === true && values['retry-incomplete'] === true)
|
|
116
|
+
throw usageIssue('--report-only', 'runs no review, so it cannot be given with --retry-incomplete');
|
|
114
117
|
return {
|
|
115
118
|
suite,
|
|
116
119
|
configs,
|
|
@@ -120,6 +123,7 @@ function checkOptions(values, positionals) {
|
|
|
120
123
|
jobs: jobs ?? 1,
|
|
121
124
|
out: values.out,
|
|
122
125
|
retryIncomplete: values['retry-incomplete'] === true,
|
|
126
|
+
reportOnly: values['report-only'] === true,
|
|
123
127
|
dryRun: values['dry-run'] === true,
|
|
124
128
|
};
|
|
125
129
|
}
|
|
@@ -131,11 +135,6 @@ function writeLines(stream, lines) {
|
|
|
131
135
|
function nameOf({ path }) {
|
|
132
136
|
return path ?? DEFAULT_CONFIGURATION;
|
|
133
137
|
}
|
|
134
|
-
function plural(count, noun) {
|
|
135
|
-
return `${count} ${noun}${count === 1
|
|
136
|
-
? ''
|
|
137
|
-
: 's'}`;
|
|
138
|
-
}
|
|
139
138
|
/** The cases of `suite` the flags select, in the suite's order; every `--case` must be one of its cases. */
|
|
140
139
|
function selectCases(suite, ids) {
|
|
141
140
|
const unknown = ids.filter(id => !suite.cases.some(suiteCase => suiteCase.id === id));
|
|
@@ -201,17 +200,25 @@ function verdictOf(suiteCase, score) {
|
|
|
201
200
|
? 'caught by a proposed finding only'
|
|
202
201
|
: 'missed';
|
|
203
202
|
}
|
|
203
|
+
/** What a denied review's line says: words the bench wrote, never the model's reply. */
|
|
204
|
+
const DENIAL_WORDS = {
|
|
205
|
+
'unusable-reply': 'the research reply could not be used',
|
|
206
|
+
refusal: 'the model refused the research request',
|
|
207
|
+
truncated: 'the research reply was cut off at its token limit',
|
|
208
|
+
};
|
|
204
209
|
/** The progress line of a review that was recorded, after `[<n>/<total>] trial <t>, case <id>, configuration <i>: `. */
|
|
205
210
|
function recordedText(record, suiteCase, score) {
|
|
211
|
+
const latency = `${(record.latencyMs / 1000).toFixed(1)} s`;
|
|
212
|
+
if ('denied' in record)
|
|
213
|
+
return `denied, ${DENIAL_WORDS[record.denied]}; ${latency}`;
|
|
206
214
|
const { kept, discarded, unreviewed, meta } = record.result;
|
|
207
|
-
return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${
|
|
215
|
+
return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${latency}`;
|
|
208
216
|
}
|
|
209
|
-
/** The plan a run prints before its first call. */
|
|
217
|
+
/** The plan a run prints before its first call; `rescored` adds the line that says its recorded reviews are scored again. */
|
|
210
218
|
function planLines(plan) {
|
|
211
|
-
const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory } = plan;
|
|
212
|
-
const bugs = selected.filter(({ kind }) => kind === 'bug').length;
|
|
219
|
+
const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory, rescored } = plan;
|
|
213
220
|
return [
|
|
214
|
-
`Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${
|
|
221
|
+
`Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${kindsText(kindCounts(selected))}`,
|
|
215
222
|
'Configurations:',
|
|
216
223
|
...loaded.flatMap(({ path, config }, index) => [
|
|
217
224
|
` ${index + 1}. ${path ?? DEFAULT_CONFIGURATION}${loaded.length > 1 && index === 0
|
|
@@ -223,6 +230,9 @@ function planLines(plan) {
|
|
|
223
230
|
`Reviews: ${planned} planned, ${recorded} recorded, ${toRun} to run${again > 0
|
|
224
231
|
? ` (${again} of them again)`
|
|
225
232
|
: ''}, at most ${jobs} at a time`,
|
|
233
|
+
...rescored
|
|
234
|
+
? [RESCORED_LINE]
|
|
235
|
+
: [],
|
|
226
236
|
`Run directory: ${directory}`,
|
|
227
237
|
];
|
|
228
238
|
}
|
|
@@ -238,6 +248,8 @@ async function check(io, options) {
|
|
|
238
248
|
const directory = options.out ?? `quorum-bench/${suite.name}`;
|
|
239
249
|
const out = resolve(io.cwd, directory);
|
|
240
250
|
const state = await inspectRunDir(out);
|
|
251
|
+
if (options.reportOnly && state.kind !== 'run')
|
|
252
|
+
throw usageIssue('--out', 'holds no run, so --report-only has nothing to report');
|
|
241
253
|
const salt = state.kind === 'run'
|
|
242
254
|
? state.run.salt
|
|
243
255
|
: newSalt();
|
|
@@ -251,7 +263,14 @@ async function check(io, options) {
|
|
|
251
263
|
const runFile = {
|
|
252
264
|
format: RUN_FORMAT,
|
|
253
265
|
quorum: packageJson.version,
|
|
254
|
-
suite: {
|
|
266
|
+
suite: {
|
|
267
|
+
name: suite.name,
|
|
268
|
+
reviewHash: suite.reviewHash,
|
|
269
|
+
scoringHash: suite.scoringHash,
|
|
270
|
+
firstScoringHash: state.kind === 'run'
|
|
271
|
+
? state.run.suite.firstScoringHash
|
|
272
|
+
: suite.scoringHash,
|
|
273
|
+
},
|
|
255
274
|
salt,
|
|
256
275
|
configurations,
|
|
257
276
|
};
|
|
@@ -266,28 +285,34 @@ async function check(io, options) {
|
|
|
266
285
|
selected,
|
|
267
286
|
loaded,
|
|
268
287
|
byId,
|
|
288
|
+
decoys: decoysOf(suite.cases),
|
|
269
289
|
directory,
|
|
270
290
|
out,
|
|
271
291
|
state,
|
|
272
292
|
runFile,
|
|
273
293
|
};
|
|
274
294
|
}
|
|
275
|
-
/** `record` scored against its case and kept in `scores` under its key, replacing an earlier record of the same review. */
|
|
276
|
-
function keepScore(scores, record, suiteCase,
|
|
277
|
-
const scored = reviewScore(record, suiteCase, window);
|
|
295
|
+
/** `record` scored against its case and its decoys, and kept in `scores` under its key, replacing an earlier record of the same review. */
|
|
296
|
+
function keepScore(scores, record, suiteCase, { suite, decoys }) {
|
|
297
|
+
const scored = reviewScore(record, suiteCase, suite.window, decoys.get(suiteCase.id));
|
|
278
298
|
scores.set(reviewKey(record.fingerprint, record.case, record.trial), scored);
|
|
279
299
|
return scored;
|
|
280
300
|
}
|
|
281
301
|
/**
|
|
282
|
-
* The score of every review the run being resumed holds, by `reviewKey`, and
|
|
283
|
-
* (
|
|
284
|
-
*
|
|
285
|
-
* A new run holds nothing.
|
|
302
|
+
* The score of every review the run being resumed holds, by `reviewKey`, and what the read found of the
|
|
303
|
+
* file (where its valid lines end and how long it was, which `openRecords` checks). A line that names a
|
|
304
|
+
* configuration the run does not have, or a case the suite does not have, refuses the run directory. A
|
|
305
|
+
* last line a killed run cut short is not scored, and says so. A new run holds nothing.
|
|
286
306
|
*/
|
|
287
|
-
async function readScores(io,
|
|
307
|
+
async function readScores(io, checked) {
|
|
308
|
+
const { byId, out, state, runFile } = checked;
|
|
288
309
|
const scores = new Map();
|
|
289
|
-
if (state.kind !== 'run')
|
|
290
|
-
return {
|
|
310
|
+
if (state.kind !== 'run') {
|
|
311
|
+
return {
|
|
312
|
+
scores,
|
|
313
|
+
records: { validBytes: 0, bytes: 0 },
|
|
314
|
+
};
|
|
315
|
+
}
|
|
291
316
|
const known = new Set(runFile.configurations.map(({ fingerprint }) => fingerprint));
|
|
292
317
|
const read = await readRecords(out, (record) => {
|
|
293
318
|
const suiteCase = byId.get(record.case);
|
|
@@ -295,25 +320,45 @@ async function readScores(io, { suite, byId, out, state, runFile }) {
|
|
|
295
320
|
return 'names a configuration this run does not have';
|
|
296
321
|
if (suiteCase === undefined)
|
|
297
322
|
return 'names a case the suite does not have';
|
|
298
|
-
keepScore(scores, record, suiteCase,
|
|
323
|
+
keepScore(scores, record, suiteCase, checked);
|
|
299
324
|
return undefined;
|
|
300
325
|
});
|
|
301
326
|
if (read.torn)
|
|
302
327
|
writeLines(io.stderr, [TORN_LINE]);
|
|
303
|
-
return {
|
|
328
|
+
return {
|
|
329
|
+
scores,
|
|
330
|
+
records: { validBytes: read.validBytes, bytes: read.bytes },
|
|
331
|
+
};
|
|
304
332
|
}
|
|
305
333
|
/**
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
* rejects, with the error of a failed write, that error is the one thrown.
|
|
334
|
+
* Write `run.json` as the run needs it: a new run is made, with its directory; a run being resumed whose
|
|
335
|
+
* recorded scoring hash is not the suite's now is written again with that scoring hash alone changed, so
|
|
336
|
+
* its first scoring hash, its configurations' paths and order, its salt and its version stay as recorded;
|
|
337
|
+
* any other run is left as it is.
|
|
311
338
|
*/
|
|
312
|
-
async function
|
|
313
|
-
|
|
314
|
-
if (state.kind !== 'run')
|
|
339
|
+
async function writeRun({ out, state, runFile }) {
|
|
340
|
+
if (state.kind !== 'run') {
|
|
315
341
|
await createRun(out, runFile);
|
|
316
|
-
|
|
342
|
+
}
|
|
343
|
+
else if (state.run.suite.scoringHash !== runFile.suite.scoringHash) {
|
|
344
|
+
await writeRunFile(out, {
|
|
345
|
+
...state.run,
|
|
346
|
+
suite: { ...state.run.suite, scoringHash: runFile.suite.scoringHash },
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Write `run.json` (see `writeRun`), open `results.jsonl` and run the reviews of `toRun`, each recorded as
|
|
352
|
+
* it resolves and each reported with its progress line. The order matters: `writeRun` renames over any
|
|
353
|
+
* run.json and `openRecords` cuts results.jsonl to the bytes the read found valid (and refuses it when
|
|
354
|
+
* its length is not the one the read found), so a run being resumed has been read before either. The
|
|
355
|
+
* writer is closed whatever happens; when the run rejects, with the error of a failed write, that error
|
|
356
|
+
* is the one thrown.
|
|
357
|
+
*/
|
|
358
|
+
async function execute(io, { checked, scores, records, toRun, jobs }) {
|
|
359
|
+
const { suite, loaded, byId, out, runFile } = checked;
|
|
360
|
+
await writeRun(checked);
|
|
361
|
+
const writer = await openRecords(out, records);
|
|
317
362
|
let finished = 0;
|
|
318
363
|
let outcome;
|
|
319
364
|
try {
|
|
@@ -321,7 +366,7 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
321
366
|
suite,
|
|
322
367
|
configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
|
|
323
368
|
reviews: toRun,
|
|
324
|
-
jobs
|
|
369
|
+
jobs,
|
|
325
370
|
env: io.env,
|
|
326
371
|
signal: io.signal,
|
|
327
372
|
append: writer.append,
|
|
@@ -329,8 +374,11 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
329
374
|
finished += 1;
|
|
330
375
|
const { trial, caseId, configuration } = event.review;
|
|
331
376
|
const suiteCase = byId.get(caseId);
|
|
377
|
+
// The plan is made from the selected cases, which `byId` holds, so a review never names another.
|
|
378
|
+
if (suiteCase === undefined)
|
|
379
|
+
throw new Error(UNPLANNED_CASE);
|
|
332
380
|
const text = event.type === 'recorded'
|
|
333
|
-
? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase,
|
|
381
|
+
? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, checked))
|
|
334
382
|
: `failed: ${errorMessage(event.error)}`;
|
|
335
383
|
writeLines(io.stderr, [`[${finished}/${toRun.length}] trial ${trial}, case ${caseId}, configuration ${configuration + 1}: ${text}`]);
|
|
336
384
|
},
|
|
@@ -346,12 +394,13 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
346
394
|
/** Build the report from the scores, write `report.md` and `report.json` into the run directory, and print the Markdown. */
|
|
347
395
|
async function writeReports(io, { suite, selected, out, runFile }, trials, scores) {
|
|
348
396
|
const report = buildReport({
|
|
349
|
-
suite: {
|
|
397
|
+
suite: { ...runFile.suite, window: suite.window },
|
|
350
398
|
quorum: packageJson.version,
|
|
351
399
|
cases: selected,
|
|
352
400
|
trials,
|
|
353
401
|
configurations: runFile.configurations,
|
|
354
402
|
scores,
|
|
403
|
+
flagged: flaggedCases(selected, suite),
|
|
355
404
|
});
|
|
356
405
|
const markdown = reportMarkdown(report);
|
|
357
406
|
await writeAtomic(out, REPORT_MARKDOWN, markdown);
|
|
@@ -362,7 +411,7 @@ async function writeReports(io, { suite, selected, out, runFile }, trials, score
|
|
|
362
411
|
async function bench(io, options) {
|
|
363
412
|
const checked = await check(io, options);
|
|
364
413
|
const { suite, selected, loaded, directory, runFile } = checked;
|
|
365
|
-
const { scores,
|
|
414
|
+
const { scores, records } = await readScores(io, checked);
|
|
366
415
|
const trials = options.trials ?? suite.trials ?? 1;
|
|
367
416
|
const planned = planReviews({
|
|
368
417
|
trials,
|
|
@@ -370,10 +419,12 @@ async function bench(io, options) {
|
|
|
370
419
|
configurations: runFile.configurations.length,
|
|
371
420
|
});
|
|
372
421
|
const scoreOf = ({ configuration, caseId, trial }) => scores.get(reviewKey(runFile.configurations[configuration].fingerprint, caseId, trial));
|
|
373
|
-
const toRun =
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
422
|
+
const toRun = options.reportOnly
|
|
423
|
+
? []
|
|
424
|
+
: planned.filter((review) => {
|
|
425
|
+
const recorded = scoreOf(review);
|
|
426
|
+
return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
|
|
427
|
+
});
|
|
377
428
|
writeLines(io.stderr, loaded.flatMap(({ warnings }, index) => warnings.map(warning => `warning: configuration ${index + 1}: ${warning}`)));
|
|
378
429
|
writeLines(options.dryRun ? io.stdout : io.stderr, planLines({
|
|
379
430
|
suite,
|
|
@@ -386,6 +437,7 @@ async function bench(io, options) {
|
|
|
386
437
|
again: toRun.filter(review => scoreOf(review) !== undefined).length,
|
|
387
438
|
jobs: options.jobs,
|
|
388
439
|
directory,
|
|
440
|
+
rescored: runFile.suite.scoringHash !== runFile.suite.firstScoringHash,
|
|
389
441
|
}));
|
|
390
442
|
if (options.dryRun) {
|
|
391
443
|
writeLines(io.stdout, ['Dry run: no review was run, and nothing was written.']);
|
|
@@ -395,7 +447,18 @@ async function bench(io, options) {
|
|
|
395
447
|
io.stderr.write('interrupted\n');
|
|
396
448
|
return BENCH_EXIT.interrupted;
|
|
397
449
|
}
|
|
398
|
-
|
|
450
|
+
if (options.reportOnly) {
|
|
451
|
+
await writeRun(checked);
|
|
452
|
+
await writeReports(io, checked, trials, scores);
|
|
453
|
+
return BENCH_EXIT.ok;
|
|
454
|
+
}
|
|
455
|
+
const outcome = await execute(io, {
|
|
456
|
+
checked,
|
|
457
|
+
scores,
|
|
458
|
+
records,
|
|
459
|
+
toRun,
|
|
460
|
+
jobs: options.jobs,
|
|
461
|
+
});
|
|
399
462
|
await writeReports(io, checked, trials, scores);
|
|
400
463
|
if (outcome.interrupted) {
|
|
401
464
|
io.stderr.write('interrupted\n');
|
|
@@ -416,33 +479,44 @@ function usageError(io, error) {
|
|
|
416
479
|
* It reviews every case of the suite the argument names with every `--config` (the usual resolution when
|
|
417
480
|
* there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
|
|
418
481
|
* in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
|
|
419
|
-
* at a time
|
|
482
|
+
* at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
|
|
483
|
+
* denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
|
|
420
484
|
* stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
|
|
421
485
|
* on stdout, which it also writes, with `report.json`, into the run directory.
|
|
422
486
|
*
|
|
423
487
|
* A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
|
|
424
488
|
* runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
|
|
425
|
-
* warning. A
|
|
426
|
-
*
|
|
427
|
-
*
|
|
489
|
+
* warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
|
|
490
|
+
* re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
|
|
491
|
+
* the new scoring hash beside the one the run began with, and the plan and the reports say so while the
|
|
492
|
+
* suite is not the one the run began with. A run directory of another suite, another quorum or other
|
|
493
|
+
* configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
|
|
494
|
+
* changed, and a suite or a configuration that does not validate or cannot be built are refused before
|
|
495
|
+
* the first call.
|
|
496
|
+
*
|
|
497
|
+
* `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
|
|
498
|
+
* the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
|
|
499
|
+
* `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
|
|
500
|
+
* holds no run, and `--retry-incomplete`, which runs reviews.
|
|
428
501
|
*
|
|
429
|
-
* Exit codes: 0 when every planned review is recorded,
|
|
430
|
-
* directory error, 3 when the run stops (a file cannot
|
|
431
|
-
*
|
|
432
|
-
*
|
|
433
|
-
*
|
|
502
|
+
* Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
|
|
503
|
+
* reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
|
|
504
|
+
* be written or read), 4 when a review failed (its research call failed, or its record was too large to
|
|
505
|
+
* write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
|
|
506
|
+
* reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
|
|
507
|
+
* checks and the plan leaves the run directory as it was, even one that does not exist yet.
|
|
434
508
|
*
|
|
435
509
|
* Every line it writes is inert (invariant I3).
|
|
436
510
|
*/
|
|
437
511
|
export async function runBench(io, argv) {
|
|
438
512
|
let options;
|
|
439
513
|
try {
|
|
440
|
-
const { values, positionals } = parseFlags(argv);
|
|
514
|
+
const { values, positionals } = parseFlags(argv, OPTIONS);
|
|
441
515
|
if (values.help) {
|
|
442
516
|
io.stdout.write(BENCH_USAGE);
|
|
443
517
|
return BENCH_EXIT.ok;
|
|
444
518
|
}
|
|
445
|
-
options = checkOptions(values, positionals);
|
|
519
|
+
options = checkOptions(values, positionals, io.cwd);
|
|
446
520
|
}
|
|
447
521
|
catch (error) {
|
|
448
522
|
return usageError(io, error);
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review,
|
|
3
|
-
* failure is written. `main.ts` and `bench.ts` both import it, so neither
|
|
2
|
+
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
|
|
3
|
+
* failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
|
|
4
|
+
* imports the other.
|
|
4
5
|
*/
|
|
6
|
+
import type { ParseArgsOptionsConfig } from 'node:util';
|
|
5
7
|
import type { Env } from '../config/env.ts';
|
|
6
8
|
/** Exit codes (ADR-0008, amended by ADR-0012). */
|
|
7
9
|
export declare const EXIT: {
|
|
@@ -43,3 +45,8 @@ export declare function errorLines(error: unknown): string;
|
|
|
43
45
|
* prototype. It never throws, so neither do the lines built from it.
|
|
44
46
|
*/
|
|
45
47
|
export declare function errorMessage(error: unknown): string;
|
|
48
|
+
/** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
49
|
+
export declare function parseFlags<T extends ParseArgsOptionsConfig>(argv: readonly string[], options: T): {
|
|
50
|
+
values: T extends ParseArgsOptionsConfig ? { -readonly [LongOption in keyof T]?: { [LongOption_2 in keyof T]: T[LongOption_2]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_2]["multiple"] ? T_1 extends false ? T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption] | undefined; } & { [LongOption_1 in keyof T as T[LongOption_1]["default"] extends {} ? LongOption_1 : never]: { [LongOption_3 in keyof T]: T[LongOption_3]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_3]["multiple"] ? T_1 extends false ? T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption_1]; } extends infer P ? { [K in keyof P]: P[K]; } : never : {};
|
|
51
|
+
positionals: string[];
|
|
52
|
+
};
|
package/dist/cli/io.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review,
|
|
3
|
-
* failure is written. `main.ts` and `bench.ts` both import it, so neither
|
|
2
|
+
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
|
|
3
|
+
* failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
|
|
4
|
+
* imports the other.
|
|
4
5
|
*/
|
|
6
|
+
import { parseArgs } from 'node:util';
|
|
5
7
|
import { ConfigError } from "../core/errors.js";
|
|
6
8
|
import { inertLine } from "../core/sanitize.js";
|
|
7
9
|
/** Exit codes (ADR-0008, amended by ADR-0012). */
|
|
@@ -43,3 +45,19 @@ export function errorMessage(error) {
|
|
|
43
45
|
return 'an error with no text form';
|
|
44
46
|
}
|
|
45
47
|
}
|
|
48
|
+
/** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
49
|
+
export function parseFlags(argv, options) {
|
|
50
|
+
try {
|
|
51
|
+
return parseArgs({
|
|
52
|
+
args: [...argv],
|
|
53
|
+
options,
|
|
54
|
+
strict: true,
|
|
55
|
+
allowPositionals: true,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
|
|
60
|
+
throw error;
|
|
61
|
+
throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/cli/main.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { createReadStream } from 'node:fs';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import { parseArgs } from 'node:util';
|
|
4
3
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
5
4
|
import { readEnv } from "../config/env.js";
|
|
6
5
|
import { fromConfig } from "../config/from-config.js";
|
|
@@ -9,7 +8,7 @@ import { decodeDiff } from "../core/diff.js";
|
|
|
9
8
|
import { ConfigError } from "../core/errors.js";
|
|
10
9
|
import { inertJson, inertLine } from "../core/sanitize.js";
|
|
11
10
|
import { runBench } from "./bench.js";
|
|
12
|
-
import { errorLines, errorMessage, EXIT } from "./io.js";
|
|
11
|
+
import { errorLines, errorMessage, EXIT, parseFlags } from "./io.js";
|
|
13
12
|
import { printConfig } from "./print-config.js";
|
|
14
13
|
import { bannerLines, progressSink } from "./render.js";
|
|
15
14
|
export { errorMessage, EXIT };
|
|
@@ -70,7 +69,7 @@ export async function runCli(io) {
|
|
|
70
69
|
return runBench(io, io.argv.slice(1));
|
|
71
70
|
let flags;
|
|
72
71
|
try {
|
|
73
|
-
flags = parseFlags(io.argv);
|
|
72
|
+
flags = parseFlags(io.argv, OPTIONS);
|
|
74
73
|
}
|
|
75
74
|
catch (error) {
|
|
76
75
|
return usageError(io, error);
|
|
@@ -188,22 +187,6 @@ export function exitCodeFor(result, options = {}) {
|
|
|
188
187
|
? EXIT.incomplete
|
|
189
188
|
: EXIT.ok;
|
|
190
189
|
}
|
|
191
|
-
/** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
192
|
-
function parseFlags(argv) {
|
|
193
|
-
try {
|
|
194
|
-
return parseArgs({
|
|
195
|
-
args: [...argv],
|
|
196
|
-
options: OPTIONS,
|
|
197
|
-
strict: true,
|
|
198
|
-
allowPositionals: true,
|
|
199
|
-
});
|
|
200
|
-
}
|
|
201
|
-
catch (error) {
|
|
202
|
-
if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
|
|
203
|
-
throw error;
|
|
204
|
-
throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
190
|
/** Write a usage error and the usage, and return its exit code. */
|
|
208
191
|
function usageError(io, error) {
|
|
209
192
|
io.stderr.write(`${errorLines(error)}\n${USAGE}`);
|
package/dist/config/load.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Stats } from 'node:fs';
|
|
1
2
|
import type { z } from 'zod';
|
|
2
3
|
import type { ConfigIssue } from '../core/errors.ts';
|
|
3
4
|
import type { Env } from './env.ts';
|
|
@@ -18,6 +19,8 @@ export type Read<T> = {
|
|
|
18
19
|
readonly problem: string;
|
|
19
20
|
readonly counted?: number;
|
|
20
21
|
};
|
|
22
|
+
/** The `lstat` of a path, or the problem a failed one means by its code: the path is missing, or it cannot be read. */
|
|
23
|
+
export declare function lstatOf(absolute: string): Promise<Read<Stats>>;
|
|
21
24
|
/**
|
|
22
25
|
* The bytes of a regular file of at most `limit` bytes. The file is opened without blocking, so a FIFO
|
|
23
26
|
* is refused instead of waiting for a writer, and at most one byte past the limit is read, however large
|
package/dist/config/load.js
CHANGED
|
@@ -54,20 +54,29 @@ function openProblem(error, noFollow) {
|
|
|
54
54
|
? 'is a symbolic link'
|
|
55
55
|
: 'cannot be read';
|
|
56
56
|
}
|
|
57
|
-
/** The
|
|
58
|
-
async function
|
|
57
|
+
/** The `lstat` of a path, or the problem a failed one means by its code: the path is missing, or it cannot be read. */
|
|
58
|
+
export async function lstatOf(absolute) {
|
|
59
59
|
try {
|
|
60
|
-
return
|
|
61
|
-
? 'is a symbolic link'
|
|
62
|
-
: undefined;
|
|
60
|
+
return { value: await lstat(absolute) };
|
|
63
61
|
}
|
|
64
62
|
catch (error) {
|
|
65
63
|
const code = error.code;
|
|
66
|
-
return
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
return {
|
|
65
|
+
problem: code === 'ENOENT' || code === 'ENOTDIR'
|
|
66
|
+
? 'does not exist'
|
|
67
|
+
: 'cannot be read',
|
|
68
|
+
};
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
+
/** The problem a path has before an open that must not follow a link: it is one, or it is missing or unreadable; undefined when it is not a link. */
|
|
72
|
+
async function linkProblem(absolute) {
|
|
73
|
+
const stats = await lstatOf(absolute);
|
|
74
|
+
if ('problem' in stats)
|
|
75
|
+
return stats.problem;
|
|
76
|
+
return stats.value.isSymbolicLink()
|
|
77
|
+
? 'is a symbolic link'
|
|
78
|
+
: undefined;
|
|
79
|
+
}
|
|
71
80
|
/**
|
|
72
81
|
* The bytes of a regular file of at most `limit` bytes. The file is opened without blocking, so a FIFO
|
|
73
82
|
* is refused instead of waiting for a writer, and at most one byte past the limit is read, however large
|
package/dist/core/diff.d.ts
CHANGED
|
@@ -52,6 +52,16 @@ export interface FindingLocation {
|
|
|
52
52
|
* hunk (`+c,0`) holds at `c`.
|
|
53
53
|
*/
|
|
54
54
|
export declare function locate(sections: ReadonlyMap<string, string>, file: string, line: number): FindingLocation | undefined;
|
|
55
|
+
/**
|
|
56
|
+
* The first line from `start` to `end` that no hunk of `file` holds, or undefined when each sits in
|
|
57
|
+
* one: the answer `locate` gives for each line in turn, found by a binary search over the spans the
|
|
58
|
+
* hunks hold, merged and sorted once per section, so a range costs a search and not a walk over every
|
|
59
|
+
* hunk for each of its lines. The file is resolved as `locate` resolves it, and a file the diff does
|
|
60
|
+
* not hold has no line in a hunk, so the answer is `start`.
|
|
61
|
+
*
|
|
62
|
+
* @throws {RangeError} unless `start` and `end` are safe integers with `start` at most `end`
|
|
63
|
+
*/
|
|
64
|
+
export declare function firstLineOutside(sections: ReadonlyMap<string, string>, file: string, start: number, end: number): number | undefined;
|
|
55
65
|
/** A line the diff adds to the new file. */
|
|
56
66
|
export interface AddedLine {
|
|
57
67
|
/** The line's number in the new file. */
|