@tobi-or-not/quorum 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +7 -4
- package/dist/bench/fingerprint.js +22 -8
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +241 -13
- package/dist/bench/records.js +64 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +47 -17
- package/dist/bench/runner.js +82 -21
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +27 -9
- package/dist/cli/bench.js +174 -83
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +100 -38
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +34 -0
- package/dist/core/diff.js +140 -18
- package/dist/core/errors.d.ts +18 -3
- package/dist/core/errors.js +26 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +126 -22
- package/dist/core/stages/context.js +319 -41
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +14 -6
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +69 -33
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +72 -15
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +17 -9
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +17 -9
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
package/dist/cli/bench.js
CHANGED
|
@@ -4,20 +4,21 @@
|
|
|
4
4
|
* caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
|
|
5
5
|
*/
|
|
6
6
|
import { resolve } from 'node:path';
|
|
7
|
-
import { parseArgs } from 'node:util';
|
|
8
7
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
8
|
+
import { flaggedCases } from "../bench/attacks.js";
|
|
9
9
|
import { fingerprint, newSalt } from "../bench/fingerprint.js";
|
|
10
10
|
import { MAX_CONFIGURATIONS, reviewKey, RUN_FORMAT } from "../bench/records.js";
|
|
11
|
-
import { buildReport, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
|
|
12
|
-
import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic } from "../bench/run-dir.js";
|
|
11
|
+
import { buildReport, kindCounts, kindsText, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
|
|
12
|
+
import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic, writeRunFile } from "../bench/run-dir.js";
|
|
13
13
|
import { planReviews, runReviews } from "../bench/runner.js";
|
|
14
|
+
import { decoysOf } from "../bench/score.js";
|
|
14
15
|
import { loadSuite, MAX_TRIALS, SUITE_ID } from "../bench/suite.js";
|
|
15
16
|
import { readEnv } from "../config/env.js";
|
|
16
17
|
import { fromConfig } from "../config/from-config.js";
|
|
17
18
|
import { loadConfig } from "../config/load.js";
|
|
18
19
|
import { ConfigError } from "../core/errors.js";
|
|
19
20
|
import { inertLine } from "../core/sanitize.js";
|
|
20
|
-
import { errorLines, errorMessage } from "./io.js";
|
|
21
|
+
import { errorLines, errorMessage, parseFlags } from "./io.js";
|
|
21
22
|
import { profileProviders } from "./render.js";
|
|
22
23
|
export const BENCH_USAGE = [
|
|
23
24
|
'Usage: quorum bench [options] <suite>',
|
|
@@ -25,20 +26,23 @@ export const BENCH_USAGE = [
|
|
|
25
26
|
'Review every case of a suite with every config, --trials times each, and report what each caught.',
|
|
26
27
|
'',
|
|
27
28
|
'Options:',
|
|
28
|
-
' -c, --config <path>
|
|
29
|
-
' -p, --profile <id>
|
|
30
|
-
' --trials <n>
|
|
31
|
-
' --case <id>
|
|
32
|
-
' -j, --jobs <n>
|
|
33
|
-
'
|
|
34
|
-
'
|
|
35
|
-
' --
|
|
36
|
-
'
|
|
29
|
+
' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
|
|
30
|
+
' -p, --profile <id> Run only this profile; repeat it to run several',
|
|
31
|
+
' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
|
|
32
|
+
' --case <id> Run only this case; repeat it to run several',
|
|
33
|
+
' -j, --jobs <n> The reviews run at once (default 1)',
|
|
34
|
+
' --max-run-tokens <n> Start no review once the run has spent <n> tokens; the next run runs the rest',
|
|
35
|
+
' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
|
|
36
|
+
' --retry-incomplete Run again the recorded reviews that came back incomplete',
|
|
37
|
+
' --report-only Write the reports of what is recorded, and run no review',
|
|
38
|
+
' --dry-run Check the suite and the configs, print the plan, and exit',
|
|
39
|
+
' -h, --help Print this help and exit',
|
|
37
40
|
'',
|
|
38
41
|
].join('\n');
|
|
39
42
|
/**
|
|
40
43
|
* The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
|
|
41
|
-
* not recorded,
|
|
44
|
+
* not recorded, or that `--max-run-tokens` left reviews unrun, not that a review left a gap or that a
|
|
45
|
+
* model's reply denied it, and `stopped` means the run could not go on.
|
|
42
46
|
*/
|
|
43
47
|
const BENCH_EXIT = {
|
|
44
48
|
ok: 0,
|
|
@@ -49,34 +53,22 @@ const BENCH_EXIT = {
|
|
|
49
53
|
};
|
|
50
54
|
const MAX_JOBS = 16;
|
|
51
55
|
const DEFAULT_CONFIGURATION = '(default configuration)';
|
|
56
|
+
const UNPLANNED_CASE = 'a review names a case the plan does not hold';
|
|
52
57
|
const TORN_LINE = 'warning: results.jsonl: its last line was cut short, as a stopped run leaves it: that review runs again';
|
|
58
|
+
const RESCORED_LINE = 'Scoring: suite.yaml changed since the run began; its recorded reviews are scored again under it, with no call';
|
|
53
59
|
const OPTIONS = {
|
|
54
60
|
config: { type: 'string', short: 'c', multiple: true },
|
|
55
61
|
profile: { type: 'string', short: 'p', multiple: true },
|
|
56
62
|
trials: { type: 'string' },
|
|
57
63
|
case: { type: 'string', multiple: true },
|
|
58
64
|
jobs: { type: 'string', short: 'j' },
|
|
65
|
+
'max-run-tokens': { type: 'string' },
|
|
59
66
|
out: { type: 'string', short: 'o' },
|
|
60
67
|
'retry-incomplete': { type: 'boolean' },
|
|
68
|
+
'report-only': { type: 'boolean' },
|
|
61
69
|
'dry-run': { type: 'boolean' },
|
|
62
70
|
help: { type: 'boolean', short: 'h' },
|
|
63
71
|
};
|
|
64
|
-
/** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
65
|
-
function parseFlags(argv) {
|
|
66
|
-
try {
|
|
67
|
-
return parseArgs({
|
|
68
|
-
args: [...argv],
|
|
69
|
-
options: OPTIONS,
|
|
70
|
-
strict: true,
|
|
71
|
-
allowPositionals: true,
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
|
|
76
|
-
throw error;
|
|
77
|
-
throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
72
|
function usageIssue(path, message) {
|
|
81
73
|
return new ConfigError([
|
|
82
74
|
{ path, message },
|
|
@@ -90,8 +82,16 @@ function wholeNumber(flag, text, max) {
|
|
|
90
82
|
throw usageIssue(flag, `must be a whole number from 1 to ${max}`);
|
|
91
83
|
return Number(text);
|
|
92
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* The first of `configs`, as given, whose file a later one names again. A relative `--config` resolves
|
|
87
|
+
* against `cwd`, as `loadConfig` resolves it, so `a.yaml`, `./a.yaml` and its absolute path are one file.
|
|
88
|
+
*/
|
|
89
|
+
function repeatedConfig(configs, cwd) {
|
|
90
|
+
const paths = configs.map(config => resolve(cwd, config));
|
|
91
|
+
return configs.find((_, index) => paths.includes(paths[index], index + 1));
|
|
92
|
+
}
|
|
93
93
|
/** The options the flags of `argv` give, or the usage error they hold. */
|
|
94
|
-
function checkOptions(values, positionals) {
|
|
94
|
+
function checkOptions(values, positionals, cwd) {
|
|
95
95
|
const [suite, ...extra] = positionals;
|
|
96
96
|
if (suite === undefined)
|
|
97
97
|
throw new Error('no suite given');
|
|
@@ -99,11 +99,15 @@ function checkOptions(values, positionals) {
|
|
|
99
99
|
throw new Error(`unexpected arguments: ${extra.join(' ')}`);
|
|
100
100
|
const trials = wholeNumber('--trials', values.trials, MAX_TRIALS);
|
|
101
101
|
const jobs = wholeNumber('--jobs', values.jobs, MAX_JOBS);
|
|
102
|
+
const maxRunTokens = wholeNumber('--max-run-tokens', values['max-run-tokens'], Number.MAX_SAFE_INTEGER);
|
|
102
103
|
const configs = values.config ?? [];
|
|
103
104
|
if (configs.includes(''))
|
|
104
105
|
throw usageIssue('--config', 'must name a file');
|
|
105
106
|
if (configs.length > MAX_CONFIGURATIONS)
|
|
106
107
|
throw usageIssue('--config', `names more than ${MAX_CONFIGURATIONS} configurations`);
|
|
108
|
+
const repeated = repeatedConfig(configs, cwd);
|
|
109
|
+
if (repeated !== undefined)
|
|
110
|
+
throw usageIssue('--config', `names the file "${repeated}" more than once`);
|
|
107
111
|
if (values.out === '')
|
|
108
112
|
throw usageIssue('--out', 'must name a directory');
|
|
109
113
|
const cases = values.case ?? [];
|
|
@@ -111,6 +115,8 @@ function checkOptions(values, positionals) {
|
|
|
111
115
|
throw usageIssue('--case', `must be a case id matching ${SUITE_ID.source}`);
|
|
112
116
|
if (new Set(cases).size < cases.length)
|
|
113
117
|
throw usageIssue('--case', 'names a case more than once');
|
|
118
|
+
if (values['report-only'] === true && values['retry-incomplete'] === true)
|
|
119
|
+
throw usageIssue('--report-only', 'runs no review, so it cannot be given with --retry-incomplete');
|
|
114
120
|
return {
|
|
115
121
|
suite,
|
|
116
122
|
configs,
|
|
@@ -118,8 +124,10 @@ function checkOptions(values, positionals) {
|
|
|
118
124
|
trials,
|
|
119
125
|
cases,
|
|
120
126
|
jobs: jobs ?? 1,
|
|
127
|
+
maxRunTokens,
|
|
121
128
|
out: values.out,
|
|
122
129
|
retryIncomplete: values['retry-incomplete'] === true,
|
|
130
|
+
reportOnly: values['report-only'] === true,
|
|
123
131
|
dryRun: values['dry-run'] === true,
|
|
124
132
|
};
|
|
125
133
|
}
|
|
@@ -131,11 +139,6 @@ function writeLines(stream, lines) {
|
|
|
131
139
|
function nameOf({ path }) {
|
|
132
140
|
return path ?? DEFAULT_CONFIGURATION;
|
|
133
141
|
}
|
|
134
|
-
function plural(count, noun) {
|
|
135
|
-
return `${count} ${noun}${count === 1
|
|
136
|
-
? ''
|
|
137
|
-
: 's'}`;
|
|
138
|
-
}
|
|
139
142
|
/** The cases of `suite` the flags select, in the suite's order; every `--case` must be one of its cases. */
|
|
140
143
|
function selectCases(suite, ids) {
|
|
141
144
|
const unknown = ids.filter(id => !suite.cases.some(suiteCase => suiteCase.id === id));
|
|
@@ -201,17 +204,25 @@ function verdictOf(suiteCase, score) {
|
|
|
201
204
|
? 'caught by a proposed finding only'
|
|
202
205
|
: 'missed';
|
|
203
206
|
}
|
|
207
|
+
/** What a denied review's line says: words the bench wrote, never the model's reply. */
|
|
208
|
+
const DENIAL_WORDS = {
|
|
209
|
+
'unusable-reply': 'the research reply could not be used',
|
|
210
|
+
refusal: 'the model refused the research request',
|
|
211
|
+
truncated: 'the research reply was cut off at its token limit',
|
|
212
|
+
};
|
|
204
213
|
/** The progress line of a review that was recorded, after `[<n>/<total>] trial <t>, case <id>, configuration <i>: `. */
|
|
205
214
|
function recordedText(record, suiteCase, score) {
|
|
215
|
+
const latency = `${(record.latencyMs / 1000).toFixed(1)} s`;
|
|
216
|
+
if ('denied' in record)
|
|
217
|
+
return `denied, ${DENIAL_WORDS[record.denied]}; ${latency}`;
|
|
206
218
|
const { kept, discarded, unreviewed, meta } = record.result;
|
|
207
|
-
return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${
|
|
219
|
+
return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${latency}`;
|
|
208
220
|
}
|
|
209
|
-
/** The plan a run prints before its first call. */
|
|
221
|
+
/** The plan a run prints before its first call; `rescored` adds the line that says its recorded reviews are scored again. */
|
|
210
222
|
function planLines(plan) {
|
|
211
|
-
const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory } = plan;
|
|
212
|
-
const bugs = selected.filter(({ kind }) => kind === 'bug').length;
|
|
223
|
+
const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory, rescored } = plan;
|
|
213
224
|
return [
|
|
214
|
-
`Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${
|
|
225
|
+
`Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${kindsText(kindCounts(selected))}`,
|
|
215
226
|
'Configurations:',
|
|
216
227
|
...loaded.flatMap(({ path, config }, index) => [
|
|
217
228
|
` ${index + 1}. ${path ?? DEFAULT_CONFIGURATION}${loaded.length > 1 && index === 0
|
|
@@ -223,6 +234,9 @@ function planLines(plan) {
|
|
|
223
234
|
`Reviews: ${planned} planned, ${recorded} recorded, ${toRun} to run${again > 0
|
|
224
235
|
? ` (${again} of them again)`
|
|
225
236
|
: ''}, at most ${jobs} at a time`,
|
|
237
|
+
...rescored
|
|
238
|
+
? [RESCORED_LINE]
|
|
239
|
+
: [],
|
|
226
240
|
`Run directory: ${directory}`,
|
|
227
241
|
];
|
|
228
242
|
}
|
|
@@ -238,6 +252,8 @@ async function check(io, options) {
|
|
|
238
252
|
const directory = options.out ?? `quorum-bench/${suite.name}`;
|
|
239
253
|
const out = resolve(io.cwd, directory);
|
|
240
254
|
const state = await inspectRunDir(out);
|
|
255
|
+
if (options.reportOnly && state.kind !== 'run')
|
|
256
|
+
throw usageIssue('--out', 'holds no run, so --report-only has nothing to report');
|
|
241
257
|
const salt = state.kind === 'run'
|
|
242
258
|
? state.run.salt
|
|
243
259
|
: newSalt();
|
|
@@ -251,7 +267,14 @@ async function check(io, options) {
|
|
|
251
267
|
const runFile = {
|
|
252
268
|
format: RUN_FORMAT,
|
|
253
269
|
quorum: packageJson.version,
|
|
254
|
-
suite: {
|
|
270
|
+
suite: {
|
|
271
|
+
name: suite.name,
|
|
272
|
+
reviewHash: suite.reviewHash,
|
|
273
|
+
scoringHash: suite.scoringHash,
|
|
274
|
+
firstScoringHash: state.kind === 'run'
|
|
275
|
+
? state.run.suite.firstScoringHash
|
|
276
|
+
: suite.scoringHash,
|
|
277
|
+
},
|
|
255
278
|
salt,
|
|
256
279
|
configurations,
|
|
257
280
|
};
|
|
@@ -266,28 +289,34 @@ async function check(io, options) {
|
|
|
266
289
|
selected,
|
|
267
290
|
loaded,
|
|
268
291
|
byId,
|
|
292
|
+
decoys: decoysOf(suite.cases),
|
|
269
293
|
directory,
|
|
270
294
|
out,
|
|
271
295
|
state,
|
|
272
296
|
runFile,
|
|
273
297
|
};
|
|
274
298
|
}
|
|
275
|
-
/** `record` scored against its case and kept in `scores` under its key, replacing an earlier record of the same review. */
|
|
276
|
-
function keepScore(scores, record, suiteCase,
|
|
277
|
-
const scored = reviewScore(record, suiteCase, window);
|
|
299
|
+
/** `record` scored against its case and its decoys, and kept in `scores` under its key, replacing an earlier record of the same review. */
|
|
300
|
+
function keepScore(scores, record, suiteCase, { suite, decoys }) {
|
|
301
|
+
const scored = reviewScore(record, suiteCase, suite.window, decoys.get(suiteCase.id));
|
|
278
302
|
scores.set(reviewKey(record.fingerprint, record.case, record.trial), scored);
|
|
279
303
|
return scored;
|
|
280
304
|
}
|
|
281
305
|
/**
|
|
282
|
-
* The score of every review the run being resumed holds, by `reviewKey`, and
|
|
283
|
-
* (
|
|
284
|
-
*
|
|
285
|
-
* A new run holds nothing.
|
|
306
|
+
* The score of every review the run being resumed holds, by `reviewKey`, and what the read found of the
|
|
307
|
+
* file (where its valid lines end and how long it was, which `openRecords` checks). A line that names a
|
|
308
|
+
* configuration the run does not have, or a case the suite does not have, refuses the run directory. A
|
|
309
|
+
* last line a killed run cut short is not scored, and says so. A new run holds nothing.
|
|
286
310
|
*/
|
|
287
|
-
async function readScores(io,
|
|
311
|
+
async function readScores(io, checked) {
|
|
312
|
+
const { byId, out, state, runFile } = checked;
|
|
288
313
|
const scores = new Map();
|
|
289
|
-
if (state.kind !== 'run')
|
|
290
|
-
return {
|
|
314
|
+
if (state.kind !== 'run') {
|
|
315
|
+
return {
|
|
316
|
+
scores,
|
|
317
|
+
records: { validBytes: 0, bytes: 0 },
|
|
318
|
+
};
|
|
319
|
+
}
|
|
291
320
|
const known = new Set(runFile.configurations.map(({ fingerprint }) => fingerprint));
|
|
292
321
|
const read = await readRecords(out, (record) => {
|
|
293
322
|
const suiteCase = byId.get(record.case);
|
|
@@ -295,25 +324,45 @@ async function readScores(io, { suite, byId, out, state, runFile }) {
|
|
|
295
324
|
return 'names a configuration this run does not have';
|
|
296
325
|
if (suiteCase === undefined)
|
|
297
326
|
return 'names a case the suite does not have';
|
|
298
|
-
keepScore(scores, record, suiteCase,
|
|
327
|
+
keepScore(scores, record, suiteCase, checked);
|
|
299
328
|
return undefined;
|
|
300
329
|
});
|
|
301
330
|
if (read.torn)
|
|
302
331
|
writeLines(io.stderr, [TORN_LINE]);
|
|
303
|
-
return {
|
|
332
|
+
return {
|
|
333
|
+
scores,
|
|
334
|
+
records: { validBytes: read.validBytes, bytes: read.bytes },
|
|
335
|
+
};
|
|
304
336
|
}
|
|
305
337
|
/**
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
* rejects, with the error of a failed write, that error is the one thrown.
|
|
338
|
+
* Write `run.json` as the run needs it: a new run is made, with its directory; a run being resumed whose
|
|
339
|
+
* recorded scoring hash is not the suite's now is written again with that scoring hash alone changed, so
|
|
340
|
+
* its first scoring hash, its configurations' paths and order, its salt and its version stay as recorded;
|
|
341
|
+
* any other run is left as it is.
|
|
311
342
|
*/
|
|
312
|
-
async function
|
|
313
|
-
|
|
314
|
-
if (state.kind !== 'run')
|
|
343
|
+
async function writeRun({ out, state, runFile }) {
|
|
344
|
+
if (state.kind !== 'run') {
|
|
315
345
|
await createRun(out, runFile);
|
|
316
|
-
|
|
346
|
+
}
|
|
347
|
+
else if (state.run.suite.scoringHash !== runFile.suite.scoringHash) {
|
|
348
|
+
await writeRunFile(out, {
|
|
349
|
+
...state.run,
|
|
350
|
+
suite: { ...state.run.suite, scoringHash: runFile.suite.scoringHash },
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Write `run.json` (see `writeRun`), open `results.jsonl` and run the reviews of `toRun`, each recorded as
|
|
356
|
+
* it resolves and each reported with its progress line. The order matters: `writeRun` renames over any
|
|
357
|
+
* run.json and `openRecords` cuts results.jsonl to the bytes the read found valid (and refuses it when
|
|
358
|
+
* its length is not the one the read found), so a run being resumed has been read before either. The
|
|
359
|
+
* writer is closed whatever happens; when the run rejects, with the error of a failed write, that error
|
|
360
|
+
* is the one thrown.
|
|
361
|
+
*/
|
|
362
|
+
async function execute(io, { checked, scores, records, toRun, jobs, maxRunTokens }) {
|
|
363
|
+
const { suite, loaded, byId, out, runFile } = checked;
|
|
364
|
+
await writeRun(checked);
|
|
365
|
+
const writer = await openRecords(out, records);
|
|
317
366
|
let finished = 0;
|
|
318
367
|
let outcome;
|
|
319
368
|
try {
|
|
@@ -321,7 +370,8 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
321
370
|
suite,
|
|
322
371
|
configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
|
|
323
372
|
reviews: toRun,
|
|
324
|
-
jobs
|
|
373
|
+
jobs,
|
|
374
|
+
maxRunTokens,
|
|
325
375
|
env: io.env,
|
|
326
376
|
signal: io.signal,
|
|
327
377
|
append: writer.append,
|
|
@@ -329,8 +379,11 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
329
379
|
finished += 1;
|
|
330
380
|
const { trial, caseId, configuration } = event.review;
|
|
331
381
|
const suiteCase = byId.get(caseId);
|
|
382
|
+
// The plan is made from the selected cases, which `byId` holds, so a review never names another.
|
|
383
|
+
if (suiteCase === undefined)
|
|
384
|
+
throw new Error(UNPLANNED_CASE);
|
|
332
385
|
const text = event.type === 'recorded'
|
|
333
|
-
? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase,
|
|
386
|
+
? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, checked))
|
|
334
387
|
: `failed: ${errorMessage(event.error)}`;
|
|
335
388
|
writeLines(io.stderr, [`[${finished}/${toRun.length}] trial ${trial}, case ${caseId}, configuration ${configuration + 1}: ${text}`]);
|
|
336
389
|
},
|
|
@@ -346,12 +399,13 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
|
|
|
346
399
|
/** Build the report from the scores, write `report.md` and `report.json` into the run directory, and print the Markdown. */
|
|
347
400
|
async function writeReports(io, { suite, selected, out, runFile }, trials, scores) {
|
|
348
401
|
const report = buildReport({
|
|
349
|
-
suite: {
|
|
402
|
+
suite: { ...runFile.suite, window: suite.window },
|
|
350
403
|
quorum: packageJson.version,
|
|
351
404
|
cases: selected,
|
|
352
405
|
trials,
|
|
353
406
|
configurations: runFile.configurations,
|
|
354
407
|
scores,
|
|
408
|
+
flagged: flaggedCases(selected, suite),
|
|
355
409
|
});
|
|
356
410
|
const markdown = reportMarkdown(report);
|
|
357
411
|
await writeAtomic(out, REPORT_MARKDOWN, markdown);
|
|
@@ -362,7 +416,7 @@ async function writeReports(io, { suite, selected, out, runFile }, trials, score
|
|
|
362
416
|
async function bench(io, options) {
|
|
363
417
|
const checked = await check(io, options);
|
|
364
418
|
const { suite, selected, loaded, directory, runFile } = checked;
|
|
365
|
-
const { scores,
|
|
419
|
+
const { scores, records } = await readScores(io, checked);
|
|
366
420
|
const trials = options.trials ?? suite.trials ?? 1;
|
|
367
421
|
const planned = planReviews({
|
|
368
422
|
trials,
|
|
@@ -370,10 +424,12 @@ async function bench(io, options) {
|
|
|
370
424
|
configurations: runFile.configurations.length,
|
|
371
425
|
});
|
|
372
426
|
const scoreOf = ({ configuration, caseId, trial }) => scores.get(reviewKey(runFile.configurations[configuration].fingerprint, caseId, trial));
|
|
373
|
-
const toRun =
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
427
|
+
const toRun = options.reportOnly
|
|
428
|
+
? []
|
|
429
|
+
: planned.filter((review) => {
|
|
430
|
+
const recorded = scoreOf(review);
|
|
431
|
+
return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
|
|
432
|
+
});
|
|
377
433
|
writeLines(io.stderr, loaded.flatMap(({ warnings }, index) => warnings.map(warning => `warning: configuration ${index + 1}: ${warning}`)));
|
|
378
434
|
writeLines(options.dryRun ? io.stdout : io.stderr, planLines({
|
|
379
435
|
suite,
|
|
@@ -386,6 +442,7 @@ async function bench(io, options) {
|
|
|
386
442
|
again: toRun.filter(review => scoreOf(review) !== undefined).length,
|
|
387
443
|
jobs: options.jobs,
|
|
388
444
|
directory,
|
|
445
|
+
rescored: runFile.suite.scoringHash !== runFile.suite.firstScoringHash,
|
|
389
446
|
}));
|
|
390
447
|
if (options.dryRun) {
|
|
391
448
|
writeLines(io.stdout, ['Dry run: no review was run, and nothing was written.']);
|
|
@@ -395,12 +452,28 @@ async function bench(io, options) {
|
|
|
395
452
|
io.stderr.write('interrupted\n');
|
|
396
453
|
return BENCH_EXIT.interrupted;
|
|
397
454
|
}
|
|
398
|
-
|
|
455
|
+
if (options.reportOnly) {
|
|
456
|
+
await writeRun(checked);
|
|
457
|
+
await writeReports(io, checked, trials, scores);
|
|
458
|
+
return BENCH_EXIT.ok;
|
|
459
|
+
}
|
|
460
|
+
const outcome = await execute(io, {
|
|
461
|
+
checked,
|
|
462
|
+
scores,
|
|
463
|
+
records,
|
|
464
|
+
toRun,
|
|
465
|
+
jobs: options.jobs,
|
|
466
|
+
maxRunTokens: options.maxRunTokens,
|
|
467
|
+
});
|
|
399
468
|
await writeReports(io, checked, trials, scores);
|
|
400
469
|
if (outcome.interrupted) {
|
|
401
470
|
io.stderr.write('interrupted\n');
|
|
402
471
|
return BENCH_EXIT.interrupted;
|
|
403
472
|
}
|
|
473
|
+
if (outcome.notRun > 0 && options.maxRunTokens !== undefined) {
|
|
474
|
+
writeLines(io.stderr, [`--max-run-tokens: the run reached its cap of ${options.maxRunTokens} tokens (it spent ${outcome.spent}), so ${outcome.notRun} review(s) were not run; the next run runs them`]);
|
|
475
|
+
return BENCH_EXIT.failed;
|
|
476
|
+
}
|
|
404
477
|
return outcome.failed > 0
|
|
405
478
|
? BENCH_EXIT.failed
|
|
406
479
|
: BENCH_EXIT.ok;
|
|
@@ -416,33 +489,51 @@ function usageError(io, error) {
|
|
|
416
489
|
* It reviews every case of the suite the argument names with every `--config` (the usual resolution when
|
|
417
490
|
* there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
|
|
418
491
|
* in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
|
|
419
|
-
* at a time
|
|
492
|
+
* at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
|
|
493
|
+
* denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
|
|
420
494
|
* stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
|
|
421
495
|
* on stdout, which it also writes, with `report.json`, into the run directory.
|
|
422
496
|
*
|
|
423
497
|
* A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
|
|
424
498
|
* runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
|
|
425
|
-
* warning. A
|
|
426
|
-
*
|
|
427
|
-
*
|
|
499
|
+
* warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
|
|
500
|
+
* re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
|
|
501
|
+
* the new scoring hash beside the one the run began with, and the plan and the reports say so while the
|
|
502
|
+
* suite is not the one the run began with. A run directory of another suite, another quorum or other
|
|
503
|
+
* configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
|
|
504
|
+
* changed, and a suite or a configuration that does not validate or cannot be built are refused before
|
|
505
|
+
* the first call.
|
|
506
|
+
*
|
|
507
|
+
* `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
|
|
508
|
+
* the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
|
|
509
|
+
* `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
|
|
510
|
+
* holds no run, and `--retry-incomplete`, which runs reviews.
|
|
511
|
+
*
|
|
512
|
+
* `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
|
|
513
|
+
* from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
|
|
514
|
+
* finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
|
|
515
|
+
* the next run runs them. The check is made before each review starts, so the reviews in flight can pass
|
|
516
|
+
* the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
|
|
428
517
|
*
|
|
429
|
-
* Exit codes: 0 when every planned review is recorded,
|
|
430
|
-
* directory error, 3 when the run stops (a file cannot
|
|
431
|
-
*
|
|
432
|
-
*
|
|
433
|
-
*
|
|
518
|
+
* Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
|
|
519
|
+
* reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
|
|
520
|
+
* be written or read), 4 when a review failed (its gating profile's research call failed, or its record
|
|
521
|
+
* was too large to write: the others are recorded, and the next run runs the failed ones) or
|
|
522
|
+
* `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
|
|
523
|
+
* so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
|
|
524
|
+
* directory as it was, even one that does not exist yet.
|
|
434
525
|
*
|
|
435
526
|
* Every line it writes is inert (invariant I3).
|
|
436
527
|
*/
|
|
437
528
|
export async function runBench(io, argv) {
|
|
438
529
|
let options;
|
|
439
530
|
try {
|
|
440
|
-
const { values, positionals } = parseFlags(argv);
|
|
531
|
+
const { values, positionals } = parseFlags(argv, OPTIONS);
|
|
441
532
|
if (values.help) {
|
|
442
533
|
io.stdout.write(BENCH_USAGE);
|
|
443
534
|
return BENCH_EXIT.ok;
|
|
444
535
|
}
|
|
445
|
-
options = checkOptions(values, positionals);
|
|
536
|
+
options = checkOptions(values, positionals, io.cwd);
|
|
446
537
|
}
|
|
447
538
|
catch (error) {
|
|
448
539
|
return usageError(io, error);
|
package/dist/cli/bin.js
CHANGED
|
@@ -13,8 +13,8 @@ function fail(error) {
|
|
|
13
13
|
}
|
|
14
14
|
process.on('uncaughtException', fail);
|
|
15
15
|
process.on('unhandledRejection', fail);
|
|
16
|
-
// Exit 3 until the run settles. A
|
|
17
|
-
// empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
|
|
16
|
+
// Exit 3 until the run settles. A config module whose loading never finishes leaves nothing to wait
|
|
17
|
+
// for and empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
|
|
18
18
|
process.exitCode = EXIT.runtime;
|
|
19
19
|
let settled = false;
|
|
20
20
|
process.once('beforeExit', () => {
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review,
|
|
3
|
-
* failure is written. `main.ts` and `bench.ts` both import it, so neither
|
|
2
|
+
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
|
|
3
|
+
* failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
|
|
4
|
+
* imports the other.
|
|
4
5
|
*/
|
|
6
|
+
import type { ParseArgsOptionsConfig } from 'node:util';
|
|
5
7
|
import type { Env } from '../config/env.ts';
|
|
6
8
|
/** Exit codes (ADR-0008, amended by ADR-0012). */
|
|
7
9
|
export declare const EXIT: {
|
|
@@ -43,3 +45,8 @@ export declare function errorLines(error: unknown): string;
|
|
|
43
45
|
* prototype. It never throws, so neither do the lines built from it.
|
|
44
46
|
*/
|
|
45
47
|
export declare function errorMessage(error: unknown): string;
|
|
48
|
+
/** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
49
|
+
export declare function parseFlags<T extends ParseArgsOptionsConfig>(argv: readonly string[], options: T): {
|
|
50
|
+
values: T extends ParseArgsOptionsConfig ? { -readonly [LongOption in keyof T]?: { [LongOption_2 in keyof T]: T[LongOption_2]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_2]["multiple"] ? T_1 extends false ? T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption] | undefined; } & { [LongOption_1 in keyof T as T[LongOption_1]["default"] extends {} ? LongOption_1 : never]: { [LongOption_3 in keyof T]: T[LongOption_3]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_3]["multiple"] ? T_1 extends false ? T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption_1]; } extends infer P ? { [K in keyof P]: P[K]; } : never : {};
|
|
51
|
+
positionals: string[];
|
|
52
|
+
};
|
package/dist/cli/io.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review,
|
|
3
|
-
* failure is written. `main.ts` and `bench.ts` both import it, so neither
|
|
2
|
+
* What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
|
|
3
|
+
* failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
|
|
4
|
+
* imports the other.
|
|
4
5
|
*/
|
|
6
|
+
import { parseArgs } from 'node:util';
|
|
5
7
|
import { ConfigError } from "../core/errors.js";
|
|
6
8
|
import { inertLine } from "../core/sanitize.js";
|
|
7
9
|
/** Exit codes (ADR-0008, amended by ADR-0012). */
|
|
@@ -43,3 +45,19 @@ export function errorMessage(error) {
|
|
|
43
45
|
return 'an error with no text form';
|
|
44
46
|
}
|
|
45
47
|
}
|
|
48
|
+
/** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
|
|
49
|
+
export function parseFlags(argv, options) {
|
|
50
|
+
try {
|
|
51
|
+
return parseArgs({
|
|
52
|
+
args: [...argv],
|
|
53
|
+
options,
|
|
54
|
+
strict: true,
|
|
55
|
+
allowPositionals: true,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
|
|
60
|
+
throw error;
|
|
61
|
+
throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/cli/main.d.ts
CHANGED
|
@@ -5,10 +5,12 @@ export { errorMessage, EXIT };
|
|
|
5
5
|
export type { CliIO };
|
|
6
6
|
/**
|
|
7
7
|
* Run one review and return the process exit code. Never throws: usage and configuration problems
|
|
8
|
-
* exit 2, runtime failures exit 3, an interrupt exits
|
|
9
|
-
* failed votes left undecided, or a
|
|
10
|
-
* was kept by a gating profile, so CI can tell a broken run
|
|
11
|
-
* (exit 1) and from a clean one (exit 0). `
|
|
8
|
+
* exit 2, a gating profile's failed research and other runtime failures exit 3, an interrupt exits
|
|
9
|
+
* 130, and an incomplete review (a finding its failed votes left undecided, or a gap) of a gating
|
|
10
|
+
* profile exits 4 unless a `high` finding was kept by a gating profile, so CI can tell a broken run
|
|
11
|
+
* and a partial review from a real finding (exit 1) and from a clean one (exit 0). `--allow-incomplete`
|
|
12
|
+
* turns that exit 4 into 0 only when every failed vote behind it failed in passing (see `exitCodeFor`).
|
|
13
|
+
* `exitCodeFor` maps the result; advisory flags never change it.
|
|
12
14
|
*
|
|
13
15
|
* It reads at most 16 MiB of diff, from stdin or a file, counted in bytes: a larger diff is a usage error,
|
|
14
16
|
* and reading stops at the chunk that crosses the cap.
|
|
@@ -40,22 +42,40 @@ export interface SignalSource {
|
|
|
40
42
|
}
|
|
41
43
|
/**
|
|
42
44
|
* The signal of one review, aborted by the first SIGINT or SIGTERM. The next one calls `exit(130)`: the
|
|
43
|
-
* way out
|
|
45
|
+
* way out when what a provider still runs, after the review settled its call, keeps the process alive.
|
|
44
46
|
*/
|
|
45
47
|
export declare function interruptSignal(source: SignalSource, exit: (code: number) => void): AbortSignal;
|
|
46
48
|
/**
|
|
47
49
|
* The exit code of a finished review: 1 when a kept finding is `high` and its profile gates, 4 when
|
|
48
|
-
* an unreviewed finding or a gap belongs to a gating profile
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
* as
|
|
50
|
+
* an unreviewed finding or a gap belongs to a gating profile, 0 otherwise. `allowIncomplete` turns
|
|
51
|
+
* that 4 into 0 only when every gap of a gating profile is a `vote-failed` one (an unreviewed finding
|
|
52
|
+
* is no gap) and, for each gating profile that has a gap, every failed vote of its own kept and
|
|
53
|
+
* unreviewed findings failed in passing, as `failedInPassing` says: a dropped connection, a timeout,
|
|
54
|
+
* or an HTTP 408, 409, 429 or 5xx. A gating profile with no gap blocks no other profile's lift. A gap
|
|
55
|
+
* with no failed vote behind it lifts nothing: an unreviewed finding whose votes hold none, or a
|
|
56
|
+
* `vote-failed` gap beside kept findings that hold none, which no review writes.
|
|
57
|
+
* A profile gates unless `meta.profiles` holds it with `gating: false`, so a profile missing from the
|
|
58
|
+
* result counts as gating and a gap never hides.
|
|
52
59
|
*
|
|
53
|
-
*
|
|
54
|
-
* request
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
60
|
+
* Why so narrow: a pull request causes the other gaps directly (ADR-0012). It pads the diff until a
|
|
61
|
+
* request is refused as too large, floods the research with decoy findings past `maxFindings`, or
|
|
62
|
+
* plants text that a voter refuses or that cuts its reply off or leaves it unusable, and one
|
|
63
|
+
* `.gitattributes` line, one NUL byte or a submodule's bump leaves a file's change out of the diff.
|
|
64
|
+
* The flag lifts only what the code names, so a reason added later is out of its reach by default,
|
|
65
|
+
* and so is what a result built by hand or by an older quorum leaves unsaid: a failed vote with no
|
|
66
|
+
* `failure`, a verdict whose `votes` is no array, a reason no quorum writes. A gating profile's
|
|
67
|
+
* `research-failed` gap, which no review records, is out too: a review that read nothing is never
|
|
68
|
+
* turned into exit 0. Planted text can still draw out a reply that ends in a lifted failure, a
|
|
69
|
+
* timeout of its own, so on a gating profile do not use the flag for a pull request you do not trust.
|
|
58
70
|
*/
|
|
59
71
|
export declare function exitCodeFor(result: QuorumResult, options?: {
|
|
60
72
|
readonly allowIncomplete?: boolean;
|
|
61
73
|
}): number;
|
|
74
|
+
/**
|
|
75
|
+
* True for a vote's failure that is infrastructure's: its own data `kind` is `network` or `timeout`, or
|
|
76
|
+
* `http` with an own data `status` of 408, 409, 429 or an integer from 500 to 599. Anything else is false:
|
|
77
|
+
* a refusal, a reply cut off or unusable, a request too large, any other status, a status that is no
|
|
78
|
+
* number, a kind no provider has, `null` and a value that is no object. It reads own data only, never
|
|
79
|
+
* runs a getter and never throws.
|
|
80
|
+
*/
|
|
81
|
+
export declare function failedInPassing(failure: unknown): boolean;
|