@tobi-or-not/quorum 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +7 -4
  5. package/dist/bench/fingerprint.js +22 -8
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +241 -13
  9. package/dist/bench/records.js +64 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +47 -17
  15. package/dist/bench/runner.js +82 -21
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +27 -9
  23. package/dist/cli/bench.js +174 -83
  24. package/dist/cli/bin.js +2 -2
  25. package/dist/cli/io.d.ts +9 -2
  26. package/dist/cli/io.js +20 -2
  27. package/dist/cli/main.d.ts +34 -14
  28. package/dist/cli/main.js +100 -38
  29. package/dist/cli/print-config.d.ts +8 -6
  30. package/dist/cli/print-config.js +21 -8
  31. package/dist/cli/render.d.ts +2 -2
  32. package/dist/cli/render.js +21 -2
  33. package/dist/config/env.d.ts +32 -19
  34. package/dist/config/env.js +95 -32
  35. package/dist/config/from-config.js +15 -3
  36. package/dist/config/load.d.ts +3 -0
  37. package/dist/config/load.js +17 -8
  38. package/dist/config/resolve.d.ts +9 -1
  39. package/dist/config/resolve.js +17 -1
  40. package/dist/config/schema.d.ts +141 -5
  41. package/dist/config/schema.js +125 -12
  42. package/dist/core/clock.d.ts +17 -0
  43. package/dist/core/clock.js +14 -0
  44. package/dist/core/coverage.d.ts +49 -0
  45. package/dist/core/coverage.js +144 -0
  46. package/dist/core/diff.d.ts +34 -0
  47. package/dist/core/diff.js +140 -18
  48. package/dist/core/errors.d.ts +18 -3
  49. package/dist/core/errors.js +26 -2
  50. package/dist/core/exclude.d.ts +42 -0
  51. package/dist/core/exclude.js +189 -0
  52. package/dist/core/limits.d.ts +82 -0
  53. package/dist/core/limits.js +163 -0
  54. package/dist/core/output.d.ts +3 -2
  55. package/dist/core/output.js +3 -2
  56. package/dist/core/ports.d.ts +57 -2
  57. package/dist/core/ports.js +36 -1
  58. package/dist/core/quorum.d.ts +24 -8
  59. package/dist/core/quorum.js +187 -56
  60. package/dist/core/retry.d.ts +36 -0
  61. package/dist/core/retry.js +46 -0
  62. package/dist/core/stages/context.d.ts +126 -22
  63. package/dist/core/stages/context.js +319 -41
  64. package/dist/core/stages/panel.d.ts +5 -4
  65. package/dist/core/stages/panel.js +30 -11
  66. package/dist/core/stages/research.d.ts +15 -6
  67. package/dist/core/stages/research.js +75 -18
  68. package/dist/core/tokens.d.ts +31 -0
  69. package/dist/core/tokens.js +81 -0
  70. package/dist/core/types.d.ts +77 -10
  71. package/dist/index.d.ts +1 -1
  72. package/dist/providers/anthropic.d.ts +2 -1
  73. package/dist/providers/anthropic.js +14 -6
  74. package/dist/providers/claude-code.d.ts +51 -11
  75. package/dist/providers/claude-code.js +69 -33
  76. package/dist/providers/gemini.d.ts +4 -1
  77. package/dist/providers/gemini.js +72 -15
  78. package/dist/providers/http.d.ts +49 -4
  79. package/dist/providers/http.js +155 -14
  80. package/dist/providers/ollama.d.ts +3 -1
  81. package/dist/providers/ollama.js +17 -9
  82. package/dist/providers/openai-compatible.d.ts +3 -1
  83. package/dist/providers/openai-compatible.js +17 -9
  84. package/dist/providers/options.d.ts +12 -3
  85. package/dist/providers/options.js +30 -10
  86. package/dist/providers/reply.d.ts +6 -2
  87. package/dist/providers/reply.js +13 -3
  88. package/dist/providers/size.d.ts +10 -14
  89. package/dist/providers/size.js +15 -69
  90. package/package.json +1 -1
  91. package/schema/quorum.schema.json +34 -1
package/dist/cli/bench.js CHANGED
@@ -4,20 +4,21 @@
4
4
  * caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
5
5
  */
6
6
  import { resolve } from 'node:path';
7
- import { parseArgs } from 'node:util';
8
7
  import packageJson from '../../package.json' with { type: 'json' };
8
+ import { flaggedCases } from "../bench/attacks.js";
9
9
  import { fingerprint, newSalt } from "../bench/fingerprint.js";
10
10
  import { MAX_CONFIGURATIONS, reviewKey, RUN_FORMAT } from "../bench/records.js";
11
- import { buildReport, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
12
- import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic } from "../bench/run-dir.js";
11
+ import { buildReport, kindCounts, kindsText, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
12
+ import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic, writeRunFile } from "../bench/run-dir.js";
13
13
  import { planReviews, runReviews } from "../bench/runner.js";
14
+ import { decoysOf } from "../bench/score.js";
14
15
  import { loadSuite, MAX_TRIALS, SUITE_ID } from "../bench/suite.js";
15
16
  import { readEnv } from "../config/env.js";
16
17
  import { fromConfig } from "../config/from-config.js";
17
18
  import { loadConfig } from "../config/load.js";
18
19
  import { ConfigError } from "../core/errors.js";
19
20
  import { inertLine } from "../core/sanitize.js";
20
- import { errorLines, errorMessage } from "./io.js";
21
+ import { errorLines, errorMessage, parseFlags } from "./io.js";
21
22
  import { profileProviders } from "./render.js";
22
23
  export const BENCH_USAGE = [
23
24
  'Usage: quorum bench [options] <suite>',
@@ -25,20 +26,23 @@ export const BENCH_USAGE = [
25
26
  'Review every case of a suite with every config, --trials times each, and report what each caught.',
26
27
  '',
27
28
  'Options:',
28
- ' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
29
- ' -p, --profile <id> Run only this profile; repeat it to run several',
30
- ' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
31
- ' --case <id> Run only this case; repeat it to run several',
32
- ' -j, --jobs <n> The reviews run at once (default 1)',
33
- ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
34
- ' --retry-incomplete Run again the recorded reviews that came back incomplete',
35
- ' --dry-run Check the suite and the configs, print the plan, and exit',
36
- ' -h, --help Print this help and exit',
29
+ ' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
30
+ ' -p, --profile <id> Run only this profile; repeat it to run several',
31
+ ' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
32
+ ' --case <id> Run only this case; repeat it to run several',
33
+ ' -j, --jobs <n> The reviews run at once (default 1)',
34
+ ' --max-run-tokens <n> Start no review once the run has spent <n> tokens; the next run runs the rest',
35
+ ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
36
+ ' --retry-incomplete Run again the recorded reviews that came back incomplete',
37
+ ' --report-only Write the reports of what is recorded, and run no review',
38
+ ' --dry-run Check the suite and the configs, print the plan, and exit',
39
+ ' -h, --help Print this help and exit',
37
40
  '',
38
41
  ].join('\n');
39
42
  /**
40
43
  * The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
41
- * not recorded, not that a review left a gap, and `stopped` means the run could not go on.
44
+ * not recorded, or that `--max-run-tokens` left reviews unrun, not that a review left a gap or that a
45
+ * model's reply denied it, and `stopped` means the run could not go on.
42
46
  */
43
47
  const BENCH_EXIT = {
44
48
  ok: 0,
@@ -49,34 +53,22 @@ const BENCH_EXIT = {
49
53
  };
50
54
  const MAX_JOBS = 16;
51
55
  const DEFAULT_CONFIGURATION = '(default configuration)';
56
+ const UNPLANNED_CASE = 'a review names a case the plan does not hold';
52
57
  const TORN_LINE = 'warning: results.jsonl: its last line was cut short, as a stopped run leaves it: that review runs again';
58
+ const RESCORED_LINE = 'Scoring: suite.yaml changed since the run began; its recorded reviews are scored again under it, with no call';
53
59
  const OPTIONS = {
54
60
  config: { type: 'string', short: 'c', multiple: true },
55
61
  profile: { type: 'string', short: 'p', multiple: true },
56
62
  trials: { type: 'string' },
57
63
  case: { type: 'string', multiple: true },
58
64
  jobs: { type: 'string', short: 'j' },
65
+ 'max-run-tokens': { type: 'string' },
59
66
  out: { type: 'string', short: 'o' },
60
67
  'retry-incomplete': { type: 'boolean' },
68
+ 'report-only': { type: 'boolean' },
61
69
  'dry-run': { type: 'boolean' },
62
70
  help: { type: 'boolean', short: 'h' },
63
71
  };
64
- /** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
65
- function parseFlags(argv) {
66
- try {
67
- return parseArgs({
68
- args: [...argv],
69
- options: OPTIONS,
70
- strict: true,
71
- allowPositionals: true,
72
- });
73
- }
74
- catch (error) {
75
- if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
76
- throw error;
77
- throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
78
- }
79
- }
80
72
  function usageIssue(path, message) {
81
73
  return new ConfigError([
82
74
  { path, message },
@@ -90,8 +82,16 @@ function wholeNumber(flag, text, max) {
90
82
  throw usageIssue(flag, `must be a whole number from 1 to ${max}`);
91
83
  return Number(text);
92
84
  }
85
+ /**
86
+ * The first of `configs`, as given, whose file a later one names again. A relative `--config` resolves
87
+ * against `cwd`, as `loadConfig` resolves it, so `a.yaml`, `./a.yaml` and its absolute path are one file.
88
+ */
89
+ function repeatedConfig(configs, cwd) {
90
+ const paths = configs.map(config => resolve(cwd, config));
91
+ return configs.find((_, index) => paths.includes(paths[index], index + 1));
92
+ }
93
93
  /** The options the flags of `argv` give, or the usage error they hold. */
94
- function checkOptions(values, positionals) {
94
+ function checkOptions(values, positionals, cwd) {
95
95
  const [suite, ...extra] = positionals;
96
96
  if (suite === undefined)
97
97
  throw new Error('no suite given');
@@ -99,11 +99,15 @@ function checkOptions(values, positionals) {
99
99
  throw new Error(`unexpected arguments: ${extra.join(' ')}`);
100
100
  const trials = wholeNumber('--trials', values.trials, MAX_TRIALS);
101
101
  const jobs = wholeNumber('--jobs', values.jobs, MAX_JOBS);
102
+ const maxRunTokens = wholeNumber('--max-run-tokens', values['max-run-tokens'], Number.MAX_SAFE_INTEGER);
102
103
  const configs = values.config ?? [];
103
104
  if (configs.includes(''))
104
105
  throw usageIssue('--config', 'must name a file');
105
106
  if (configs.length > MAX_CONFIGURATIONS)
106
107
  throw usageIssue('--config', `names more than ${MAX_CONFIGURATIONS} configurations`);
108
+ const repeated = repeatedConfig(configs, cwd);
109
+ if (repeated !== undefined)
110
+ throw usageIssue('--config', `names the file "${repeated}" more than once`);
107
111
  if (values.out === '')
108
112
  throw usageIssue('--out', 'must name a directory');
109
113
  const cases = values.case ?? [];
@@ -111,6 +115,8 @@ function checkOptions(values, positionals) {
111
115
  throw usageIssue('--case', `must be a case id matching ${SUITE_ID.source}`);
112
116
  if (new Set(cases).size < cases.length)
113
117
  throw usageIssue('--case', 'names a case more than once');
118
+ if (values['report-only'] === true && values['retry-incomplete'] === true)
119
+ throw usageIssue('--report-only', 'runs no review, so it cannot be given with --retry-incomplete');
114
120
  return {
115
121
  suite,
116
122
  configs,
@@ -118,8 +124,10 @@ function checkOptions(values, positionals) {
118
124
  trials,
119
125
  cases,
120
126
  jobs: jobs ?? 1,
127
+ maxRunTokens,
121
128
  out: values.out,
122
129
  retryIncomplete: values['retry-incomplete'] === true,
130
+ reportOnly: values['report-only'] === true,
123
131
  dryRun: values['dry-run'] === true,
124
132
  };
125
133
  }
@@ -131,11 +139,6 @@ function writeLines(stream, lines) {
131
139
  function nameOf({ path }) {
132
140
  return path ?? DEFAULT_CONFIGURATION;
133
141
  }
134
- function plural(count, noun) {
135
- return `${count} ${noun}${count === 1
136
- ? ''
137
- : 's'}`;
138
- }
139
142
  /** The cases of `suite` the flags select, in the suite's order; every `--case` must be one of its cases. */
140
143
  function selectCases(suite, ids) {
141
144
  const unknown = ids.filter(id => !suite.cases.some(suiteCase => suiteCase.id === id));
@@ -201,17 +204,25 @@ function verdictOf(suiteCase, score) {
201
204
  ? 'caught by a proposed finding only'
202
205
  : 'missed';
203
206
  }
207
+ /** What a denied review's line says: words the bench wrote, never the model's reply. */
208
+ const DENIAL_WORDS = {
209
+ 'unusable-reply': 'the research reply could not be used',
210
+ refusal: 'the model refused the research request',
211
+ truncated: 'the research reply was cut off at its token limit',
212
+ };
204
213
  /** The progress line of a review that was recorded, after `[<n>/<total>] trial <t>, case <id>, configuration <i>: `. */
205
214
  function recordedText(record, suiteCase, score) {
215
+ const latency = `${(record.latencyMs / 1000).toFixed(1)} s`;
216
+ if ('denied' in record)
217
+ return `denied, ${DENIAL_WORDS[record.denied]}; ${latency}`;
206
218
  const { kept, discarded, unreviewed, meta } = record.result;
207
- return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${(record.latencyMs / 1000).toFixed(1)} s`;
219
+ return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${latency}`;
208
220
  }
209
- /** The plan a run prints before its first call. */
221
+ /** The plan a run prints before its first call; `rescored` adds the line that says its recorded reviews are scored again. */
210
222
  function planLines(plan) {
211
- const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory } = plan;
212
- const bugs = selected.filter(({ kind }) => kind === 'bug').length;
223
+ const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory, rescored } = plan;
213
224
  return [
214
- `Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${plural(bugs, 'bug case')} and ${plural(selected.length - bugs, 'control')}`,
225
+ `Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${kindsText(kindCounts(selected))}`,
215
226
  'Configurations:',
216
227
  ...loaded.flatMap(({ path, config }, index) => [
217
228
  ` ${index + 1}. ${path ?? DEFAULT_CONFIGURATION}${loaded.length > 1 && index === 0
@@ -223,6 +234,9 @@ function planLines(plan) {
223
234
  `Reviews: ${planned} planned, ${recorded} recorded, ${toRun} to run${again > 0
224
235
  ? ` (${again} of them again)`
225
236
  : ''}, at most ${jobs} at a time`,
237
+ ...rescored
238
+ ? [RESCORED_LINE]
239
+ : [],
226
240
  `Run directory: ${directory}`,
227
241
  ];
228
242
  }
@@ -238,6 +252,8 @@ async function check(io, options) {
238
252
  const directory = options.out ?? `quorum-bench/${suite.name}`;
239
253
  const out = resolve(io.cwd, directory);
240
254
  const state = await inspectRunDir(out);
255
+ if (options.reportOnly && state.kind !== 'run')
256
+ throw usageIssue('--out', 'holds no run, so --report-only has nothing to report');
241
257
  const salt = state.kind === 'run'
242
258
  ? state.run.salt
243
259
  : newSalt();
@@ -251,7 +267,14 @@ async function check(io, options) {
251
267
  const runFile = {
252
268
  format: RUN_FORMAT,
253
269
  quorum: packageJson.version,
254
- suite: { name: suite.name, hash: suite.hash },
270
+ suite: {
271
+ name: suite.name,
272
+ reviewHash: suite.reviewHash,
273
+ scoringHash: suite.scoringHash,
274
+ firstScoringHash: state.kind === 'run'
275
+ ? state.run.suite.firstScoringHash
276
+ : suite.scoringHash,
277
+ },
255
278
  salt,
256
279
  configurations,
257
280
  };
@@ -266,28 +289,34 @@ async function check(io, options) {
266
289
  selected,
267
290
  loaded,
268
291
  byId,
292
+ decoys: decoysOf(suite.cases),
269
293
  directory,
270
294
  out,
271
295
  state,
272
296
  runFile,
273
297
  };
274
298
  }
275
- /** `record` scored against its case and kept in `scores` under its key, replacing an earlier record of the same review. */
276
- function keepScore(scores, record, suiteCase, window) {
277
- const scored = reviewScore(record, suiteCase, window);
299
+ /** `record` scored against its case and its decoys, and kept in `scores` under its key, replacing an earlier record of the same review. */
300
+ function keepScore(scores, record, suiteCase, { suite, decoys }) {
301
+ const scored = reviewScore(record, suiteCase, suite.window, decoys.get(suiteCase.id));
278
302
  scores.set(reviewKey(record.fingerprint, record.case, record.trial), scored);
279
303
  return scored;
280
304
  }
281
305
  /**
282
- * The score of every review the run being resumed holds, by `reviewKey`, and where its valid lines end
283
- * (what `openRecords` cuts to). A line that names a configuration the run does not have, or a case the suite
284
- * does not have, refuses the run directory. A last line a killed run cut short is not scored, and says so.
285
- * A new run holds nothing.
306
+ * The score of every review the run being resumed holds, by `reviewKey`, and what the read found of the
307
+ * file (where its valid lines end and how long it was, which `openRecords` checks). A line that names a
308
+ * configuration the run does not have, or a case the suite does not have, refuses the run directory. A
309
+ * last line a killed run cut short is not scored, and says so. A new run holds nothing.
286
310
  */
287
- async function readScores(io, { suite, byId, out, state, runFile }) {
311
+ async function readScores(io, checked) {
312
+ const { byId, out, state, runFile } = checked;
288
313
  const scores = new Map();
289
- if (state.kind !== 'run')
290
- return { scores, validBytes: 0 };
314
+ if (state.kind !== 'run') {
315
+ return {
316
+ scores,
317
+ records: { validBytes: 0, bytes: 0 },
318
+ };
319
+ }
291
320
  const known = new Set(runFile.configurations.map(({ fingerprint }) => fingerprint));
292
321
  const read = await readRecords(out, (record) => {
293
322
  const suiteCase = byId.get(record.case);
@@ -295,25 +324,45 @@ async function readScores(io, { suite, byId, out, state, runFile }) {
295
324
  return 'names a configuration this run does not have';
296
325
  if (suiteCase === undefined)
297
326
  return 'names a case the suite does not have';
298
- keepScore(scores, record, suiteCase, suite.window);
327
+ keepScore(scores, record, suiteCase, checked);
299
328
  return undefined;
300
329
  });
301
330
  if (read.torn)
302
331
  writeLines(io.stderr, [TORN_LINE]);
303
- return { scores, validBytes: read.validBytes };
332
+ return {
333
+ scores,
334
+ records: { validBytes: read.validBytes, bytes: read.bytes },
335
+ };
304
336
  }
305
337
  /**
306
- * Make the run (when it is new), open `results.jsonl` and run the reviews of `toRun`, each recorded as it
307
- * resolves and each reported with its progress line. The order matters: `createRun` renames over any
308
- * run.json and `openRecords` cuts results.jsonl to the bytes it is given, so a run being resumed has been
309
- * read before either, and only a new run is created. The writer is closed whatever happens; when the run
310
- * rejects, with the error of a failed write, that error is the one thrown.
338
+ * Write `run.json` as the run needs it: a new run is made, with its directory; a run being resumed whose
339
+ * recorded scoring hash is not the suite's now is written again with that scoring hash alone changed, so
340
+ * its first scoring hash, its configurations' paths and order, its salt and its version stay as recorded;
341
+ * any other run is left as it is.
311
342
  */
312
- async function execute(io, options, checked, scores, validBytes, toRun) {
313
- const { suite, loaded, byId, out, state, runFile } = checked;
314
- if (state.kind !== 'run')
343
+ async function writeRun({ out, state, runFile }) {
344
+ if (state.kind !== 'run') {
315
345
  await createRun(out, runFile);
316
- const writer = await openRecords(out, validBytes);
346
+ }
347
+ else if (state.run.suite.scoringHash !== runFile.suite.scoringHash) {
348
+ await writeRunFile(out, {
349
+ ...state.run,
350
+ suite: { ...state.run.suite, scoringHash: runFile.suite.scoringHash },
351
+ });
352
+ }
353
+ }
354
+ /**
355
+ * Write `run.json` (see `writeRun`), open `results.jsonl` and run the reviews of `toRun`, each recorded as
356
+ * it resolves and each reported with its progress line. The order matters: `writeRun` renames over any
357
+ * run.json and `openRecords` cuts results.jsonl to the bytes the read found valid (and refuses it when
358
+ * its length is not the one the read found), so a run being resumed has been read before either. The
359
+ * writer is closed whatever happens; when the run rejects, with the error of a failed write, that error
360
+ * is the one thrown.
361
+ */
362
+ async function execute(io, { checked, scores, records, toRun, jobs, maxRunTokens }) {
363
+ const { suite, loaded, byId, out, runFile } = checked;
364
+ await writeRun(checked);
365
+ const writer = await openRecords(out, records);
317
366
  let finished = 0;
318
367
  let outcome;
319
368
  try {
@@ -321,7 +370,8 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
321
370
  suite,
322
371
  configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
323
372
  reviews: toRun,
324
- jobs: options.jobs,
373
+ jobs,
374
+ maxRunTokens,
325
375
  env: io.env,
326
376
  signal: io.signal,
327
377
  append: writer.append,
@@ -329,8 +379,11 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
329
379
  finished += 1;
330
380
  const { trial, caseId, configuration } = event.review;
331
381
  const suiteCase = byId.get(caseId);
382
+ // The plan is made from the selected cases, which `byId` holds, so a review never names another.
383
+ if (suiteCase === undefined)
384
+ throw new Error(UNPLANNED_CASE);
332
385
  const text = event.type === 'recorded'
333
- ? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, suite.window))
386
+ ? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, checked))
334
387
  : `failed: ${errorMessage(event.error)}`;
335
388
  writeLines(io.stderr, [`[${finished}/${toRun.length}] trial ${trial}, case ${caseId}, configuration ${configuration + 1}: ${text}`]);
336
389
  },
@@ -346,12 +399,13 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
346
399
  /** Build the report from the scores, write `report.md` and `report.json` into the run directory, and print the Markdown. */
347
400
  async function writeReports(io, { suite, selected, out, runFile }, trials, scores) {
348
401
  const report = buildReport({
349
- suite: { name: suite.name, window: suite.window },
402
+ suite: { ...runFile.suite, window: suite.window },
350
403
  quorum: packageJson.version,
351
404
  cases: selected,
352
405
  trials,
353
406
  configurations: runFile.configurations,
354
407
  scores,
408
+ flagged: flaggedCases(selected, suite),
355
409
  });
356
410
  const markdown = reportMarkdown(report);
357
411
  await writeAtomic(out, REPORT_MARKDOWN, markdown);
@@ -362,7 +416,7 @@ async function writeReports(io, { suite, selected, out, runFile }, trials, score
362
416
  async function bench(io, options) {
363
417
  const checked = await check(io, options);
364
418
  const { suite, selected, loaded, directory, runFile } = checked;
365
- const { scores, validBytes } = await readScores(io, checked);
419
+ const { scores, records } = await readScores(io, checked);
366
420
  const trials = options.trials ?? suite.trials ?? 1;
367
421
  const planned = planReviews({
368
422
  trials,
@@ -370,10 +424,12 @@ async function bench(io, options) {
370
424
  configurations: runFile.configurations.length,
371
425
  });
372
426
  const scoreOf = ({ configuration, caseId, trial }) => scores.get(reviewKey(runFile.configurations[configuration].fingerprint, caseId, trial));
373
- const toRun = planned.filter((review) => {
374
- const recorded = scoreOf(review);
375
- return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
376
- });
427
+ const toRun = options.reportOnly
428
+ ? []
429
+ : planned.filter((review) => {
430
+ const recorded = scoreOf(review);
431
+ return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
432
+ });
377
433
  writeLines(io.stderr, loaded.flatMap(({ warnings }, index) => warnings.map(warning => `warning: configuration ${index + 1}: ${warning}`)));
378
434
  writeLines(options.dryRun ? io.stdout : io.stderr, planLines({
379
435
  suite,
@@ -386,6 +442,7 @@ async function bench(io, options) {
386
442
  again: toRun.filter(review => scoreOf(review) !== undefined).length,
387
443
  jobs: options.jobs,
388
444
  directory,
445
+ rescored: runFile.suite.scoringHash !== runFile.suite.firstScoringHash,
389
446
  }));
390
447
  if (options.dryRun) {
391
448
  writeLines(io.stdout, ['Dry run: no review was run, and nothing was written.']);
@@ -395,12 +452,28 @@ async function bench(io, options) {
395
452
  io.stderr.write('interrupted\n');
396
453
  return BENCH_EXIT.interrupted;
397
454
  }
398
- const outcome = await execute(io, options, checked, scores, validBytes, toRun);
455
+ if (options.reportOnly) {
456
+ await writeRun(checked);
457
+ await writeReports(io, checked, trials, scores);
458
+ return BENCH_EXIT.ok;
459
+ }
460
+ const outcome = await execute(io, {
461
+ checked,
462
+ scores,
463
+ records,
464
+ toRun,
465
+ jobs: options.jobs,
466
+ maxRunTokens: options.maxRunTokens,
467
+ });
399
468
  await writeReports(io, checked, trials, scores);
400
469
  if (outcome.interrupted) {
401
470
  io.stderr.write('interrupted\n');
402
471
  return BENCH_EXIT.interrupted;
403
472
  }
473
+ if (outcome.notRun > 0 && options.maxRunTokens !== undefined) {
474
+ writeLines(io.stderr, [`--max-run-tokens: the run reached its cap of ${options.maxRunTokens} tokens (it spent ${outcome.spent}), so ${outcome.notRun} review(s) were not run; the next run runs them`]);
475
+ return BENCH_EXIT.failed;
476
+ }
404
477
  return outcome.failed > 0
405
478
  ? BENCH_EXIT.failed
406
479
  : BENCH_EXIT.ok;
@@ -416,33 +489,51 @@ function usageError(io, error) {
416
489
  * It reviews every case of the suite the argument names with every `--config` (the usual resolution when
417
490
  * there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
418
491
  * in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
419
- * at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
492
+ * at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
493
+ * denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
420
494
  * stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
421
495
  * on stdout, which it also writes, with `report.json`, into the run directory.
422
496
  *
423
497
  * A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
424
498
  * runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
425
- * warning. A run directory of another suite, another quorum or other configurations, one that holds files
426
- * of another kind, and a suite or a configuration that does not validate or cannot be built are refused
427
- * before the first call.
499
+ * warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
500
+ * re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
501
+ * the new scoring hash beside the one the run began with, and the plan and the reports say so while the
502
+ * suite is not the one the run began with. A run directory of another suite, another quorum or other
503
+ * configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
504
+ * changed, and a suite or a configuration that does not validate or cannot be built are refused before
505
+ * the first call.
506
+ *
507
+ * `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
508
+ * the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
509
+ * `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
510
+ * holds no run, and `--retry-incomplete`, which runs reviews.
511
+ *
512
+ * `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
513
+ * from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
514
+ * finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
515
+ * the next run runs them. The check is made before each review starts, so the reviews in flight can pass
516
+ * the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
428
517
  *
429
- * Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
430
- * directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
431
- * others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
432
- * written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
433
- * first review starts leaves the run directory untouched, even one that does not exist yet.
518
+ * Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
519
+ * reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
520
+ * be written or read), 4 when a review failed (its gating profile's research call failed, or its record
521
+ * was too large to write: the others are recorded, and the next run runs the failed ones) or
522
+ * `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
523
+ * so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
524
+ * directory as it was, even one that does not exist yet.
434
525
  *
435
526
  * Every line it writes is inert (invariant I3).
436
527
  */
437
528
  export async function runBench(io, argv) {
438
529
  let options;
439
530
  try {
440
- const { values, positionals } = parseFlags(argv);
531
+ const { values, positionals } = parseFlags(argv, OPTIONS);
441
532
  if (values.help) {
442
533
  io.stdout.write(BENCH_USAGE);
443
534
  return BENCH_EXIT.ok;
444
535
  }
445
- options = checkOptions(values, positionals);
536
+ options = checkOptions(values, positionals, io.cwd);
446
537
  }
447
538
  catch (error) {
448
539
  return usageError(io, error);
package/dist/cli/bin.js CHANGED
@@ -13,8 +13,8 @@ function fail(error) {
13
13
  }
14
14
  process.on('uncaughtException', fail);
15
15
  process.on('unhandledRejection', fail);
16
- // Exit 3 until the run settles. A provider whose promise never settles, and holds no timer or socket,
17
- // empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
16
+ // Exit 3 until the run settles. A config module whose loading never finishes leaves nothing to wait
17
+ // for and empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
18
18
  process.exitCode = EXIT.runtime;
19
19
  let settled = false;
20
20
  process.once('beforeExit', () => {
package/dist/cli/io.d.ts CHANGED
@@ -1,7 +1,9 @@
1
1
  /**
2
- * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
- * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
3
+ * failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
4
+ * imports the other.
4
5
  */
6
+ import type { ParseArgsOptionsConfig } from 'node:util';
5
7
  import type { Env } from '../config/env.ts';
6
8
  /** Exit codes (ADR-0008, amended by ADR-0012). */
7
9
  export declare const EXIT: {
@@ -43,3 +45,8 @@ export declare function errorLines(error: unknown): string;
43
45
  * prototype. It never throws, so neither do the lines built from it.
44
46
  */
45
47
  export declare function errorMessage(error: unknown): string;
48
+ /** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
49
+ export declare function parseFlags<T extends ParseArgsOptionsConfig>(argv: readonly string[], options: T): {
50
+ values: T extends ParseArgsOptionsConfig ? { -readonly [LongOption in keyof T]?: { [LongOption_2 in keyof T]: T[LongOption_2]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_2]["multiple"] ? T_1 extends false ? T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption] | undefined; } & { [LongOption_1 in keyof T as T[LongOption_1]["default"] extends {} ? LongOption_1 : never]: { [LongOption_3 in keyof T]: T[LongOption_3]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_3]["multiple"] ? T_1 extends false ? T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption_1]; } extends infer P ? { [K in keyof P]: P[K]; } : never : {};
51
+ positionals: string[];
52
+ };
package/dist/cli/io.js CHANGED
@@ -1,7 +1,9 @@
1
1
  /**
2
- * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
- * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
3
+ * failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
4
+ * imports the other.
4
5
  */
6
+ import { parseArgs } from 'node:util';
5
7
  import { ConfigError } from "../core/errors.js";
6
8
  import { inertLine } from "../core/sanitize.js";
7
9
  /** Exit codes (ADR-0008, amended by ADR-0012). */
@@ -43,3 +45,19 @@ export function errorMessage(error) {
43
45
  return 'an error with no text form';
44
46
  }
45
47
  }
48
+ /** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
49
+ export function parseFlags(argv, options) {
50
+ try {
51
+ return parseArgs({
52
+ args: [...argv],
53
+ options,
54
+ strict: true,
55
+ allowPositionals: true,
56
+ });
57
+ }
58
+ catch (error) {
59
+ if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
60
+ throw error;
61
+ throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
62
+ }
63
+ }
@@ -5,10 +5,12 @@ export { errorMessage, EXIT };
5
5
  export type { CliIO };
6
6
  /**
7
7
  * Run one review and return the process exit code. Never throws: usage and configuration problems
8
- * exit 2, runtime failures exit 3, an interrupt exits 130, and an incomplete review (a finding its
9
- * failed votes left undecided, or a research gap) of a gating profile exits 4 unless a `high` finding
10
- * was kept by a gating profile, so CI can tell a broken run and a partial review from a real finding
11
- * (exit 1) and from a clean one (exit 0). `exitCodeFor` maps the result; advisory flags never change it.
8
+ * exit 2, a gating profile's failed research and other runtime failures exit 3, an interrupt exits
9
+ * 130, and an incomplete review (a finding its failed votes left undecided, or a gap) of a gating
10
+ * profile exits 4 unless a `high` finding was kept by a gating profile, so CI can tell a broken run
11
+ * and a partial review from a real finding (exit 1) and from a clean one (exit 0). `--allow-incomplete`
12
+ * turns that exit 4 into 0 only when every failed vote behind it failed in passing (see `exitCodeFor`).
13
+ * `exitCodeFor` maps the result; advisory flags never change it.
12
14
  *
13
15
  * It reads at most 16 MiB of diff, from stdin or a file, counted in bytes: a larger diff is a usage error,
14
16
  * and reading stops at the chunk that crosses the cap.
@@ -40,22 +42,40 @@ export interface SignalSource {
40
42
  }
41
43
  /**
42
44
  * The signal of one review, aborted by the first SIGINT or SIGTERM. The next one calls `exit(130)`: the
43
- * way out of a provider that ignores its signal, since no timer waits for it.
45
+ * way out when what a provider still runs, after the review settled its call, keeps the process alive.
44
46
  */
45
47
  export declare function interruptSignal(source: SignalSource, exit: (code: number) => void): AbortSignal;
46
48
  /**
47
49
  * The exit code of a finished review: 1 when a kept finding is `high` and its profile gates, 4 when
48
- * an unreviewed finding or a gap belongs to a gating profile (unless `allowIncomplete` turns it into
49
- * 0, which it does for every gap but a `request-too-large` one), 0 otherwise. A profile gates
50
- * unless `meta.profiles` holds it with `gating: false`, so a profile missing from the result counts
51
- * as gating and a gap never hides.
50
+ * an unreviewed finding or a gap belongs to a gating profile, 0 otherwise. `allowIncomplete` turns
51
+ * that 4 into 0 only when every gap of a gating profile is a `vote-failed` one (an unreviewed finding
52
+ * is no gap) and, for each gating profile that has a gap, every failed vote of its own kept and
53
+ * unreviewed findings failed in passing, as `failedInPassing` says: a dropped connection, a timeout,
54
+ * or an HTTP 408, 409, 429 or 5xx. A gating profile with no gap blocks no other profile's lift. A gap
55
+ * with no failed vote behind it lifts nothing: an unreviewed finding whose votes hold none, or a
56
+ * `vote-failed` gap beside kept findings that hold none, which no review writes.
57
+ * A profile gates unless `meta.profiles` holds it with `gating: false`, so a profile missing from the
58
+ * result counts as gating and a gap never hides.
52
59
  *
53
- * A `request-too-large` gap of a gating profile exits 4 whatever `allowIncomplete` says: a pull
54
- * request can cause that gap by padding the diff until a request is refused, so no flag lifts it
55
- * (ADR-0012). The flag lifts every other gap, including gaps a pull request can cause: decoy findings
56
- * that push a real one past `maxFindings`, or text that trips a hosted voter's content filter. On a
57
- * gating profile, do not use it for a pull request you do not trust.
60
+ * Why so narrow: a pull request causes the other gaps directly (ADR-0012). It pads the diff until a
61
+ * request is refused as too large, floods the research with decoy findings past `maxFindings`, or
62
+ * plants text that a voter refuses or that cuts its reply off or leaves it unusable, and one
63
+ * `.gitattributes` line, one NUL byte or a submodule's bump leaves a file's change out of the diff.
64
+ * The flag lifts only what the code names, so a reason added later is out of its reach by default,
65
+ * and so is what a result built by hand or by an older quorum leaves unsaid: a failed vote with no
66
+ * `failure`, a verdict whose `votes` is no array, a reason no quorum writes. A gating profile's
67
+ * `research-failed` gap, which no review records, is out too: a review that read nothing is never
68
+ * turned into exit 0. Planted text can still draw out a reply that ends in a lifted failure, a
69
+ * timeout of its own, so on a gating profile do not use the flag for a pull request you do not trust.
58
70
  */
59
71
  export declare function exitCodeFor(result: QuorumResult, options?: {
60
72
  readonly allowIncomplete?: boolean;
61
73
  }): number;
74
+ /**
75
+ * True for a vote's failure that is infrastructure's: its own data `kind` is `network` or `timeout`, or
76
+ * `http` with an own data `status` of 408, 409, 429 or an integer from 500 to 599. Anything else is false:
77
+ * a refusal, a reply cut off or unusable, a request too large, any other status, a status that is no
78
+ * number, a kind no provider has, `null` and a value that is no object. It reads own data only, never
79
+ * runs a getter and never throws.
80
+ */
81
+ export declare function failedInPassing(failure: unknown): boolean;