@tobi-or-not/quorum 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +3 -2
  5. package/dist/bench/fingerprint.js +3 -2
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +171 -13
  9. package/dist/bench/records.js +34 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +29 -17
  15. package/dist/bench/runner.js +53 -18
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +20 -9
  23. package/dist/cli/bench.js +148 -74
  24. package/dist/cli/io.d.ts +9 -2
  25. package/dist/cli/io.js +20 -2
  26. package/dist/cli/main.js +2 -19
  27. package/dist/config/load.d.ts +3 -0
  28. package/dist/config/load.js +17 -8
  29. package/dist/core/diff.d.ts +10 -0
  30. package/dist/core/diff.js +77 -0
  31. package/dist/core/errors.d.ts +6 -0
  32. package/dist/core/errors.js +5 -0
  33. package/dist/core/stages/context.d.ts +12 -3
  34. package/dist/core/stages/context.js +23 -3
  35. package/dist/providers/anthropic.js +6 -3
  36. package/dist/providers/claude-code.js +11 -10
  37. package/dist/providers/gemini.js +9 -4
  38. package/dist/providers/ollama.js +6 -4
  39. package/dist/providers/openai-compatible.js +6 -4
  40. package/dist/providers/reply.d.ts +6 -2
  41. package/dist/providers/reply.js +13 -3
  42. package/package.json +1 -1
package/dist/cli/bench.js CHANGED
@@ -4,20 +4,21 @@
4
4
  * caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
5
5
  */
6
6
  import { resolve } from 'node:path';
7
- import { parseArgs } from 'node:util';
8
7
  import packageJson from '../../package.json' with { type: 'json' };
8
+ import { flaggedCases } from "../bench/attacks.js";
9
9
  import { fingerprint, newSalt } from "../bench/fingerprint.js";
10
10
  import { MAX_CONFIGURATIONS, reviewKey, RUN_FORMAT } from "../bench/records.js";
11
- import { buildReport, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
12
- import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic } from "../bench/run-dir.js";
11
+ import { buildReport, kindCounts, kindsText, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
12
+ import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic, writeRunFile } from "../bench/run-dir.js";
13
13
  import { planReviews, runReviews } from "../bench/runner.js";
14
+ import { decoysOf } from "../bench/score.js";
14
15
  import { loadSuite, MAX_TRIALS, SUITE_ID } from "../bench/suite.js";
15
16
  import { readEnv } from "../config/env.js";
16
17
  import { fromConfig } from "../config/from-config.js";
17
18
  import { loadConfig } from "../config/load.js";
18
19
  import { ConfigError } from "../core/errors.js";
19
20
  import { inertLine } from "../core/sanitize.js";
20
- import { errorLines, errorMessage } from "./io.js";
21
+ import { errorLines, errorMessage, parseFlags } from "./io.js";
21
22
  import { profileProviders } from "./render.js";
22
23
  export const BENCH_USAGE = [
23
24
  'Usage: quorum bench [options] <suite>',
@@ -32,13 +33,15 @@ export const BENCH_USAGE = [
32
33
  ' -j, --jobs <n> The reviews run at once (default 1)',
33
34
  ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
34
35
  ' --retry-incomplete Run again the recorded reviews that came back incomplete',
36
+ ' --report-only Write the reports of what is recorded, and run no review',
35
37
  ' --dry-run Check the suite and the configs, print the plan, and exit',
36
38
  ' -h, --help Print this help and exit',
37
39
  '',
38
40
  ].join('\n');
39
41
  /**
40
42
  * The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
41
- * not recorded, not that a review left a gap, and `stopped` means the run could not go on.
43
+ * not recorded, not that a review left a gap or that a model's reply denied it, and `stopped` means the
44
+ * run could not go on.
42
45
  */
43
46
  const BENCH_EXIT = {
44
47
  ok: 0,
@@ -49,7 +52,9 @@ const BENCH_EXIT = {
49
52
  };
50
53
  const MAX_JOBS = 16;
51
54
  const DEFAULT_CONFIGURATION = '(default configuration)';
55
+ const UNPLANNED_CASE = 'a review names a case the plan does not hold';
52
56
  const TORN_LINE = 'warning: results.jsonl: its last line was cut short, as a stopped run leaves it: that review runs again';
57
+ const RESCORED_LINE = 'Scoring: suite.yaml changed since the run began; its recorded reviews are scored again under it, with no call';
53
58
  const OPTIONS = {
54
59
  config: { type: 'string', short: 'c', multiple: true },
55
60
  profile: { type: 'string', short: 'p', multiple: true },
@@ -58,25 +63,10 @@ const OPTIONS = {
58
63
  jobs: { type: 'string', short: 'j' },
59
64
  out: { type: 'string', short: 'o' },
60
65
  'retry-incomplete': { type: 'boolean' },
66
+ 'report-only': { type: 'boolean' },
61
67
  'dry-run': { type: 'boolean' },
62
68
  help: { type: 'boolean', short: 'h' },
63
69
  };
64
- /** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
65
- function parseFlags(argv) {
66
- try {
67
- return parseArgs({
68
- args: [...argv],
69
- options: OPTIONS,
70
- strict: true,
71
- allowPositionals: true,
72
- });
73
- }
74
- catch (error) {
75
- if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
76
- throw error;
77
- throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
78
- }
79
- }
80
70
  function usageIssue(path, message) {
81
71
  return new ConfigError([
82
72
  { path, message },
@@ -90,8 +80,16 @@ function wholeNumber(flag, text, max) {
90
80
  throw usageIssue(flag, `must be a whole number from 1 to ${max}`);
91
81
  return Number(text);
92
82
  }
83
+ /**
84
+ * The first of `configs`, as given, whose file a later one names again. A relative `--config` resolves
85
+ * against `cwd`, as `loadConfig` resolves it, so `a.yaml`, `./a.yaml` and its absolute path are one file.
86
+ */
87
+ function repeatedConfig(configs, cwd) {
88
+ const paths = configs.map(config => resolve(cwd, config));
89
+ return configs.find((_, index) => paths.includes(paths[index], index + 1));
90
+ }
93
91
  /** The options the flags of `argv` give, or the usage error they hold. */
94
- function checkOptions(values, positionals) {
92
+ function checkOptions(values, positionals, cwd) {
95
93
  const [suite, ...extra] = positionals;
96
94
  if (suite === undefined)
97
95
  throw new Error('no suite given');
@@ -104,6 +102,9 @@ function checkOptions(values, positionals) {
104
102
  throw usageIssue('--config', 'must name a file');
105
103
  if (configs.length > MAX_CONFIGURATIONS)
106
104
  throw usageIssue('--config', `names more than ${MAX_CONFIGURATIONS} configurations`);
105
+ const repeated = repeatedConfig(configs, cwd);
106
+ if (repeated !== undefined)
107
+ throw usageIssue('--config', `names the file "${repeated}" more than once`);
107
108
  if (values.out === '')
108
109
  throw usageIssue('--out', 'must name a directory');
109
110
  const cases = values.case ?? [];
@@ -111,6 +112,8 @@ function checkOptions(values, positionals) {
111
112
  throw usageIssue('--case', `must be a case id matching ${SUITE_ID.source}`);
112
113
  if (new Set(cases).size < cases.length)
113
114
  throw usageIssue('--case', 'names a case more than once');
115
+ if (values['report-only'] === true && values['retry-incomplete'] === true)
116
+ throw usageIssue('--report-only', 'runs no review, so it cannot be given with --retry-incomplete');
114
117
  return {
115
118
  suite,
116
119
  configs,
@@ -120,6 +123,7 @@ function checkOptions(values, positionals) {
120
123
  jobs: jobs ?? 1,
121
124
  out: values.out,
122
125
  retryIncomplete: values['retry-incomplete'] === true,
126
+ reportOnly: values['report-only'] === true,
123
127
  dryRun: values['dry-run'] === true,
124
128
  };
125
129
  }
@@ -131,11 +135,6 @@ function writeLines(stream, lines) {
131
135
  function nameOf({ path }) {
132
136
  return path ?? DEFAULT_CONFIGURATION;
133
137
  }
134
- function plural(count, noun) {
135
- return `${count} ${noun}${count === 1
136
- ? ''
137
- : 's'}`;
138
- }
139
138
  /** The cases of `suite` the flags select, in the suite's order; every `--case` must be one of its cases. */
140
139
  function selectCases(suite, ids) {
141
140
  const unknown = ids.filter(id => !suite.cases.some(suiteCase => suiteCase.id === id));
@@ -201,17 +200,25 @@ function verdictOf(suiteCase, score) {
201
200
  ? 'caught by a proposed finding only'
202
201
  : 'missed';
203
202
  }
203
+ /** What a denied review's line says: words the bench wrote, never the model's reply. */
204
+ const DENIAL_WORDS = {
205
+ 'unusable-reply': 'the research reply could not be used',
206
+ refusal: 'the model refused the research request',
207
+ truncated: 'the research reply was cut off at its token limit',
208
+ };
204
209
  /** The progress line of a review that was recorded, after `[<n>/<total>] trial <t>, case <id>, configuration <i>: `. */
205
210
  function recordedText(record, suiteCase, score) {
211
+ const latency = `${(record.latencyMs / 1000).toFixed(1)} s`;
212
+ if ('denied' in record)
213
+ return `denied, ${DENIAL_WORDS[record.denied]}; ${latency}`;
206
214
  const { kept, discarded, unreviewed, meta } = record.result;
207
- return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${(record.latencyMs / 1000).toFixed(1)} s`;
215
+ return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${latency}`;
208
216
  }
209
- /** The plan a run prints before its first call. */
217
+ /** The plan a run prints before its first call; `rescored` adds the line that says its recorded reviews are scored again. */
210
218
  function planLines(plan) {
211
- const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory } = plan;
212
- const bugs = selected.filter(({ kind }) => kind === 'bug').length;
219
+ const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory, rescored } = plan;
213
220
  return [
214
- `Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${plural(bugs, 'bug case')} and ${plural(selected.length - bugs, 'control')}`,
221
+ `Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${kindsText(kindCounts(selected))}`,
215
222
  'Configurations:',
216
223
  ...loaded.flatMap(({ path, config }, index) => [
217
224
  ` ${index + 1}. ${path ?? DEFAULT_CONFIGURATION}${loaded.length > 1 && index === 0
@@ -223,6 +230,9 @@ function planLines(plan) {
223
230
  `Reviews: ${planned} planned, ${recorded} recorded, ${toRun} to run${again > 0
224
231
  ? ` (${again} of them again)`
225
232
  : ''}, at most ${jobs} at a time`,
233
+ ...rescored
234
+ ? [RESCORED_LINE]
235
+ : [],
226
236
  `Run directory: ${directory}`,
227
237
  ];
228
238
  }
@@ -238,6 +248,8 @@ async function check(io, options) {
238
248
  const directory = options.out ?? `quorum-bench/${suite.name}`;
239
249
  const out = resolve(io.cwd, directory);
240
250
  const state = await inspectRunDir(out);
251
+ if (options.reportOnly && state.kind !== 'run')
252
+ throw usageIssue('--out', 'holds no run, so --report-only has nothing to report');
241
253
  const salt = state.kind === 'run'
242
254
  ? state.run.salt
243
255
  : newSalt();
@@ -251,7 +263,14 @@ async function check(io, options) {
251
263
  const runFile = {
252
264
  format: RUN_FORMAT,
253
265
  quorum: packageJson.version,
254
- suite: { name: suite.name, hash: suite.hash },
266
+ suite: {
267
+ name: suite.name,
268
+ reviewHash: suite.reviewHash,
269
+ scoringHash: suite.scoringHash,
270
+ firstScoringHash: state.kind === 'run'
271
+ ? state.run.suite.firstScoringHash
272
+ : suite.scoringHash,
273
+ },
255
274
  salt,
256
275
  configurations,
257
276
  };
@@ -266,28 +285,34 @@ async function check(io, options) {
266
285
  selected,
267
286
  loaded,
268
287
  byId,
288
+ decoys: decoysOf(suite.cases),
269
289
  directory,
270
290
  out,
271
291
  state,
272
292
  runFile,
273
293
  };
274
294
  }
275
- /** `record` scored against its case and kept in `scores` under its key, replacing an earlier record of the same review. */
276
- function keepScore(scores, record, suiteCase, window) {
277
- const scored = reviewScore(record, suiteCase, window);
295
+ /** `record` scored against its case and its decoys, and kept in `scores` under its key, replacing an earlier record of the same review. */
296
+ function keepScore(scores, record, suiteCase, { suite, decoys }) {
297
+ const scored = reviewScore(record, suiteCase, suite.window, decoys.get(suiteCase.id));
278
298
  scores.set(reviewKey(record.fingerprint, record.case, record.trial), scored);
279
299
  return scored;
280
300
  }
281
301
  /**
282
- * The score of every review the run being resumed holds, by `reviewKey`, and where its valid lines end
283
- * (what `openRecords` cuts to). A line that names a configuration the run does not have, or a case the suite
284
- * does not have, refuses the run directory. A last line a killed run cut short is not scored, and says so.
285
- * A new run holds nothing.
302
+ * The score of every review the run being resumed holds, by `reviewKey`, and what the read found of the
303
+ * file (where its valid lines end and how long it was, which `openRecords` checks). A line that names a
304
+ * configuration the run does not have, or a case the suite does not have, refuses the run directory. A
305
+ * last line a killed run cut short is not scored, and says so. A new run holds nothing.
286
306
  */
287
- async function readScores(io, { suite, byId, out, state, runFile }) {
307
+ async function readScores(io, checked) {
308
+ const { byId, out, state, runFile } = checked;
288
309
  const scores = new Map();
289
- if (state.kind !== 'run')
290
- return { scores, validBytes: 0 };
310
+ if (state.kind !== 'run') {
311
+ return {
312
+ scores,
313
+ records: { validBytes: 0, bytes: 0 },
314
+ };
315
+ }
291
316
  const known = new Set(runFile.configurations.map(({ fingerprint }) => fingerprint));
292
317
  const read = await readRecords(out, (record) => {
293
318
  const suiteCase = byId.get(record.case);
@@ -295,25 +320,45 @@ async function readScores(io, { suite, byId, out, state, runFile }) {
295
320
  return 'names a configuration this run does not have';
296
321
  if (suiteCase === undefined)
297
322
  return 'names a case the suite does not have';
298
- keepScore(scores, record, suiteCase, suite.window);
323
+ keepScore(scores, record, suiteCase, checked);
299
324
  return undefined;
300
325
  });
301
326
  if (read.torn)
302
327
  writeLines(io.stderr, [TORN_LINE]);
303
- return { scores, validBytes: read.validBytes };
328
+ return {
329
+ scores,
330
+ records: { validBytes: read.validBytes, bytes: read.bytes },
331
+ };
304
332
  }
305
333
  /**
306
- * Make the run (when it is new), open `results.jsonl` and run the reviews of `toRun`, each recorded as it
307
- * resolves and each reported with its progress line. The order matters: `createRun` renames over any
308
- * run.json and `openRecords` cuts results.jsonl to the bytes it is given, so a run being resumed has been
309
- * read before either, and only a new run is created. The writer is closed whatever happens; when the run
310
- * rejects, with the error of a failed write, that error is the one thrown.
334
+ * Write `run.json` as the run needs it: a new run is made, with its directory; a run being resumed whose
335
+ * recorded scoring hash is not the suite's now is written again with that scoring hash alone changed, so
336
+ * its first scoring hash, its configurations' paths and order, its salt and its version stay as recorded;
337
+ * any other run is left as it is.
311
338
  */
312
- async function execute(io, options, checked, scores, validBytes, toRun) {
313
- const { suite, loaded, byId, out, state, runFile } = checked;
314
- if (state.kind !== 'run')
339
+ async function writeRun({ out, state, runFile }) {
340
+ if (state.kind !== 'run') {
315
341
  await createRun(out, runFile);
316
- const writer = await openRecords(out, validBytes);
342
+ }
343
+ else if (state.run.suite.scoringHash !== runFile.suite.scoringHash) {
344
+ await writeRunFile(out, {
345
+ ...state.run,
346
+ suite: { ...state.run.suite, scoringHash: runFile.suite.scoringHash },
347
+ });
348
+ }
349
+ }
350
+ /**
351
+ * Write `run.json` (see `writeRun`), open `results.jsonl` and run the reviews of `toRun`, each recorded as
352
+ * it resolves and each reported with its progress line. The order matters: `writeRun` renames over any
353
+ * run.json and `openRecords` cuts results.jsonl to the bytes the read found valid (and refuses it when
354
+ * its length is not the one the read found), so a run being resumed has been read before either. The
355
+ * writer is closed whatever happens; when the run rejects, with the error of a failed write, that error
356
+ * is the one thrown.
357
+ */
358
+ async function execute(io, { checked, scores, records, toRun, jobs }) {
359
+ const { suite, loaded, byId, out, runFile } = checked;
360
+ await writeRun(checked);
361
+ const writer = await openRecords(out, records);
317
362
  let finished = 0;
318
363
  let outcome;
319
364
  try {
@@ -321,7 +366,7 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
321
366
  suite,
322
367
  configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
323
368
  reviews: toRun,
324
- jobs: options.jobs,
369
+ jobs,
325
370
  env: io.env,
326
371
  signal: io.signal,
327
372
  append: writer.append,
@@ -329,8 +374,11 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
329
374
  finished += 1;
330
375
  const { trial, caseId, configuration } = event.review;
331
376
  const suiteCase = byId.get(caseId);
377
+ // The plan is made from the selected cases, which `byId` holds, so a review never names another.
378
+ if (suiteCase === undefined)
379
+ throw new Error(UNPLANNED_CASE);
332
380
  const text = event.type === 'recorded'
333
- ? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, suite.window))
381
+ ? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, checked))
334
382
  : `failed: ${errorMessage(event.error)}`;
335
383
  writeLines(io.stderr, [`[${finished}/${toRun.length}] trial ${trial}, case ${caseId}, configuration ${configuration + 1}: ${text}`]);
336
384
  },
@@ -346,12 +394,13 @@ async function execute(io, options, checked, scores, validBytes, toRun) {
346
394
  /** Build the report from the scores, write `report.md` and `report.json` into the run directory, and print the Markdown. */
347
395
  async function writeReports(io, { suite, selected, out, runFile }, trials, scores) {
348
396
  const report = buildReport({
349
- suite: { name: suite.name, window: suite.window },
397
+ suite: { ...runFile.suite, window: suite.window },
350
398
  quorum: packageJson.version,
351
399
  cases: selected,
352
400
  trials,
353
401
  configurations: runFile.configurations,
354
402
  scores,
403
+ flagged: flaggedCases(selected, suite),
355
404
  });
356
405
  const markdown = reportMarkdown(report);
357
406
  await writeAtomic(out, REPORT_MARKDOWN, markdown);
@@ -362,7 +411,7 @@ async function writeReports(io, { suite, selected, out, runFile }, trials, score
362
411
  async function bench(io, options) {
363
412
  const checked = await check(io, options);
364
413
  const { suite, selected, loaded, directory, runFile } = checked;
365
- const { scores, validBytes } = await readScores(io, checked);
414
+ const { scores, records } = await readScores(io, checked);
366
415
  const trials = options.trials ?? suite.trials ?? 1;
367
416
  const planned = planReviews({
368
417
  trials,
@@ -370,10 +419,12 @@ async function bench(io, options) {
370
419
  configurations: runFile.configurations.length,
371
420
  });
372
421
  const scoreOf = ({ configuration, caseId, trial }) => scores.get(reviewKey(runFile.configurations[configuration].fingerprint, caseId, trial));
373
- const toRun = planned.filter((review) => {
374
- const recorded = scoreOf(review);
375
- return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
376
- });
422
+ const toRun = options.reportOnly
423
+ ? []
424
+ : planned.filter((review) => {
425
+ const recorded = scoreOf(review);
426
+ return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
427
+ });
377
428
  writeLines(io.stderr, loaded.flatMap(({ warnings }, index) => warnings.map(warning => `warning: configuration ${index + 1}: ${warning}`)));
378
429
  writeLines(options.dryRun ? io.stdout : io.stderr, planLines({
379
430
  suite,
@@ -386,6 +437,7 @@ async function bench(io, options) {
386
437
  again: toRun.filter(review => scoreOf(review) !== undefined).length,
387
438
  jobs: options.jobs,
388
439
  directory,
440
+ rescored: runFile.suite.scoringHash !== runFile.suite.firstScoringHash,
389
441
  }));
390
442
  if (options.dryRun) {
391
443
  writeLines(io.stdout, ['Dry run: no review was run, and nothing was written.']);
@@ -395,7 +447,18 @@ async function bench(io, options) {
395
447
  io.stderr.write('interrupted\n');
396
448
  return BENCH_EXIT.interrupted;
397
449
  }
398
- const outcome = await execute(io, options, checked, scores, validBytes, toRun);
450
+ if (options.reportOnly) {
451
+ await writeRun(checked);
452
+ await writeReports(io, checked, trials, scores);
453
+ return BENCH_EXIT.ok;
454
+ }
455
+ const outcome = await execute(io, {
456
+ checked,
457
+ scores,
458
+ records,
459
+ toRun,
460
+ jobs: options.jobs,
461
+ });
399
462
  await writeReports(io, checked, trials, scores);
400
463
  if (outcome.interrupted) {
401
464
  io.stderr.write('interrupted\n');
@@ -416,33 +479,44 @@ function usageError(io, error) {
416
479
  * It reviews every case of the suite the argument names with every `--config` (the usual resolution when
417
480
  * there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
418
481
  * in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
419
- * at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
482
+ * at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
483
+ * denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
420
484
  * stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
421
485
  * on stdout, which it also writes, with `report.json`, into the run directory.
422
486
  *
423
487
  * A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
424
488
  * runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
425
- * warning. A run directory of another suite, another quorum or other configurations, one that holds files
426
- * of another kind, and a suite or a configuration that does not validate or cannot be built are refused
427
- * before the first call.
489
+ * warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
490
+ * re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
491
+ * the new scoring hash beside the one the run began with, and the plan and the reports say so while the
492
+ * suite is not the one the run began with. A run directory of another suite, another quorum or other
493
+ * configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
494
+ * changed, and a suite or a configuration that does not validate or cannot be built are refused before
495
+ * the first call.
496
+ *
497
+ * `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
498
+ * the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
499
+ * `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
500
+ * holds no run, and `--retry-incomplete`, which runs reviews.
428
501
  *
429
- * Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
430
- * directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
431
- * others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
432
- * written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
433
- * first review starts leaves the run directory untouched, even one that does not exist yet.
502
+ * Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
503
+ * reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
504
+ * be written or read), 4 when a review failed (its research call failed, or its record was too large to
505
+ * write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
506
+ * reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
507
+ * checks and the plan leaves the run directory as it was, even one that does not exist yet.
434
508
  *
435
509
  * Every line it writes is inert (invariant I3).
436
510
  */
437
511
  export async function runBench(io, argv) {
438
512
  let options;
439
513
  try {
440
- const { values, positionals } = parseFlags(argv);
514
+ const { values, positionals } = parseFlags(argv, OPTIONS);
441
515
  if (values.help) {
442
516
  io.stdout.write(BENCH_USAGE);
443
517
  return BENCH_EXIT.ok;
444
518
  }
445
- options = checkOptions(values, positionals);
519
+ options = checkOptions(values, positionals, io.cwd);
446
520
  }
447
521
  catch (error) {
448
522
  return usageError(io, error);
package/dist/cli/io.d.ts CHANGED
@@ -1,7 +1,9 @@
1
1
  /**
2
- * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
- * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
3
+ * failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
4
+ * imports the other.
4
5
  */
6
+ import type { ParseArgsOptionsConfig } from 'node:util';
5
7
  import type { Env } from '../config/env.ts';
6
8
  /** Exit codes (ADR-0008, amended by ADR-0012). */
7
9
  export declare const EXIT: {
@@ -43,3 +45,8 @@ export declare function errorLines(error: unknown): string;
43
45
  * prototype. It never throws, so neither do the lines built from it.
44
46
  */
45
47
  export declare function errorMessage(error: unknown): string;
48
+ /** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
49
+ export declare function parseFlags<T extends ParseArgsOptionsConfig>(argv: readonly string[], options: T): {
50
+ values: T extends ParseArgsOptionsConfig ? { -readonly [LongOption in keyof T]?: { [LongOption_2 in keyof T]: T[LongOption_2]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_2]["multiple"] ? T_1 extends false ? T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_2]["type"] extends "string" ? string : T[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption] | undefined; } & { [LongOption_1 in keyof T as T[LongOption_1]["default"] extends {} ? LongOption_1 : never]: { [LongOption_3 in keyof T]: T[LongOption_3]["multiple"] extends infer T_1 ? T_1 extends T[LongOption_3]["multiple"] ? T_1 extends false ? T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : T_1 extends true ? (T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean)[] : T[LongOption_3]["type"] extends "string" ? string : T[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption_1]; } extends infer P ? { [K in keyof P]: P[K]; } : never : {};
51
+ positionals: string[];
52
+ };
package/dist/cli/io.js CHANGED
@@ -1,7 +1,9 @@
1
1
  /**
2
- * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
- * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, how a
3
+ * failure is written and the strict flag parser. `main.ts` and `bench.ts` both import it, so neither
4
+ * imports the other.
4
5
  */
6
+ import { parseArgs } from 'node:util';
5
7
  import { ConfigError } from "../core/errors.js";
6
8
  import { inertLine } from "../core/sanitize.js";
7
9
  /** Exit codes (ADR-0008, amended by ADR-0012). */
@@ -43,3 +45,19 @@ export function errorMessage(error) {
43
45
  return 'an error with no text form';
44
46
  }
45
47
  }
48
+ /** The flags of `argv` under `options`, and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
49
+ export function parseFlags(argv, options) {
50
+ try {
51
+ return parseArgs({
52
+ args: [...argv],
53
+ options,
54
+ strict: true,
55
+ allowPositionals: true,
56
+ });
57
+ }
58
+ catch (error) {
59
+ if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
60
+ throw error;
61
+ throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
62
+ }
63
+ }
package/dist/cli/main.js CHANGED
@@ -1,6 +1,5 @@
1
1
  import { createReadStream } from 'node:fs';
2
2
  import { resolve } from 'node:path';
3
- import { parseArgs } from 'node:util';
4
3
  import packageJson from '../../package.json' with { type: 'json' };
5
4
  import { readEnv } from "../config/env.js";
6
5
  import { fromConfig } from "../config/from-config.js";
@@ -9,7 +8,7 @@ import { decodeDiff } from "../core/diff.js";
9
8
  import { ConfigError } from "../core/errors.js";
10
9
  import { inertJson, inertLine } from "../core/sanitize.js";
11
10
  import { runBench } from "./bench.js";
12
- import { errorLines, errorMessage, EXIT } from "./io.js";
11
+ import { errorLines, errorMessage, EXIT, parseFlags } from "./io.js";
13
12
  import { printConfig } from "./print-config.js";
14
13
  import { bannerLines, progressSink } from "./render.js";
15
14
  export { errorMessage, EXIT };
@@ -70,7 +69,7 @@ export async function runCli(io) {
70
69
  return runBench(io, io.argv.slice(1));
71
70
  let flags;
72
71
  try {
73
- flags = parseFlags(io.argv);
72
+ flags = parseFlags(io.argv, OPTIONS);
74
73
  }
75
74
  catch (error) {
76
75
  return usageError(io, error);
@@ -188,22 +187,6 @@ export function exitCodeFor(result, options = {}) {
188
187
  ? EXIT.incomplete
189
188
  : EXIT.ok;
190
189
  }
191
- /** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
192
- function parseFlags(argv) {
193
- try {
194
- return parseArgs({
195
- args: [...argv],
196
- options: OPTIONS,
197
- strict: true,
198
- allowPositionals: true,
199
- });
200
- }
201
- catch (error) {
202
- if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
203
- throw error;
204
- throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
205
- }
206
- }
207
190
  /** Write a usage error and the usage, and return its exit code. */
208
191
  function usageError(io, error) {
209
192
  io.stderr.write(`${errorLines(error)}\n${USAGE}`);
@@ -1,3 +1,4 @@
1
+ import type { Stats } from 'node:fs';
1
2
  import type { z } from 'zod';
2
3
  import type { ConfigIssue } from '../core/errors.ts';
3
4
  import type { Env } from './env.ts';
@@ -18,6 +19,8 @@ export type Read<T> = {
18
19
  readonly problem: string;
19
20
  readonly counted?: number;
20
21
  };
22
+ /** The `lstat` of a path, or the problem a failed one means by its code: the path is missing, or it cannot be read. */
23
+ export declare function lstatOf(absolute: string): Promise<Read<Stats>>;
21
24
  /**
22
25
  * The bytes of a regular file of at most `limit` bytes. The file is opened without blocking, so a FIFO
23
26
  * is refused instead of waiting for a writer, and at most one byte past the limit is read, however large
@@ -54,20 +54,29 @@ function openProblem(error, noFollow) {
54
54
  ? 'is a symbolic link'
55
55
  : 'cannot be read';
56
56
  }
57
- /** The problem a path has before an open that must not follow a link: it is one, or it is missing or unreadable; undefined when it is not a link. */
58
- async function linkProblem(absolute) {
57
+ /** The `lstat` of a path, or the problem a failed one means by its code: the path is missing, or it cannot be read. */
58
+ export async function lstatOf(absolute) {
59
59
  try {
60
- return (await lstat(absolute)).isSymbolicLink()
61
- ? 'is a symbolic link'
62
- : undefined;
60
+ return { value: await lstat(absolute) };
63
61
  }
64
62
  catch (error) {
65
63
  const code = error.code;
66
- return code === 'ENOENT' || code === 'ENOTDIR'
67
- ? 'does not exist'
68
- : 'cannot be read';
64
+ return {
65
+ problem: code === 'ENOENT' || code === 'ENOTDIR'
66
+ ? 'does not exist'
67
+ : 'cannot be read',
68
+ };
69
69
  }
70
70
  }
71
+ /** The problem a path has before an open that must not follow a link: it is one, or it is missing or unreadable; undefined when it is not a link. */
72
+ async function linkProblem(absolute) {
73
+ const stats = await lstatOf(absolute);
74
+ if ('problem' in stats)
75
+ return stats.problem;
76
+ return stats.value.isSymbolicLink()
77
+ ? 'is a symbolic link'
78
+ : undefined;
79
+ }
71
80
  /**
72
81
  * The bytes of a regular file of at most `limit` bytes. The file is opened without blocking, so a FIFO
73
82
  * is refused instead of waiting for a writer, and at most one byte past the limit is read, however large
@@ -52,6 +52,16 @@ export interface FindingLocation {
52
52
  * hunk (`+c,0`) holds at `c`.
53
53
  */
54
54
  export declare function locate(sections: ReadonlyMap<string, string>, file: string, line: number): FindingLocation | undefined;
55
+ /**
56
+ * The first line from `start` to `end` that no hunk of `file` holds, or undefined when each sits in
57
+ * one: the answer `locate` gives for each line in turn, found by a binary search over the spans the
58
+ * hunks hold, merged and sorted once per section, so a range costs a search and not a walk over every
59
+ * hunk for each of its lines. The file is resolved as `locate` resolves it, and a file the diff does
60
+ * not hold has no line in a hunk, so the answer is `start`.
61
+ *
62
+ * @throws {RangeError} unless `start` and `end` are safe integers with `start` at most `end`
63
+ */
64
+ export declare function firstLineOutside(sections: ReadonlyMap<string, string>, file: string, start: number, end: number): number | undefined;
55
65
  /** A line the diff adds to the new file. */
56
66
  export interface AddedLine {
57
67
  /** The line's number in the new file. */