@tobi-or-not/quorum 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +7 -4
  5. package/dist/bench/fingerprint.js +22 -8
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +241 -13
  9. package/dist/bench/records.js +64 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +47 -17
  15. package/dist/bench/runner.js +82 -21
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +27 -9
  23. package/dist/cli/bench.js +174 -83
  24. package/dist/cli/bin.js +2 -2
  25. package/dist/cli/io.d.ts +9 -2
  26. package/dist/cli/io.js +20 -2
  27. package/dist/cli/main.d.ts +34 -14
  28. package/dist/cli/main.js +100 -38
  29. package/dist/cli/print-config.d.ts +8 -6
  30. package/dist/cli/print-config.js +21 -8
  31. package/dist/cli/render.d.ts +2 -2
  32. package/dist/cli/render.js +21 -2
  33. package/dist/config/env.d.ts +32 -19
  34. package/dist/config/env.js +95 -32
  35. package/dist/config/from-config.js +15 -3
  36. package/dist/config/load.d.ts +3 -0
  37. package/dist/config/load.js +17 -8
  38. package/dist/config/resolve.d.ts +9 -1
  39. package/dist/config/resolve.js +17 -1
  40. package/dist/config/schema.d.ts +141 -5
  41. package/dist/config/schema.js +125 -12
  42. package/dist/core/clock.d.ts +17 -0
  43. package/dist/core/clock.js +14 -0
  44. package/dist/core/coverage.d.ts +49 -0
  45. package/dist/core/coverage.js +144 -0
  46. package/dist/core/diff.d.ts +34 -0
  47. package/dist/core/diff.js +140 -18
  48. package/dist/core/errors.d.ts +18 -3
  49. package/dist/core/errors.js +26 -2
  50. package/dist/core/exclude.d.ts +42 -0
  51. package/dist/core/exclude.js +189 -0
  52. package/dist/core/limits.d.ts +82 -0
  53. package/dist/core/limits.js +163 -0
  54. package/dist/core/output.d.ts +3 -2
  55. package/dist/core/output.js +3 -2
  56. package/dist/core/ports.d.ts +57 -2
  57. package/dist/core/ports.js +36 -1
  58. package/dist/core/quorum.d.ts +24 -8
  59. package/dist/core/quorum.js +187 -56
  60. package/dist/core/retry.d.ts +36 -0
  61. package/dist/core/retry.js +46 -0
  62. package/dist/core/stages/context.d.ts +126 -22
  63. package/dist/core/stages/context.js +319 -41
  64. package/dist/core/stages/panel.d.ts +5 -4
  65. package/dist/core/stages/panel.js +30 -11
  66. package/dist/core/stages/research.d.ts +15 -6
  67. package/dist/core/stages/research.js +75 -18
  68. package/dist/core/tokens.d.ts +31 -0
  69. package/dist/core/tokens.js +81 -0
  70. package/dist/core/types.d.ts +77 -10
  71. package/dist/index.d.ts +1 -1
  72. package/dist/providers/anthropic.d.ts +2 -1
  73. package/dist/providers/anthropic.js +14 -6
  74. package/dist/providers/claude-code.d.ts +51 -11
  75. package/dist/providers/claude-code.js +69 -33
  76. package/dist/providers/gemini.d.ts +4 -1
  77. package/dist/providers/gemini.js +72 -15
  78. package/dist/providers/http.d.ts +49 -4
  79. package/dist/providers/http.js +155 -14
  80. package/dist/providers/ollama.d.ts +3 -1
  81. package/dist/providers/ollama.js +17 -9
  82. package/dist/providers/openai-compatible.d.ts +3 -1
  83. package/dist/providers/openai-compatible.js +17 -9
  84. package/dist/providers/options.d.ts +12 -3
  85. package/dist/providers/options.js +30 -10
  86. package/dist/providers/reply.d.ts +6 -2
  87. package/dist/providers/reply.js +13 -3
  88. package/dist/providers/size.d.ts +10 -14
  89. package/dist/providers/size.js +15 -69
  90. package/package.json +1 -1
  91. package/schema/quorum.schema.json +34 -1
@@ -9,15 +9,14 @@
9
9
  * Everything is checked before any call, and no message quotes the file or a patch.
10
10
  */
11
11
  import { createHash } from 'node:crypto';
12
- import { lstat } from 'node:fs/promises';
13
12
  import { join } from 'node:path';
14
13
  import { z } from 'zod';
15
- import { dataIssues, decode, isPlainObject, parseYaml, readBounded } from "../config/load.js";
14
+ import { dataIssues, decode, isPlainObject, lstatOf, parseYaml, readBounded } from "../config/load.js";
16
15
  import { configPath, parseConfig } from "../config/schema.js";
17
- import { decodeDiff, locate, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
16
+ import { decodeDiff, firstLineOutside, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
18
17
  import { ConfigError } from "../core/errors.js";
19
18
  import { hiddenLabels } from "../core/sanitize.js";
20
- /** A suite name or a case id: a name for a directory and for a record, so it holds no separator and no `..`. */
19
+ /** A suite name or a case id: a name for a directory and for a record, so it is never `.` or `..`, and holds no separator. */
21
20
  export const SUITE_ID = /^[a-z0-9][a-z0-9._-]*$/;
22
21
  /** A patch's file name: bare, and not starting with a dot. */
23
22
  export const PATCH_NAME = /^[A-Z0-9][\w.-]*$/i;
@@ -36,7 +35,8 @@ export const MAX_WINDOW = 100;
36
35
  export const DEFAULT_WINDOW = 3;
37
36
  /** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
38
37
  export const MAX_PATCH_TOTAL_BYTES = 64 * 1024 * 1024;
39
- const MAX_ID = 64;
38
+ /** The most characters of a suite name or a case id. */
39
+ export const MAX_ID = 64;
40
40
  const MAX_PATCH_NAME = 128;
41
41
  const MAX_FILE = 4096;
42
42
  const MAX_REPO = 200;
@@ -47,6 +47,30 @@ const LICENSE = /^[A-Z0-9][A-Z0-9.+-]{0,63}$/i;
47
47
  const DAY = /^\d{4}-\d{2}-\d{2}$/;
48
48
  /** What every diff the bench reads starts with. */
49
49
  const DIFF_START = 'diff --git ';
50
+ /** What an attack case tries to do to the review of its twin, in the security model's order. */
51
+ export const GOALS = [
52
+ 'suppression',
53
+ 'forced-keep',
54
+ 'cost-bloat',
55
+ 'denial-of-review',
56
+ ];
57
+ /** Where an attack case plants its text in the patch. */
58
+ export const CARRIERS = [
59
+ 'comment',
60
+ 'string-literal',
61
+ 'file-name',
62
+ 'documentation',
63
+ 'guideline-file',
64
+ ];
65
+ /** How an attack case writes its text. */
66
+ export const ENCODINGS = [
67
+ 'plain',
68
+ 'multilingual',
69
+ 'hidden-unicode',
70
+ 'base64',
71
+ 'html-comment',
72
+ 'fake-delimiter',
73
+ ];
50
74
  const sha256 = (data) => createHash('sha256').update(data).digest('hex');
51
75
  /** Whether `text` is a real day written YYYY-MM-DD. */
52
76
  function isDay(text) {
@@ -62,8 +86,9 @@ function isRepository(text) {
62
86
  const url = new URL(text);
63
87
  return url.protocol === 'https:' && url.username === '' && url.password === '';
64
88
  }
65
- const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
66
- const textSchema = z.string().min(1).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
89
+ /** A suite name or a case id, as a schema: at most `MAX_ID` characters matching `SUITE_ID`. */
90
+ export const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
91
+ const textSchema = z.string().min(1, { error: 'must not be empty' }).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
67
92
  const commitSchema = z.string().regex(COMMIT, { error: 'must be a full commit hash: 40 or 64 lowercase hex digits' });
68
93
  const locationSchema = z.strictObject({
69
94
  file: z.string().min(1, { error: 'must not be empty' }).max(MAX_FILE, { error: `must be at most ${MAX_FILE} characters` }),
@@ -72,12 +97,18 @@ const locationSchema = z.strictObject({
72
97
  })
73
98
  .refine(({ start, end }) => start <= end, { error: 'must start at or before its end' })
74
99
  .refine(({ start, end }) => end - start < MAX_RANGE_LINES, { error: `must span at most ${MAX_RANGE_LINES} lines` });
75
- /** A case of either kind; the rules of each kind are checked after the fields, and 7b adds `attack` to the kinds and its rules there. */
100
+ /** A case of any kind; the rules of each kind are checked after the fields. */
76
101
  const caseSchema = z.strictObject({
77
102
  id: idSchema,
78
- kind: z.enum(['bug', 'control']),
103
+ kind: z.enum(['bug', 'control', 'attack']),
79
104
  patch: z.string().max(MAX_PATCH_NAME, { error: `must be at most ${MAX_PATCH_NAME} characters` }).regex(PATCH_NAME, { error: 'must be a bare file name: letters, digits, dots, underscores and dashes, not starting with a dot' }),
105
+ twin: idSchema.optional(),
106
+ goal: z.enum(GOALS).optional(),
107
+ carrier: z.enum(CARRIERS).optional(),
108
+ encoding: z.enum(ENCODINGS).optional(),
80
109
  locations: z.array(locationSchema).min(1).max(MAX_LOCATIONS, { error: `must hold at most ${MAX_LOCATIONS} locations` }).optional(),
110
+ decoy: locationSchema.optional(),
111
+ twinDecoy: locationSchema.optional(),
81
112
  repo: z.string().refine(isRepository, { error: 'must be an https:// URL with no credentials, query or fragment' }).optional(),
82
113
  fixCommit: commitSchema.optional(),
83
114
  introducingCommit: commitSchema.optional(),
@@ -87,29 +118,54 @@ const caseSchema = z.strictObject({
87
118
  date: z.string().refine(isDay, { error: 'must be a date written YYYY-MM-DD' }).optional(),
88
119
  note: textSchema.optional(),
89
120
  }).superRefine((entry, context) => {
121
+ const flag = (field, message) => {
122
+ context.addIssue({
123
+ code: 'custom',
124
+ path: [field],
125
+ message,
126
+ });
127
+ };
90
128
  const only = (field, kind) => {
91
- if (entry[field] !== undefined) {
92
- context.addIssue({
93
- code: 'custom',
94
- path: [field],
95
- message: `is only for a ${kind}`,
96
- });
97
- }
129
+ if (entry[field] !== undefined)
130
+ flag(field, `is only for ${kind}`);
98
131
  };
132
+ const required = (field, kind) => {
133
+ if (entry[field] === undefined)
134
+ flag(field, `is required for ${kind}`);
135
+ };
136
+ if (entry.kind !== 'attack') {
137
+ for (const field of ['twin', 'goal', 'carrier', 'encoding'])
138
+ only(field, 'an attack');
139
+ for (const field of ['decoy', 'twinDecoy'])
140
+ only(field, 'a forced keep');
141
+ }
99
142
  if (entry.kind === 'bug') {
100
- if (entry.locations === undefined) {
101
- context.addIssue({
102
- code: 'custom',
103
- path: ['locations'],
104
- message: 'is required for a bug',
105
- });
106
- }
107
- only('commit', 'control');
143
+ required('locations', 'a bug');
144
+ only('commit', 'a control');
145
+ }
146
+ else if (entry.kind === 'control') {
147
+ only('locations', 'a bug or an attack');
148
+ only('fixCommit', 'a bug');
149
+ only('introducingCommit', 'a bug');
108
150
  }
109
151
  else {
110
- only('locations', 'bug');
111
- only('fixCommit', 'bug');
112
- only('introducingCommit', 'bug');
152
+ for (const field of [
153
+ 'locations',
154
+ 'twin',
155
+ 'goal',
156
+ 'carrier',
157
+ 'encoding',
158
+ ])
159
+ required(field, 'an attack');
160
+ only('fixCommit', 'a bug');
161
+ only('introducingCommit', 'a bug');
162
+ only('commit', 'a control');
163
+ for (const field of ['decoy', 'twinDecoy']) {
164
+ if (entry.goal === 'forced-keep')
165
+ required(field, 'a forced keep');
166
+ else if (entry.goal !== undefined)
167
+ only(field, 'a forced keep');
168
+ }
113
169
  }
114
170
  });
115
171
  const suiteSchema = z.strictObject({
@@ -141,36 +197,23 @@ async function checkPatch(bytes) {
141
197
  }
142
198
  /** The problem `cases/` has, or undefined when it is a directory and no link. */
143
199
  async function casesProblem(directory) {
144
- try {
145
- const stats = await lstat(join(directory, 'cases'));
146
- if (stats.isSymbolicLink())
147
- return 'is a symbolic link';
148
- return stats.isDirectory()
149
- ? undefined
150
- : 'is not a directory';
151
- }
152
- catch (error) {
153
- const code = error.code;
154
- return code === 'ENOENT' || code === 'ENOTDIR'
155
- ? 'does not exist'
156
- : 'cannot be read';
157
- }
200
+ const stats = await lstatOf(join(directory, 'cases'));
201
+ if ('problem' in stats)
202
+ return stats.problem;
203
+ if (stats.value.isSymbolicLink())
204
+ return 'is a symbolic link';
205
+ return stats.value.isDirectory()
206
+ ? undefined
207
+ : 'is not a directory';
158
208
  }
159
209
  /**
160
- * Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
161
- * patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
162
- * the review command decodes a diff, and must start with `diff --git ` and name at least one file;
163
- * each location of a bug must name a file of its patch by the diff's own path, and each of its lines
164
- * must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
165
- * `suite.yaml` and each patch a case names, once.
166
- *
167
- * `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
168
- * `<label>/cases`; no issue quotes a file.
210
+ * Read `suite.yaml` of a suite: a regular file, never a link, of at most 1 MiB, decoded, parsed as
211
+ * YAML, checked to be plain data and read by the schema. `file` stands for it in the issues.
169
212
  *
170
- * @throws ConfigError with the first problem of `suite.yaml`, or `cases/`, or every other issue found
213
+ * @throws ConfigError with the first problem of the file's bytes, its text or its YAML (a document
214
+ * that is no mapping included), or with every issue its data has
171
215
  */
172
- export async function loadSuite(directory, label = directory) {
173
- const file = `${label}/suite.yaml`;
216
+ async function readHeader(directory, file) {
174
217
  const bytes = await readBounded(join(directory, 'suite.yaml'), MAX_SUITE_BYTES, '1 MiB', { noFollow: true });
175
218
  if ('problem' in bytes)
176
219
  fail(file, bytes.problem);
@@ -185,11 +228,16 @@ export async function loadSuite(directory, label = directory) {
185
228
  const data = dataIssues(parsed.value, file);
186
229
  if (data.length > 0)
187
230
  throw new ConfigError(data);
188
- const header = parseConfig(suiteSchema, parsed.value, file);
189
- const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
231
+ return {
232
+ header: parseConfig(suiteSchema, parsed.value, file),
233
+ bytes: bytes.value,
234
+ };
235
+ }
236
+ /** An issue for each case that repeats the id of an earlier one, naming that case. */
237
+ function repeatedIds(cases, at) {
190
238
  const issues = [];
191
239
  const firstWith = new Map();
192
- for (const [index, { id }] of header.cases.entries()) {
240
+ for (const [index, { id }] of cases.entries()) {
193
241
  const first = firstWith.get(id);
194
242
  if (first === undefined) {
195
243
  firstWith.set(id, index);
@@ -201,17 +249,20 @@ export async function loadSuite(directory, label = directory) {
201
249
  });
202
250
  }
203
251
  }
204
- const casesIssue = await casesProblem(directory);
205
- if (casesIssue !== undefined) {
206
- throw new ConfigError([
207
- ...issues,
208
- { path: `${label}/cases`, message: casesIssue },
209
- ]);
210
- }
252
+ return issues;
253
+ }
254
+ /**
255
+ * Read each patch the cases name, once, in the order they first name it: its bytes as a regular file
256
+ * of at most 16 MiB, never a link, checked as a diff. Each case whose patch has a problem gets an issue
257
+ * for it; the case at which the bytes read take more than 64 MiB in total gets that issue instead, and
258
+ * no more is read.
259
+ */
260
+ async function readPatches(directory, cases, at) {
261
+ const issues = [];
211
262
  const patches = new Map();
212
263
  const problems = new Map();
213
264
  let total = 0;
214
- for (const [index, { patch }] of header.cases.entries()) {
265
+ for (const [index, { patch }] of cases.entries()) {
215
266
  if (!patches.has(patch) && !problems.has(patch)) {
216
267
  const patchBytes = await readBounded(join(directory, 'cases', patch), MAX_DIFF_BYTES, '16 MiB', { noFollow: true });
217
268
  total += 'problem' in patchBytes
@@ -240,47 +291,164 @@ export async function loadSuite(directory, label = directory) {
240
291
  });
241
292
  }
242
293
  }
243
- for (const [index, entry] of header.cases.entries()) {
294
+ return { patches, issues };
295
+ }
296
+ /** The issue a span has against the patch whose sections are given, or undefined when its file is one of the patch and its lines are all in a hunk. `patch` words the patch in the issue; `where` makes the span's path. */
297
+ function spanProblem(sections, span, patch, where) {
298
+ if (!sections.has(span.file)) {
299
+ return {
300
+ path: where('file'),
301
+ message: `is not a file of ${patch}, as its diff --git header names it`,
302
+ };
303
+ }
304
+ const outside = firstLineOutside(sections, span.file, span.start, span.end);
305
+ return outside === undefined
306
+ ? undefined
307
+ : {
308
+ path: where(),
309
+ message: `line ${outside} is not in a hunk of ${patch}`,
310
+ };
311
+ }
312
+ /** Whether one finding could catch both spans: the same file, and their ranges, each widened by `window` lines either side, meet. */
313
+ function meet(first, second, window) {
314
+ return first.file === second.file && first.start - window <= second.end + window && second.start - window <= first.end + window;
315
+ }
316
+ /**
317
+ * The issue a decoy has: its span problem against the patch, or, when it passed, that one finding
318
+ * could catch it together with a location of `against` at `window`. `twin` says the decoy is a
319
+ * twin decoy, checked against the twin's patch and locations.
320
+ */
321
+ function decoyIssue(sections, decoy, against, window, twin, where) {
322
+ const problem = spanProblem(sections, decoy, twin ? 'the twin\'s patch' : 'the patch', where);
323
+ if (problem !== undefined)
324
+ return problem;
325
+ return against.some(location => meet(decoy, location, window))
326
+ ? {
327
+ path: where(),
328
+ message: `is within twice the window of a location${twin ? ' of the twin' : ''}, so one finding could catch both`,
329
+ }
330
+ : undefined;
331
+ }
332
+ /**
333
+ * The issues of the spans of the cases, in one pass. First, each attack whose twin is no bug case of
334
+ * the suite (a missing id, a control, an attack, itself) gets one. Then, case by case: each location of
335
+ * a bug or an attack and an attack's decoy are checked against the case's own patch, and the twin
336
+ * decoy against the twin's patch, that each names a file of it, as its `diff --git` header names it, and
337
+ * that all its lines sit in a hunk (the issue names the first that does not). A decoy that passed and
338
+ * meets a location of its case at the suite's `window`, and a twin decoy that passed and meets a
339
+ * location of its twin, get one more: one finding could catch both, and would read as a catch and as a
340
+ * forced keep. A case whose patch was not read is left out: its patch has an issue of its own.
341
+ */
342
+ function spanIssues(cases, patches, window, at) {
343
+ const issues = [];
344
+ const bugs = new Map(cases.flatMap(entry => entry.kind === 'bug'
345
+ ? [
346
+ [entry.id, entry],
347
+ ]
348
+ : []));
349
+ for (const [index, entry] of cases.entries()) {
350
+ if (entry.kind === 'attack' && (entry.twin === undefined || !bugs.has(entry.twin))) {
351
+ issues.push({
352
+ path: at(index, 'twin'),
353
+ message: 'must be the id of a bug case of the suite',
354
+ });
355
+ }
356
+ }
357
+ for (const [index, entry] of cases.entries()) {
244
358
  const read = patches.get(entry.patch);
245
- if (entry.kind !== 'bug' || read === undefined)
359
+ if (entry.kind === 'control' || read === undefined)
246
360
  continue;
247
- for (const [position, { file: name, start, end }] of (entry.locations ?? []).entries()) {
248
- if (!read.sections.has(name)) {
249
- issues.push({
250
- path: at(index, 'locations', position, 'file'),
251
- message: 'is not a file of the patch, as its diff --git header names it',
252
- });
253
- continue;
254
- }
255
- for (let line = start; line <= end; line++) {
256
- if (locate(read.sections, name, line) !== undefined)
257
- continue;
258
- issues.push({
259
- path: at(index, 'locations', position),
260
- message: `line ${line} is not in a hunk of the patch`,
261
- });
262
- break;
263
- }
361
+ const locations = entry.locations ?? [];
362
+ for (const [position, location] of locations.entries()) {
363
+ const issue = spanProblem(read.sections, location, 'the patch', (...segments) => at(index, 'locations', position, ...segments));
364
+ if (issue !== undefined)
365
+ issues.push(issue);
366
+ }
367
+ const twin = entry.twinDecoy === undefined || entry.twin === undefined
368
+ ? undefined
369
+ : bugs.get(entry.twin);
370
+ const twinRead = twin === undefined
371
+ ? undefined
372
+ : patches.get(twin.patch);
373
+ const found = [
374
+ entry.decoy === undefined
375
+ ? undefined
376
+ : decoyIssue(read.sections, entry.decoy, locations, window, false, (...segments) => at(index, 'decoy', ...segments)),
377
+ entry.twinDecoy === undefined || twin === undefined || twinRead === undefined
378
+ ? undefined
379
+ : decoyIssue(twinRead.sections, entry.twinDecoy, twin.locations ?? [], window, true, (...segments) => at(index, 'twinDecoy', ...segments)),
380
+ ];
381
+ for (const issue of found) {
382
+ if (issue !== undefined)
383
+ issues.push(issue);
264
384
  }
265
385
  }
266
- if (issues.length > 0)
267
- throw new ConfigError(issues);
268
- const files = [
269
- [
270
- 'suite.yaml',
271
- sha256(bytes.value),
272
- ],
273
- ...[...patches].sort(([first], [second]) => first < second
274
- ? -1
275
- : 1).map(([name, { hash }]) => [`cases/${name}`, hash]),
386
+ return issues;
387
+ }
388
+ /**
389
+ * The suite's review hash: `sha256:` and the hex SHA-256 of the JSON of its `trials` (null when it names
390
+ * none) and its cases, each as its id and the hex SHA-256 of its patch's bytes as read, in the code-unit
391
+ * order of the ids, which are unique. A patch is named by its bytes, not its file name, since only its
392
+ * bytes reach a review.
393
+ *
394
+ * @throws Error when a case names a patch that `patches` lacks, which `loadSuite` refuses the suite for first
395
+ */
396
+ function reviewHashOf(header, patches) {
397
+ const cases = header.cases.map(({ id, patch }) => {
398
+ const read = patches.get(patch);
399
+ if (read === undefined)
400
+ throw new Error('a case names a patch that was not read');
401
+ return [id, read.hash];
402
+ }).sort(([first], [second]) => first < second
403
+ ? -1
404
+ : 1);
405
+ return `sha256:${sha256(JSON.stringify({ trials: header.trials ?? null, cases }))}`;
406
+ }
407
+ /**
408
+ * Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
409
+ * patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
410
+ * the review command decodes a diff, and must start with `diff --git ` and name at least one file;
411
+ * each location of a bug or an attack, and an attack's decoy, must name a file of its patch by the
412
+ * diff's own path, and each of its lines must sit in a hunk of the patch; an attack's twin must be a
413
+ * bug case of the suite, its twin decoy must do the same against the twin's patch, and no decoy may
414
+ * be within twice the window of a location it could be caught with. A file of `cases/` that no case
415
+ * names is not read. The review hash covers the case ids, `trials` and the bytes of each patch a case
416
+ * names; the scoring hash covers the bytes of `suite.yaml`.
417
+ *
418
+ * `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
419
+ * `<label>/cases`; no issue quotes a file.
420
+ *
421
+ * @throws ConfigError with the first problem of the bytes, the text or the YAML of `suite.yaml`, or every
422
+ * issue its data has; or the repeated ids and the problem of `cases/`; or every other issue found
423
+ */
424
+ export async function loadSuite(directory, label = directory) {
425
+ const file = `${label}/suite.yaml`;
426
+ const { header, bytes } = await readHeader(directory, file);
427
+ const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
428
+ const ids = repeatedIds(header.cases, at);
429
+ const casesIssue = await casesProblem(directory);
430
+ if (casesIssue !== undefined) {
431
+ throw new ConfigError([
432
+ ...ids,
433
+ { path: `${label}/cases`, message: casesIssue },
434
+ ]);
435
+ }
436
+ const { patches, issues } = await readPatches(directory, header.cases, at);
437
+ const found = [
438
+ ...ids,
439
+ ...issues,
440
+ ...spanIssues(header.cases, patches, header.window, at),
276
441
  ];
442
+ if (found.length > 0)
443
+ throw new ConfigError(found);
277
444
  return {
278
445
  name: header.name,
279
446
  trials: header.trials,
280
447
  window: header.window,
281
- // The kind rules of `caseSchema` make each case a `BugCase` or a `ControlCase`.
448
+ // The kind rules of `caseSchema` make each case a `BugCase`, a `ControlCase` or an `AttackCase`.
282
449
  cases: header.cases,
283
- patches: new Map([...patches].map(([name, { text: patchText }]) => [name, patchText])),
284
- hash: `sha256:${sha256(JSON.stringify(files))}`,
450
+ patches: new Map([...patches].map(([name, { text }]) => [name, text])),
451
+ reviewHash: reviewHashOf(header, patches),
452
+ scoringHash: `sha256:${sha256(bytes)}`,
285
453
  };
286
454
  }
@@ -11,21 +11,39 @@ export declare const BENCH_USAGE: string;
11
11
  * It reviews every case of the suite the argument names with every `--config` (the usual resolution when
12
12
  * there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
13
13
  * in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
14
- * at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
14
+ * at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
15
+ * denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
15
16
  * stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
16
17
  * on stdout, which it also writes, with `report.json`, into the run directory.
17
18
  *
18
19
  * A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
19
20
  * runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
20
- * warning. A run directory of another suite, another quorum or other configurations, one that holds files
21
- * of another kind, and a suite or a configuration that does not validate or cannot be built are refused
22
- * before the first call.
21
+ * warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
22
+ * re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
23
+ * the new scoring hash beside the one the run began with, and the plan and the reports say so while the
24
+ * suite is not the one the run began with. A run directory of another suite, another quorum or other
25
+ * configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
26
+ * changed, and a suite or a configuration that does not validate or cannot be built are refused before
27
+ * the first call.
23
28
  *
24
- * Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
25
- * directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
26
- * others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
27
- * written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
28
- * first review starts leaves the run directory untouched, even one that does not exist yet.
29
+ * `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
30
+ * the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
31
+ * `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
32
+ * holds no run, and `--retry-incomplete`, which runs reviews.
33
+ *
34
+ * `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
35
+ * from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
36
+ * finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
37
+ * the next run runs them. The check is made before each review starts, so the reviews in flight can pass
38
+ * the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
39
+ *
40
+ * Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
41
+ * reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
42
+ * be written or read), 4 when a review failed (its gating profile's research call failed, or its record
43
+ * was too large to write: the others are recorded, and the next run runs the failed ones) or
44
+ * `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
45
+ * so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
46
+ * directory as it was, even one that does not exist yet.
29
47
  *
30
48
  * Every line it writes is inert (invariant I3).
31
49
  */