@tobi-or-not/quorum 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +5 -0
  3. package/dist/bench/fingerprint.d.ts +36 -0
  4. package/dist/bench/fingerprint.js +79 -0
  5. package/dist/bench/records.d.ts +62 -0
  6. package/dist/bench/records.js +161 -0
  7. package/dist/bench/report.d.ts +110 -0
  8. package/dist/bench/report.js +332 -0
  9. package/dist/bench/run-dir.d.ts +101 -0
  10. package/dist/bench/run-dir.js +379 -0
  11. package/dist/bench/runner.d.ts +80 -0
  12. package/dist/bench/runner.js +160 -0
  13. package/dist/bench/score.d.ts +30 -0
  14. package/dist/bench/score.js +38 -0
  15. package/dist/bench/stats.d.ts +51 -0
  16. package/dist/bench/stats.js +83 -0
  17. package/dist/bench/suite.d.ts +87 -0
  18. package/dist/bench/suite.js +286 -0
  19. package/dist/cli/bench.d.ts +32 -0
  20. package/dist/cli/bench.js +463 -0
  21. package/dist/cli/bin.js +9 -1
  22. package/dist/cli/io.d.ts +45 -0
  23. package/dist/cli/io.js +45 -0
  24. package/dist/cli/main.d.ts +7 -37
  25. package/dist/cli/main.js +47 -60
  26. package/dist/cli/print-config.js +38 -9
  27. package/dist/cli/render.d.ts +10 -0
  28. package/dist/cli/render.js +37 -16
  29. package/dist/config/claude-code.js +42 -11
  30. package/dist/config/env.js +135 -32
  31. package/dist/config/from-config.js +43 -9
  32. package/dist/config/load.d.ts +43 -0
  33. package/dist/config/load.js +205 -46
  34. package/dist/config/profiles.js +87 -20
  35. package/dist/config/resolve.js +194 -47
  36. package/dist/config/schema.js +98 -23
  37. package/dist/core/diff.d.ts +10 -0
  38. package/dist/core/diff.js +90 -17
  39. package/dist/core/errors.js +14 -5
  40. package/dist/core/flags.js +30 -6
  41. package/dist/core/output.js +49 -13
  42. package/dist/core/policy.js +6 -2
  43. package/dist/core/profile.js +42 -15
  44. package/dist/core/prompts.d.ts +3 -1
  45. package/dist/core/prompts.js +28 -8
  46. package/dist/core/quorum.js +128 -33
  47. package/dist/core/sanitize.d.ts +16 -1
  48. package/dist/core/sanitize.js +31 -5
  49. package/dist/core/stages/context.js +30 -5
  50. package/dist/core/stages/panel.js +55 -9
  51. package/dist/core/stages/research.js +13 -3
  52. package/dist/core/verify.js +6 -2
  53. package/dist/index.js +5 -1
  54. package/dist/providers/anthropic.js +28 -5
  55. package/dist/providers/build.js +57 -16
  56. package/dist/providers/claude-code.js +206 -51
  57. package/dist/providers/gemini.js +61 -14
  58. package/dist/providers/http.js +32 -9
  59. package/dist/providers/messages.js +14 -2
  60. package/dist/providers/ollama.js +31 -8
  61. package/dist/providers/openai-compatible.js +51 -15
  62. package/dist/providers/options.js +120 -40
  63. package/dist/providers/presets.js +143 -16
  64. package/dist/providers/reply.js +15 -5
  65. package/dist/providers/size.js +11 -3
  66. package/package.json +6 -5
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Scoring a review against a suite case: whether its findings catch the case's bug, and how many kept
3
+ * findings catch nothing. Pure code over a result and a case, so a record can be scored again later.
4
+ */
5
+ /**
6
+ * Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
7
+ * line from `window` lines before the location's start to `window` lines after its end. The window is
8
+ * taken as given, 0 included.
9
+ */
10
+ export function catches(finding, location, window) {
11
+ return finding.file === location.file && finding.line >= location.start - window && finding.line <= location.end + window;
12
+ }
13
+ /**
14
+ * Score a review against a case. A bug is caught by a kept finding (`keptCatch`) or by any finding the
15
+ * finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window` lines of one of its
16
+ * locations. A control has nothing to catch, so every kept finding is unmatched. Every profile of the
17
+ * review counts, for the catches and for the gaps alike.
18
+ */
19
+ export function scoreReview(result, suiteCase, window) {
20
+ const incomplete = result.unreviewed.length > 0 || result.meta.incomplete.length > 0;
21
+ if (suiteCase.kind === 'control') {
22
+ return {
23
+ keptCatch: false,
24
+ proposedCatch: false,
25
+ unmatchedKept: result.kept.length,
26
+ incomplete,
27
+ };
28
+ }
29
+ const hits = (verdict) => suiteCase.locations.some(location => catches(verdict, location, window));
30
+ const keptCatch = result.kept.some(hits);
31
+ const proposedCatch = keptCatch || result.discarded.some(hits) || result.unreviewed.some(hits);
32
+ return {
33
+ keptCatch,
34
+ proposedCatch,
35
+ unmatchedKept: result.kept.filter(verdict => !hits(verdict)).length,
36
+ incomplete,
37
+ };
38
+ }
@@ -0,0 +1,51 @@
1
+ /**
2
+ * The statistics of the report, written by hand and held to reference values: the Wilson score
3
+ * interval of a rate, the exact two-sided sign test and nearest-rank percentiles. Pure code, with no
4
+ * package behind it; each function throws a `RangeError` for an argument outside its domain.
5
+ */
6
+ /** The two-sided 95% quantile of the standard normal distribution. */
7
+ export declare const Z95 = 1.959963984540054;
8
+ /** A confidence interval of a rate, inside 0 and 1. */
9
+ export interface Interval {
10
+ readonly low: number;
11
+ readonly high: number;
12
+ }
13
+ /**
14
+ * The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
15
+ * With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
16
+ * `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
17
+ * lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
18
+ * reach to within a rounding error.
19
+ *
20
+ * `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
21
+ * fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
22
+ *
23
+ * Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
24
+ * and 1 gives [0.7225, 1].
25
+ *
26
+ * @throws {RangeError} when `n` is not a positive integer, or `rate` is outside [0, 1].
27
+ */
28
+ export declare function wilson(rate: number, n: number, z?: number): Interval;
29
+ /**
30
+ * The exact two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
31
+ * `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
32
+ * and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
33
+ * (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
34
+ * `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number` for any `n` and
35
+ * the reference values come out exact.
36
+ *
37
+ * Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
38
+ * 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
39
+ *
40
+ * @throws {RangeError} when a count is not an integer of at least 0.
41
+ */
42
+ export declare function signTest(better: number, worse: number): number;
43
+ /**
44
+ * The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
45
+ * ascending, the product taken first so the rank stays an integer. The values are not changed.
46
+ *
47
+ * Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
48
+ *
49
+ * @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
50
+ */
51
+ export declare function nearestRank(values: readonly number[], percent: number): number;
@@ -0,0 +1,83 @@
1
+ /**
2
+ * The statistics of the report, written by hand and held to reference values: the Wilson score
3
+ * interval of a rate, the exact two-sided sign test and nearest-rank percentiles. Pure code, with no
4
+ * package behind it; each function throws a `RangeError` for an argument outside its domain.
5
+ */
6
+ /** The two-sided 95% quantile of the standard normal distribution. */
7
+ export const Z95 = 1.959963984540054;
8
+ /**
9
+ * The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
10
+ * With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
11
+ * `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
12
+ * lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
13
+ * reach to within a rounding error.
14
+ *
15
+ * `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
16
+ * fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
17
+ *
18
+ * Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
19
+ * and 1 gives [0.7225, 1].
20
+ *
21
+ * @throws {RangeError} when `n` is not a positive integer, or `rate` is outside [0, 1].
22
+ */
23
+ export function wilson(rate, n, z = Z95) {
24
+ if (!Number.isInteger(n) || n < 1)
25
+ throw new RangeError('the number of cases must be a positive integer');
26
+ if (!(rate >= 0 && rate <= 1))
27
+ throw new RangeError('the rate must be between 0 and 1');
28
+ const d = 1 + z * z / n;
29
+ const center = (rate + z * z / (2 * n)) / d;
30
+ const halfWidth = z / d * Math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n));
31
+ const low = rate === 0
32
+ ? 0
33
+ : Math.max(0, center - halfWidth);
34
+ const high = rate === 1
35
+ ? 1
36
+ : Math.min(1, center + halfWidth);
37
+ return { low, high };
38
+ }
39
+ /**
40
+ * The exact two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
41
+ * `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
42
+ * and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
43
+ * (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
44
+ * `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number` for any `n` and
45
+ * the reference values come out exact.
46
+ *
47
+ * Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
48
+ * 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
49
+ *
50
+ * @throws {RangeError} when a count is not an integer of at least 0.
51
+ */
52
+ export function signTest(better, worse) {
53
+ if (!Number.isInteger(better) || better < 0 || !Number.isInteger(worse) || worse < 0)
54
+ throw new RangeError('a count must be a whole number of at least 0');
55
+ const n = better + worse;
56
+ if (n === 0)
57
+ return 1;
58
+ const k = Math.min(better, worse);
59
+ let coefficient = 1n;
60
+ let sum = 1n;
61
+ for (let i = 1; i <= k; i++) {
62
+ coefficient = coefficient * BigInt(n - i + 1) / BigInt(i);
63
+ sum += coefficient;
64
+ }
65
+ const shift = BigInt(Math.max(0, n + 1 - 1000));
66
+ return Math.min(1, Number((2n * sum) >> shift) / Number((1n << BigInt(n)) >> shift));
67
+ }
68
+ /**
69
+ * The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
70
+ * ascending, the product taken first so the rank stays an integer. The values are not changed.
71
+ *
72
+ * Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
73
+ *
74
+ * @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
75
+ */
76
+ export function nearestRank(values, percent) {
77
+ if (values.length === 0)
78
+ throw new RangeError('there must be at least one value');
79
+ if (!(percent > 0 && percent <= 100))
80
+ throw new RangeError('the percent must be above 0 and at most 100');
81
+ const rank = Math.max(1, Math.ceil(percent * values.length / 100));
82
+ return [...values].sort((a, b) => a - b)[rank - 1];
83
+ }
@@ -0,0 +1,87 @@
1
+ /**
2
+ * Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
3
+ * patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
4
+ * file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
5
+ * YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
6
+ * deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
7
+ * `cases/` and every patch must not be a symbolic link.
8
+ *
9
+ * Everything is checked before any call, and no message quotes the file or a patch.
10
+ */
11
+ /** A suite name or a case id: a name for a directory and for a record, so it holds no separator and no `..`. */
12
+ export declare const SUITE_ID: RegExp;
13
+ /** A patch's file name: bare, and not starting with a dot. */
14
+ export declare const PATCH_NAME: RegExp;
15
+ /** The most `suite.yaml` may hold, in bytes. */
16
+ export declare const MAX_SUITE_BYTES: number;
17
+ export declare const MAX_CASES = 1000;
18
+ export declare const MAX_LOCATIONS = 32;
19
+ /** The most lines one location may span. */
20
+ export declare const MAX_RANGE_LINES = 500;
21
+ /** The most characters of a note or of one copyright line. */
22
+ export declare const MAX_TEXT = 500;
23
+ export declare const MAX_COPYRIGHT_LINES = 20;
24
+ export declare const MAX_TRIALS = 100;
25
+ export declare const MAX_WINDOW = 100;
26
+ /** How many lines either side of a location still count as a catch, when the suite does not say. */
27
+ export declare const DEFAULT_WINDOW = 3;
28
+ /** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
29
+ export declare const MAX_PATCH_TOTAL_BYTES: number;
30
+ /** A span of lines of the new file that holds a bug, at the file's own path in the patch. */
31
+ export interface Location {
32
+ readonly file: string;
33
+ readonly start: number;
34
+ readonly end: number;
35
+ }
36
+ /** What a case says about where it comes from, whatever its kind. */
37
+ interface CaseOrigin {
38
+ readonly id: string;
39
+ /** The name of its patch in `cases/`. */
40
+ readonly patch: string;
41
+ readonly repo?: string;
42
+ readonly license?: string;
43
+ readonly copyright?: readonly string[];
44
+ readonly date?: string;
45
+ readonly note?: string;
46
+ }
47
+ /** A patch that introduces a vulnerability at its locations. */
48
+ export interface BugCase extends CaseOrigin {
49
+ readonly kind: 'bug';
50
+ readonly locations: readonly Location[];
51
+ readonly fixCommit?: string;
52
+ readonly introducingCommit?: string;
53
+ }
54
+ /** A patch that introduces none. */
55
+ export interface ControlCase extends CaseOrigin {
56
+ readonly kind: 'control';
57
+ readonly commit?: string;
58
+ }
59
+ export type SuiteCase = BugCase | ControlCase;
60
+ /** A suite once read and checked. */
61
+ export interface Suite {
62
+ readonly name: string;
63
+ /** How many trials a run takes when it is not told, if the suite says. */
64
+ readonly trials?: number;
65
+ /** How many lines either side of a location still count as a catch. */
66
+ readonly window: number;
67
+ readonly cases: readonly SuiteCase[];
68
+ /** The decoded text of each patch by name, in the order the cases first name them. */
69
+ readonly patches: ReadonlyMap<string, string>;
70
+ /** `sha256:` and the hex SHA-256 of the files' own hashes: `suite.yaml` and each patch a case names. */
71
+ readonly hash: string;
72
+ }
73
+ /**
74
+ * Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
75
+ * patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
76
+ * the review command decodes a diff, and must start with `diff --git ` and name at least one file;
77
+ * each location of a bug must name a file of its patch by the diff's own path, and each of its lines
78
+ * must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
79
+ * `suite.yaml` and each patch a case names, once.
80
+ *
81
+ * `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
82
+ * `<label>/cases`; no issue quotes a file.
83
+ *
84
+ * @throws ConfigError with the first problem of `suite.yaml`, or `cases/`, or every other issue found
85
+ */
86
+ export declare function loadSuite(directory: string, label?: string): Promise<Suite>;
87
+ export {};
@@ -0,0 +1,286 @@
1
+ /**
2
+ * Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
3
+ * patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
4
+ * file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
5
+ * YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
6
+ * deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
7
+ * `cases/` and every patch must not be a symbolic link.
8
+ *
9
+ * Everything is checked before any call, and no message quotes the file or a patch.
10
+ */
11
+ import { createHash } from 'node:crypto';
12
+ import { lstat } from 'node:fs/promises';
13
+ import { join } from 'node:path';
14
+ import { z } from 'zod';
15
+ import { dataIssues, decode, isPlainObject, parseYaml, readBounded } from "../config/load.js";
16
+ import { configPath, parseConfig } from "../config/schema.js";
17
+ import { decodeDiff, locate, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
18
+ import { ConfigError } from "../core/errors.js";
19
+ import { hiddenLabels } from "../core/sanitize.js";
20
+ /** A suite name or a case id: a name for a directory and for a record, so it holds no separator and no `..`. */
21
+ export const SUITE_ID = /^[a-z0-9][a-z0-9._-]*$/;
22
+ /** A patch's file name: bare, and not starting with a dot. */
23
+ export const PATCH_NAME = /^[A-Z0-9][\w.-]*$/i;
24
+ /** The most `suite.yaml` may hold, in bytes. */
25
+ export const MAX_SUITE_BYTES = 1024 * 1024;
26
+ export const MAX_CASES = 1000;
27
+ export const MAX_LOCATIONS = 32;
28
+ /** The most lines one location may span. */
29
+ export const MAX_RANGE_LINES = 500;
30
+ /** The most characters of a note or of one copyright line. */
31
+ export const MAX_TEXT = 500;
32
+ export const MAX_COPYRIGHT_LINES = 20;
33
+ export const MAX_TRIALS = 100;
34
+ export const MAX_WINDOW = 100;
35
+ /** How many lines either side of a location still count as a catch, when the suite does not say. */
36
+ export const DEFAULT_WINDOW = 3;
37
+ /** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
38
+ export const MAX_PATCH_TOTAL_BYTES = 64 * 1024 * 1024;
39
+ const MAX_ID = 64;
40
+ const MAX_PATCH_NAME = 128;
41
+ const MAX_FILE = 4096;
42
+ const MAX_REPO = 200;
43
+ /** A full commit hash: SHA-1 (40 digits) or SHA-256 (64). */
44
+ const COMMIT = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
45
+ /** One SPDX license id, not an expression. */
46
+ const LICENSE = /^[A-Z0-9][A-Z0-9.+-]{0,63}$/i;
47
+ const DAY = /^\d{4}-\d{2}-\d{2}$/;
48
+ /** What every diff the bench reads starts with. */
49
+ const DIFF_START = 'diff --git ';
50
+ const sha256 = (data) => createHash('sha256').update(data).digest('hex');
51
+ /** Whether `text` is a real day written YYYY-MM-DD. */
52
+ function isDay(text) {
53
+ if (!DAY.test(text))
54
+ return false;
55
+ const time = Date.parse(`${text}T00:00:00Z`);
56
+ return !Number.isNaN(time) && new Date(time).toISOString().startsWith(text);
57
+ }
58
+ /** Whether `text` is an https:// URL with no credentials, query or fragment. */
59
+ function isRepository(text) {
60
+ if (text.length > MAX_REPO || /[\s?#]/u.test(text) || hiddenLabels(text).length > 0 || !URL.canParse(text))
61
+ return false;
62
+ const url = new URL(text);
63
+ return url.protocol === 'https:' && url.username === '' && url.password === '';
64
+ }
65
+ const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
66
+ const textSchema = z.string().min(1).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
67
+ const commitSchema = z.string().regex(COMMIT, { error: 'must be a full commit hash: 40 or 64 lowercase hex digits' });
68
+ const locationSchema = z.strictObject({
69
+ file: z.string().min(1, { error: 'must not be empty' }).max(MAX_FILE, { error: `must be at most ${MAX_FILE} characters` }),
70
+ start: z.int().min(1),
71
+ end: z.int().min(1),
72
+ })
73
+ .refine(({ start, end }) => start <= end, { error: 'must start at or before its end' })
74
+ .refine(({ start, end }) => end - start < MAX_RANGE_LINES, { error: `must span at most ${MAX_RANGE_LINES} lines` });
75
+ /** A case of either kind; the rules of each kind are checked after the fields, and 7b adds `attack` to the kinds and its rules there. */
76
+ const caseSchema = z.strictObject({
77
+ id: idSchema,
78
+ kind: z.enum(['bug', 'control']),
79
+ patch: z.string().max(MAX_PATCH_NAME, { error: `must be at most ${MAX_PATCH_NAME} characters` }).regex(PATCH_NAME, { error: 'must be a bare file name: letters, digits, dots, underscores and dashes, not starting with a dot' }),
80
+ locations: z.array(locationSchema).min(1).max(MAX_LOCATIONS, { error: `must hold at most ${MAX_LOCATIONS} locations` }).optional(),
81
+ repo: z.string().refine(isRepository, { error: 'must be an https:// URL with no credentials, query or fragment' }).optional(),
82
+ fixCommit: commitSchema.optional(),
83
+ introducingCommit: commitSchema.optional(),
84
+ commit: commitSchema.optional(),
85
+ license: z.string().regex(LICENSE, { error: 'must be one SPDX license id, such as MIT or Apache-2.0' }).optional(),
86
+ copyright: z.array(textSchema).min(1).max(MAX_COPYRIGHT_LINES, { error: `must hold at most ${MAX_COPYRIGHT_LINES} lines` }).optional(),
87
+ date: z.string().refine(isDay, { error: 'must be a date written YYYY-MM-DD' }).optional(),
88
+ note: textSchema.optional(),
89
+ }).superRefine((entry, context) => {
90
+ const only = (field, kind) => {
91
+ if (entry[field] !== undefined) {
92
+ context.addIssue({
93
+ code: 'custom',
94
+ path: [field],
95
+ message: `is only for a ${kind}`,
96
+ });
97
+ }
98
+ };
99
+ if (entry.kind === 'bug') {
100
+ if (entry.locations === undefined) {
101
+ context.addIssue({
102
+ code: 'custom',
103
+ path: ['locations'],
104
+ message: 'is required for a bug',
105
+ });
106
+ }
107
+ only('commit', 'control');
108
+ }
109
+ else {
110
+ only('locations', 'bug');
111
+ only('fixCommit', 'bug');
112
+ only('introducingCommit', 'bug');
113
+ }
114
+ });
115
+ const suiteSchema = z.strictObject({
116
+ name: idSchema,
117
+ trials: z.int().min(1).max(MAX_TRIALS).optional(),
118
+ window: z.int().min(0).max(MAX_WINDOW).default(DEFAULT_WINDOW),
119
+ cases: z.array(caseSchema).min(1).max(MAX_CASES, { error: `must hold at most ${MAX_CASES} cases` }),
120
+ });
121
+ function fail(path, message) {
122
+ throw new ConfigError([
123
+ { path, message },
124
+ ]);
125
+ }
126
+ /** A patch's bytes decoded as the review command decodes a diff, and checked to be a pure diff of at least one file quorum reads. */
127
+ async function checkPatch(bytes) {
128
+ const text = await decodeDiff([bytes]);
129
+ if (text === undefined)
130
+ return { problem: 'is over 16 MiB' };
131
+ if (!text.startsWith(DIFF_START))
132
+ return { problem: 'must be a pure diff that starts at its first diff --git line' };
133
+ const sections = splitDiff(text);
134
+ if (sections.size === 0)
135
+ return { problem: 'names no file in a diff --git header quorum reads' };
136
+ return {
137
+ text,
138
+ sections,
139
+ hash: sha256(bytes),
140
+ };
141
+ }
142
+ /** The problem `cases/` has, or undefined when it is a directory and no link. */
143
+ async function casesProblem(directory) {
144
+ try {
145
+ const stats = await lstat(join(directory, 'cases'));
146
+ if (stats.isSymbolicLink())
147
+ return 'is a symbolic link';
148
+ return stats.isDirectory()
149
+ ? undefined
150
+ : 'is not a directory';
151
+ }
152
+ catch (error) {
153
+ const code = error.code;
154
+ return code === 'ENOENT' || code === 'ENOTDIR'
155
+ ? 'does not exist'
156
+ : 'cannot be read';
157
+ }
158
+ }
159
+ /**
160
+ * Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
161
+ * patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
162
+ * the review command decodes a diff, and must start with `diff --git ` and name at least one file;
163
+ * each location of a bug must name a file of its patch by the diff's own path, and each of its lines
164
+ * must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
165
+ * `suite.yaml` and each patch a case names, once.
166
+ *
167
+ * `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
168
+ * `<label>/cases`; no issue quotes a file.
169
+ *
170
+ * @throws ConfigError with the first problem of `suite.yaml`, or `cases/`, or every other issue found
171
+ */
172
+ export async function loadSuite(directory, label = directory) {
173
+ const file = `${label}/suite.yaml`;
174
+ const bytes = await readBounded(join(directory, 'suite.yaml'), MAX_SUITE_BYTES, '1 MiB', { noFollow: true });
175
+ if ('problem' in bytes)
176
+ fail(file, bytes.problem);
177
+ const text = decode(bytes.value);
178
+ if ('problem' in text)
179
+ fail(file, text.problem);
180
+ const parsed = parseYaml(text.value);
181
+ if ('problem' in parsed)
182
+ fail(file, parsed.problem);
183
+ if (!isPlainObject(parsed.value))
184
+ fail(file, 'must hold a mapping');
185
+ const data = dataIssues(parsed.value, file);
186
+ if (data.length > 0)
187
+ throw new ConfigError(data);
188
+ const header = parseConfig(suiteSchema, parsed.value, file);
189
+ const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
190
+ const issues = [];
191
+ const firstWith = new Map();
192
+ for (const [index, { id }] of header.cases.entries()) {
193
+ const first = firstWith.get(id);
194
+ if (first === undefined) {
195
+ firstWith.set(id, index);
196
+ }
197
+ else {
198
+ issues.push({
199
+ path: at(index, 'id'),
200
+ message: `repeats the id of cases.${first}`,
201
+ });
202
+ }
203
+ }
204
+ const casesIssue = await casesProblem(directory);
205
+ if (casesIssue !== undefined) {
206
+ throw new ConfigError([
207
+ ...issues,
208
+ { path: `${label}/cases`, message: casesIssue },
209
+ ]);
210
+ }
211
+ const patches = new Map();
212
+ const problems = new Map();
213
+ let total = 0;
214
+ for (const [index, { patch }] of header.cases.entries()) {
215
+ if (!patches.has(patch) && !problems.has(patch)) {
216
+ const patchBytes = await readBounded(join(directory, 'cases', patch), MAX_DIFF_BYTES, '16 MiB', { noFollow: true });
217
+ total += 'problem' in patchBytes
218
+ ? patchBytes.counted ?? 0
219
+ : patchBytes.value.length;
220
+ if (total > MAX_PATCH_TOTAL_BYTES) {
221
+ issues.push({
222
+ path: at(index, 'patch'),
223
+ message: 'takes the patches over 64 MiB in total',
224
+ });
225
+ break;
226
+ }
227
+ const read = 'problem' in patchBytes
228
+ ? patchBytes
229
+ : await checkPatch(patchBytes.value);
230
+ if ('problem' in read)
231
+ problems.set(patch, read.problem);
232
+ else
233
+ patches.set(patch, read);
234
+ }
235
+ const problem = problems.get(patch);
236
+ if (problem !== undefined) {
237
+ issues.push({
238
+ path: at(index, 'patch'),
239
+ message: problem,
240
+ });
241
+ }
242
+ }
243
+ for (const [index, entry] of header.cases.entries()) {
244
+ const read = patches.get(entry.patch);
245
+ if (entry.kind !== 'bug' || read === undefined)
246
+ continue;
247
+ for (const [position, { file: name, start, end }] of (entry.locations ?? []).entries()) {
248
+ if (!read.sections.has(name)) {
249
+ issues.push({
250
+ path: at(index, 'locations', position, 'file'),
251
+ message: 'is not a file of the patch, as its diff --git header names it',
252
+ });
253
+ continue;
254
+ }
255
+ for (let line = start; line <= end; line++) {
256
+ if (locate(read.sections, name, line) !== undefined)
257
+ continue;
258
+ issues.push({
259
+ path: at(index, 'locations', position),
260
+ message: `line ${line} is not in a hunk of the patch`,
261
+ });
262
+ break;
263
+ }
264
+ }
265
+ }
266
+ if (issues.length > 0)
267
+ throw new ConfigError(issues);
268
+ const files = [
269
+ [
270
+ 'suite.yaml',
271
+ sha256(bytes.value),
272
+ ],
273
+ ...[...patches].sort(([first], [second]) => first < second
274
+ ? -1
275
+ : 1).map(([name, { hash }]) => [`cases/${name}`, hash]),
276
+ ];
277
+ return {
278
+ name: header.name,
279
+ trials: header.trials,
280
+ window: header.window,
281
+ // The kind rules of `caseSchema` make each case a `BugCase` or a `ControlCase`.
282
+ cases: header.cases,
283
+ patches: new Map([...patches].map(([name, { text: patchText }]) => [name, patchText])),
284
+ hash: `sha256:${sha256(JSON.stringify(files))}`,
285
+ };
286
+ }
@@ -0,0 +1,32 @@
1
+ /**
2
+ * The `quorum bench` subcommand: it reviews every case of a suite with every configuration, `--trials`
3
+ * times each, records each review in a run directory it can resume, and reports what each configuration
4
+ * caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
5
+ */
6
+ import type { CliIO } from './io.ts';
7
+ export declare const BENCH_USAGE: string;
8
+ /**
9
+ * Run `quorum bench` with its arguments (those after `bench`) and return the exit code. Never throws.
10
+ *
11
+ * It reviews every case of the suite the argument names with every `--config` (the usual resolution when
12
+ * there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
13
+ * in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
14
+ * at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
15
+ * stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
16
+ * on stdout, which it also writes, with `report.json`, into the run directory.
17
+ *
18
+ * A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
19
+ * runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
20
+ * warning. A run directory of another suite, another quorum or other configurations, one that holds files
21
+ * of another kind, and a suite or a configuration that does not validate or cannot be built are refused
22
+ * before the first call.
23
+ *
24
+ * Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
25
+ * directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
26
+ * others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
27
+ * written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
28
+ * first review starts leaves the run directory untouched, even one that does not exist yet.
29
+ *
30
+ * Every line it writes is inert (invariant I3).
31
+ */
32
+ export declare function runBench(io: CliIO, argv: readonly string[]): Promise<number>;