@tobi-or-not/quorum 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/bench/fingerprint.d.ts +36 -0
- package/dist/bench/fingerprint.js +79 -0
- package/dist/bench/records.d.ts +62 -0
- package/dist/bench/records.js +161 -0
- package/dist/bench/report.d.ts +110 -0
- package/dist/bench/report.js +332 -0
- package/dist/bench/run-dir.d.ts +101 -0
- package/dist/bench/run-dir.js +379 -0
- package/dist/bench/runner.d.ts +80 -0
- package/dist/bench/runner.js +160 -0
- package/dist/bench/score.d.ts +30 -0
- package/dist/bench/score.js +38 -0
- package/dist/bench/stats.d.ts +51 -0
- package/dist/bench/stats.js +83 -0
- package/dist/bench/suite.d.ts +87 -0
- package/dist/bench/suite.js +286 -0
- package/dist/cli/bench.d.ts +32 -0
- package/dist/cli/bench.js +463 -0
- package/dist/cli/io.d.ts +45 -0
- package/dist/cli/io.js +45 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +17 -67
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +20 -11
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.js +47 -14
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +25 -0
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +8 -3
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +28 -4
- package/package.json +2 -2
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring a review against a suite case: whether its findings catch the case's bug, and how many kept
|
|
3
|
+
* findings catch nothing. Pure code over a result and a case, so a record can be scored again later.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
|
|
7
|
+
* line from `window` lines before the location's start to `window` lines after its end. The window is
|
|
8
|
+
* taken as given, 0 included.
|
|
9
|
+
*/
|
|
10
|
+
export function catches(finding, location, window) {
|
|
11
|
+
return finding.file === location.file && finding.line >= location.start - window && finding.line <= location.end + window;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Score a review against a case. A bug is caught by a kept finding (`keptCatch`) or by any finding the
|
|
15
|
+
* finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window` lines of one of its
|
|
16
|
+
* locations. A control has nothing to catch, so every kept finding is unmatched. Every profile of the
|
|
17
|
+
* review counts, for the catches and for the gaps alike.
|
|
18
|
+
*/
|
|
19
|
+
export function scoreReview(result, suiteCase, window) {
|
|
20
|
+
const incomplete = result.unreviewed.length > 0 || result.meta.incomplete.length > 0;
|
|
21
|
+
if (suiteCase.kind === 'control') {
|
|
22
|
+
return {
|
|
23
|
+
keptCatch: false,
|
|
24
|
+
proposedCatch: false,
|
|
25
|
+
unmatchedKept: result.kept.length,
|
|
26
|
+
incomplete,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
const hits = (verdict) => suiteCase.locations.some(location => catches(verdict, location, window));
|
|
30
|
+
const keptCatch = result.kept.some(hits);
|
|
31
|
+
const proposedCatch = keptCatch || result.discarded.some(hits) || result.unreviewed.some(hits);
|
|
32
|
+
return {
|
|
33
|
+
keptCatch,
|
|
34
|
+
proposedCatch,
|
|
35
|
+
unmatchedKept: result.kept.filter(verdict => !hits(verdict)).length,
|
|
36
|
+
incomplete,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The statistics of the report, written by hand and held to reference values: the Wilson score
|
|
3
|
+
* interval of a rate, the exact two-sided sign test and nearest-rank percentiles. Pure code, with no
|
|
4
|
+
* package behind it; each function throws a `RangeError` for an argument outside its domain.
|
|
5
|
+
*/
|
|
6
|
+
/** The two-sided 95% quantile of the standard normal distribution. */
|
|
7
|
+
export declare const Z95 = 1.959963984540054;
|
|
8
|
+
/** A confidence interval of a rate, inside 0 and 1. */
|
|
9
|
+
export interface Interval {
|
|
10
|
+
readonly low: number;
|
|
11
|
+
readonly high: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
|
|
15
|
+
* With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
|
|
16
|
+
* `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
|
|
17
|
+
* lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
|
|
18
|
+
* reach to within a rounding error.
|
|
19
|
+
*
|
|
20
|
+
* `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
|
|
21
|
+
* fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
|
|
22
|
+
*
|
|
23
|
+
* Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
|
|
24
|
+
* and 1 gives [0.7225, 1].
|
|
25
|
+
*
|
|
26
|
+
* @throws {RangeError} when `n` is not a positive integer, or `rate` is outside [0, 1].
|
|
27
|
+
*/
|
|
28
|
+
export declare function wilson(rate: number, n: number, z?: number): Interval;
|
|
29
|
+
/**
|
|
30
|
+
* The exact two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
|
|
31
|
+
* `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
|
|
32
|
+
* and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
|
|
33
|
+
* (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
|
|
34
|
+
* `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number` for any `n` and
|
|
35
|
+
* the reference values come out exact.
|
|
36
|
+
*
|
|
37
|
+
* Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
|
|
38
|
+
* 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
|
|
39
|
+
*
|
|
40
|
+
* @throws {RangeError} when a count is not an integer of at least 0.
|
|
41
|
+
*/
|
|
42
|
+
export declare function signTest(better: number, worse: number): number;
|
|
43
|
+
/**
|
|
44
|
+
* The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
|
|
45
|
+
* ascending, the product taken first so the rank stays an integer. The values are not changed.
|
|
46
|
+
*
|
|
47
|
+
* Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
|
|
48
|
+
*
|
|
49
|
+
* @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
|
|
50
|
+
*/
|
|
51
|
+
export declare function nearestRank(values: readonly number[], percent: number): number;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The statistics of the report, written by hand and held to reference values: the Wilson score
|
|
3
|
+
* interval of a rate, the exact two-sided sign test and nearest-rank percentiles. Pure code, with no
|
|
4
|
+
* package behind it; each function throws a `RangeError` for an argument outside its domain.
|
|
5
|
+
*/
|
|
6
|
+
/** The two-sided 95% quantile of the standard normal distribution. */
|
|
7
|
+
export const Z95 = 1.959963984540054;
|
|
8
|
+
/**
|
|
9
|
+
* The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
|
|
10
|
+
* With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
|
|
11
|
+
* `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
|
|
12
|
+
* lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
|
|
13
|
+
* reach to within a rounding error.
|
|
14
|
+
*
|
|
15
|
+
* `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
|
|
16
|
+
* fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
|
|
17
|
+
*
|
|
18
|
+
* Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
|
|
19
|
+
* and 1 gives [0.7225, 1].
|
|
20
|
+
*
|
|
21
|
+
* @throws {RangeError} when `n` is not a positive integer, or `rate` is outside [0, 1].
|
|
22
|
+
*/
|
|
23
|
+
export function wilson(rate, n, z = Z95) {
|
|
24
|
+
if (!Number.isInteger(n) || n < 1)
|
|
25
|
+
throw new RangeError('the number of cases must be a positive integer');
|
|
26
|
+
if (!(rate >= 0 && rate <= 1))
|
|
27
|
+
throw new RangeError('the rate must be between 0 and 1');
|
|
28
|
+
const d = 1 + z * z / n;
|
|
29
|
+
const center = (rate + z * z / (2 * n)) / d;
|
|
30
|
+
const halfWidth = z / d * Math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n));
|
|
31
|
+
const low = rate === 0
|
|
32
|
+
? 0
|
|
33
|
+
: Math.max(0, center - halfWidth);
|
|
34
|
+
const high = rate === 1
|
|
35
|
+
? 1
|
|
36
|
+
: Math.min(1, center + halfWidth);
|
|
37
|
+
return { low, high };
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The exact two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
|
|
41
|
+
* `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
|
|
42
|
+
* and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
|
|
43
|
+
* (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
|
|
44
|
+
* `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number` for any `n` and
|
|
45
|
+
* the reference values come out exact.
|
|
46
|
+
*
|
|
47
|
+
* Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
|
|
48
|
+
* 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
|
|
49
|
+
*
|
|
50
|
+
* @throws {RangeError} when a count is not an integer of at least 0.
|
|
51
|
+
*/
|
|
52
|
+
export function signTest(better, worse) {
|
|
53
|
+
if (!Number.isInteger(better) || better < 0 || !Number.isInteger(worse) || worse < 0)
|
|
54
|
+
throw new RangeError('a count must be a whole number of at least 0');
|
|
55
|
+
const n = better + worse;
|
|
56
|
+
if (n === 0)
|
|
57
|
+
return 1;
|
|
58
|
+
const k = Math.min(better, worse);
|
|
59
|
+
let coefficient = 1n;
|
|
60
|
+
let sum = 1n;
|
|
61
|
+
for (let i = 1; i <= k; i++) {
|
|
62
|
+
coefficient = coefficient * BigInt(n - i + 1) / BigInt(i);
|
|
63
|
+
sum += coefficient;
|
|
64
|
+
}
|
|
65
|
+
const shift = BigInt(Math.max(0, n + 1 - 1000));
|
|
66
|
+
return Math.min(1, Number((2n * sum) >> shift) / Number((1n << BigInt(n)) >> shift));
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
|
|
70
|
+
* ascending, the product taken first so the rank stays an integer. The values are not changed.
|
|
71
|
+
*
|
|
72
|
+
* Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
|
|
73
|
+
*
|
|
74
|
+
* @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
|
|
75
|
+
*/
|
|
76
|
+
export function nearestRank(values, percent) {
|
|
77
|
+
if (values.length === 0)
|
|
78
|
+
throw new RangeError('there must be at least one value');
|
|
79
|
+
if (!(percent > 0 && percent <= 100))
|
|
80
|
+
throw new RangeError('the percent must be above 0 and at most 100');
|
|
81
|
+
const rank = Math.max(1, Math.ceil(percent * values.length / 100));
|
|
82
|
+
return [...values].sort((a, b) => a - b)[rank - 1];
|
|
83
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
|
|
3
|
+
* patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
|
|
4
|
+
* file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
|
|
5
|
+
* YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
|
|
6
|
+
* deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
|
|
7
|
+
* `cases/` and every patch must not be a symbolic link.
|
|
8
|
+
*
|
|
9
|
+
* Everything is checked before any call, and no message quotes the file or a patch.
|
|
10
|
+
*/
|
|
11
|
+
/** A suite name or a case id: a name for a directory and for a record, so it holds no separator and no `..`. */
|
|
12
|
+
export declare const SUITE_ID: RegExp;
|
|
13
|
+
/** A patch's file name: bare, and not starting with a dot. */
|
|
14
|
+
export declare const PATCH_NAME: RegExp;
|
|
15
|
+
/** The most `suite.yaml` may hold, in bytes. */
|
|
16
|
+
export declare const MAX_SUITE_BYTES: number;
|
|
17
|
+
export declare const MAX_CASES = 1000;
|
|
18
|
+
export declare const MAX_LOCATIONS = 32;
|
|
19
|
+
/** The most lines one location may span. */
|
|
20
|
+
export declare const MAX_RANGE_LINES = 500;
|
|
21
|
+
/** The most characters of a note or of one copyright line. */
|
|
22
|
+
export declare const MAX_TEXT = 500;
|
|
23
|
+
export declare const MAX_COPYRIGHT_LINES = 20;
|
|
24
|
+
export declare const MAX_TRIALS = 100;
|
|
25
|
+
export declare const MAX_WINDOW = 100;
|
|
26
|
+
/** How many lines either side of a location still count as a catch, when the suite does not say. */
|
|
27
|
+
export declare const DEFAULT_WINDOW = 3;
|
|
28
|
+
/** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
|
|
29
|
+
export declare const MAX_PATCH_TOTAL_BYTES: number;
|
|
30
|
+
/** A span of lines of the new file that holds a bug, at the file's own path in the patch. */
|
|
31
|
+
export interface Location {
|
|
32
|
+
readonly file: string;
|
|
33
|
+
readonly start: number;
|
|
34
|
+
readonly end: number;
|
|
35
|
+
}
|
|
36
|
+
/** What a case says about where it comes from, whatever its kind. */
|
|
37
|
+
interface CaseOrigin {
|
|
38
|
+
readonly id: string;
|
|
39
|
+
/** The name of its patch in `cases/`. */
|
|
40
|
+
readonly patch: string;
|
|
41
|
+
readonly repo?: string;
|
|
42
|
+
readonly license?: string;
|
|
43
|
+
readonly copyright?: readonly string[];
|
|
44
|
+
readonly date?: string;
|
|
45
|
+
readonly note?: string;
|
|
46
|
+
}
|
|
47
|
+
/** A patch that introduces a vulnerability at its locations. */
|
|
48
|
+
export interface BugCase extends CaseOrigin {
|
|
49
|
+
readonly kind: 'bug';
|
|
50
|
+
readonly locations: readonly Location[];
|
|
51
|
+
readonly fixCommit?: string;
|
|
52
|
+
readonly introducingCommit?: string;
|
|
53
|
+
}
|
|
54
|
+
/** A patch that introduces none. */
|
|
55
|
+
export interface ControlCase extends CaseOrigin {
|
|
56
|
+
readonly kind: 'control';
|
|
57
|
+
readonly commit?: string;
|
|
58
|
+
}
|
|
59
|
+
export type SuiteCase = BugCase | ControlCase;
|
|
60
|
+
/** A suite once read and checked. */
|
|
61
|
+
export interface Suite {
|
|
62
|
+
readonly name: string;
|
|
63
|
+
/** How many trials a run takes when it is not told, if the suite says. */
|
|
64
|
+
readonly trials?: number;
|
|
65
|
+
/** How many lines either side of a location still count as a catch. */
|
|
66
|
+
readonly window: number;
|
|
67
|
+
readonly cases: readonly SuiteCase[];
|
|
68
|
+
/** The decoded text of each patch by name, in the order the cases first name them. */
|
|
69
|
+
readonly patches: ReadonlyMap<string, string>;
|
|
70
|
+
/** `sha256:` and the hex SHA-256 of the files' own hashes: `suite.yaml` and each patch a case names. */
|
|
71
|
+
readonly hash: string;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
|
|
75
|
+
* patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
|
|
76
|
+
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
77
|
+
* each location of a bug must name a file of its patch by the diff's own path, and each of its lines
|
|
78
|
+
* must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
|
|
79
|
+
* `suite.yaml` and each patch a case names, once.
|
|
80
|
+
*
|
|
81
|
+
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
82
|
+
* `<label>/cases`; no issue quotes a file.
|
|
83
|
+
*
|
|
84
|
+
* @throws ConfigError with the first problem of `suite.yaml`, or `cases/`, or every other issue found
|
|
85
|
+
*/
|
|
86
|
+
export declare function loadSuite(directory: string, label?: string): Promise<Suite>;
|
|
87
|
+
export {};
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
|
|
3
|
+
* patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
|
|
4
|
+
* file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
|
|
5
|
+
* YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
|
|
6
|
+
* deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
|
|
7
|
+
* `cases/` and every patch must not be a symbolic link.
|
|
8
|
+
*
|
|
9
|
+
* Everything is checked before any call, and no message quotes the file or a patch.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { lstat } from 'node:fs/promises';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { z } from 'zod';
|
|
15
|
+
import { dataIssues, decode, isPlainObject, parseYaml, readBounded } from "../config/load.js";
|
|
16
|
+
import { configPath, parseConfig } from "../config/schema.js";
|
|
17
|
+
import { decodeDiff, locate, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
|
|
18
|
+
import { ConfigError } from "../core/errors.js";
|
|
19
|
+
import { hiddenLabels } from "../core/sanitize.js";
|
|
20
|
+
/** A suite name or a case id: a name for a directory and for a record, so it holds no separator and no `..`. */
|
|
21
|
+
export const SUITE_ID = /^[a-z0-9][a-z0-9._-]*$/;
|
|
22
|
+
/** A patch's file name: bare, and not starting with a dot. */
|
|
23
|
+
export const PATCH_NAME = /^[A-Z0-9][\w.-]*$/i;
|
|
24
|
+
/** The most `suite.yaml` may hold, in bytes. */
|
|
25
|
+
export const MAX_SUITE_BYTES = 1024 * 1024;
|
|
26
|
+
export const MAX_CASES = 1000;
|
|
27
|
+
export const MAX_LOCATIONS = 32;
|
|
28
|
+
/** The most lines one location may span. */
|
|
29
|
+
export const MAX_RANGE_LINES = 500;
|
|
30
|
+
/** The most characters of a note or of one copyright line. */
|
|
31
|
+
export const MAX_TEXT = 500;
|
|
32
|
+
export const MAX_COPYRIGHT_LINES = 20;
|
|
33
|
+
export const MAX_TRIALS = 100;
|
|
34
|
+
export const MAX_WINDOW = 100;
|
|
35
|
+
/** How many lines either side of a location still count as a catch, when the suite does not say. */
|
|
36
|
+
export const DEFAULT_WINDOW = 3;
|
|
37
|
+
/** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
|
|
38
|
+
export const MAX_PATCH_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
39
|
+
const MAX_ID = 64;
|
|
40
|
+
const MAX_PATCH_NAME = 128;
|
|
41
|
+
const MAX_FILE = 4096;
|
|
42
|
+
const MAX_REPO = 200;
|
|
43
|
+
/** A full commit hash: SHA-1 (40 digits) or SHA-256 (64). */
|
|
44
|
+
const COMMIT = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
|
|
45
|
+
/** One SPDX license id, not an expression. */
|
|
46
|
+
const LICENSE = /^[A-Z0-9][A-Z0-9.+-]{0,63}$/i;
|
|
47
|
+
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
48
|
+
/** What every diff the bench reads starts with. */
|
|
49
|
+
const DIFF_START = 'diff --git ';
|
|
50
|
+
const sha256 = (data) => createHash('sha256').update(data).digest('hex');
|
|
51
|
+
/** Whether `text` is a real day written YYYY-MM-DD. */
|
|
52
|
+
function isDay(text) {
|
|
53
|
+
if (!DAY.test(text))
|
|
54
|
+
return false;
|
|
55
|
+
const time = Date.parse(`${text}T00:00:00Z`);
|
|
56
|
+
return !Number.isNaN(time) && new Date(time).toISOString().startsWith(text);
|
|
57
|
+
}
|
|
58
|
+
/** Whether `text` is an https:// URL with no credentials, query or fragment. */
|
|
59
|
+
function isRepository(text) {
|
|
60
|
+
if (text.length > MAX_REPO || /[\s?#]/u.test(text) || hiddenLabels(text).length > 0 || !URL.canParse(text))
|
|
61
|
+
return false;
|
|
62
|
+
const url = new URL(text);
|
|
63
|
+
return url.protocol === 'https:' && url.username === '' && url.password === '';
|
|
64
|
+
}
|
|
65
|
+
const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
|
|
66
|
+
const textSchema = z.string().min(1).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
|
|
67
|
+
const commitSchema = z.string().regex(COMMIT, { error: 'must be a full commit hash: 40 or 64 lowercase hex digits' });
|
|
68
|
+
const locationSchema = z.strictObject({
|
|
69
|
+
file: z.string().min(1, { error: 'must not be empty' }).max(MAX_FILE, { error: `must be at most ${MAX_FILE} characters` }),
|
|
70
|
+
start: z.int().min(1),
|
|
71
|
+
end: z.int().min(1),
|
|
72
|
+
})
|
|
73
|
+
.refine(({ start, end }) => start <= end, { error: 'must start at or before its end' })
|
|
74
|
+
.refine(({ start, end }) => end - start < MAX_RANGE_LINES, { error: `must span at most ${MAX_RANGE_LINES} lines` });
|
|
75
|
+
/** A case of either kind; the rules of each kind are checked after the fields, and 7b adds `attack` to the kinds and its rules there. */
|
|
76
|
+
const caseSchema = z.strictObject({
|
|
77
|
+
id: idSchema,
|
|
78
|
+
kind: z.enum(['bug', 'control']),
|
|
79
|
+
patch: z.string().max(MAX_PATCH_NAME, { error: `must be at most ${MAX_PATCH_NAME} characters` }).regex(PATCH_NAME, { error: 'must be a bare file name: letters, digits, dots, underscores and dashes, not starting with a dot' }),
|
|
80
|
+
locations: z.array(locationSchema).min(1).max(MAX_LOCATIONS, { error: `must hold at most ${MAX_LOCATIONS} locations` }).optional(),
|
|
81
|
+
repo: z.string().refine(isRepository, { error: 'must be an https:// URL with no credentials, query or fragment' }).optional(),
|
|
82
|
+
fixCommit: commitSchema.optional(),
|
|
83
|
+
introducingCommit: commitSchema.optional(),
|
|
84
|
+
commit: commitSchema.optional(),
|
|
85
|
+
license: z.string().regex(LICENSE, { error: 'must be one SPDX license id, such as MIT or Apache-2.0' }).optional(),
|
|
86
|
+
copyright: z.array(textSchema).min(1).max(MAX_COPYRIGHT_LINES, { error: `must hold at most ${MAX_COPYRIGHT_LINES} lines` }).optional(),
|
|
87
|
+
date: z.string().refine(isDay, { error: 'must be a date written YYYY-MM-DD' }).optional(),
|
|
88
|
+
note: textSchema.optional(),
|
|
89
|
+
}).superRefine((entry, context) => {
|
|
90
|
+
const only = (field, kind) => {
|
|
91
|
+
if (entry[field] !== undefined) {
|
|
92
|
+
context.addIssue({
|
|
93
|
+
code: 'custom',
|
|
94
|
+
path: [field],
|
|
95
|
+
message: `is only for a ${kind}`,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
if (entry.kind === 'bug') {
|
|
100
|
+
if (entry.locations === undefined) {
|
|
101
|
+
context.addIssue({
|
|
102
|
+
code: 'custom',
|
|
103
|
+
path: ['locations'],
|
|
104
|
+
message: 'is required for a bug',
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
only('commit', 'control');
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
only('locations', 'bug');
|
|
111
|
+
only('fixCommit', 'bug');
|
|
112
|
+
only('introducingCommit', 'bug');
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
const suiteSchema = z.strictObject({
|
|
116
|
+
name: idSchema,
|
|
117
|
+
trials: z.int().min(1).max(MAX_TRIALS).optional(),
|
|
118
|
+
window: z.int().min(0).max(MAX_WINDOW).default(DEFAULT_WINDOW),
|
|
119
|
+
cases: z.array(caseSchema).min(1).max(MAX_CASES, { error: `must hold at most ${MAX_CASES} cases` }),
|
|
120
|
+
});
|
|
121
|
+
function fail(path, message) {
|
|
122
|
+
throw new ConfigError([
|
|
123
|
+
{ path, message },
|
|
124
|
+
]);
|
|
125
|
+
}
|
|
126
|
+
/** A patch's bytes decoded as the review command decodes a diff, and checked to be a pure diff of at least one file quorum reads. */
|
|
127
|
+
async function checkPatch(bytes) {
|
|
128
|
+
const text = await decodeDiff([bytes]);
|
|
129
|
+
if (text === undefined)
|
|
130
|
+
return { problem: 'is over 16 MiB' };
|
|
131
|
+
if (!text.startsWith(DIFF_START))
|
|
132
|
+
return { problem: 'must be a pure diff that starts at its first diff --git line' };
|
|
133
|
+
const sections = splitDiff(text);
|
|
134
|
+
if (sections.size === 0)
|
|
135
|
+
return { problem: 'names no file in a diff --git header quorum reads' };
|
|
136
|
+
return {
|
|
137
|
+
text,
|
|
138
|
+
sections,
|
|
139
|
+
hash: sha256(bytes),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
/** The problem `cases/` has, or undefined when it is a directory and no link. */
|
|
143
|
+
async function casesProblem(directory) {
|
|
144
|
+
try {
|
|
145
|
+
const stats = await lstat(join(directory, 'cases'));
|
|
146
|
+
if (stats.isSymbolicLink())
|
|
147
|
+
return 'is a symbolic link';
|
|
148
|
+
return stats.isDirectory()
|
|
149
|
+
? undefined
|
|
150
|
+
: 'is not a directory';
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
const code = error.code;
|
|
154
|
+
return code === 'ENOENT' || code === 'ENOTDIR'
|
|
155
|
+
? 'does not exist'
|
|
156
|
+
: 'cannot be read';
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
|
|
161
|
+
* patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
|
|
162
|
+
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
163
|
+
* each location of a bug must name a file of its patch by the diff's own path, and each of its lines
|
|
164
|
+
* must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
|
|
165
|
+
* `suite.yaml` and each patch a case names, once.
|
|
166
|
+
*
|
|
167
|
+
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
168
|
+
* `<label>/cases`; no issue quotes a file.
|
|
169
|
+
*
|
|
170
|
+
* @throws ConfigError with the first problem of `suite.yaml`, or `cases/`, or every other issue found
|
|
171
|
+
*/
|
|
172
|
+
export async function loadSuite(directory, label = directory) {
|
|
173
|
+
const file = `${label}/suite.yaml`;
|
|
174
|
+
const bytes = await readBounded(join(directory, 'suite.yaml'), MAX_SUITE_BYTES, '1 MiB', { noFollow: true });
|
|
175
|
+
if ('problem' in bytes)
|
|
176
|
+
fail(file, bytes.problem);
|
|
177
|
+
const text = decode(bytes.value);
|
|
178
|
+
if ('problem' in text)
|
|
179
|
+
fail(file, text.problem);
|
|
180
|
+
const parsed = parseYaml(text.value);
|
|
181
|
+
if ('problem' in parsed)
|
|
182
|
+
fail(file, parsed.problem);
|
|
183
|
+
if (!isPlainObject(parsed.value))
|
|
184
|
+
fail(file, 'must hold a mapping');
|
|
185
|
+
const data = dataIssues(parsed.value, file);
|
|
186
|
+
if (data.length > 0)
|
|
187
|
+
throw new ConfigError(data);
|
|
188
|
+
const header = parseConfig(suiteSchema, parsed.value, file);
|
|
189
|
+
const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
|
|
190
|
+
const issues = [];
|
|
191
|
+
const firstWith = new Map();
|
|
192
|
+
for (const [index, { id }] of header.cases.entries()) {
|
|
193
|
+
const first = firstWith.get(id);
|
|
194
|
+
if (first === undefined) {
|
|
195
|
+
firstWith.set(id, index);
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
issues.push({
|
|
199
|
+
path: at(index, 'id'),
|
|
200
|
+
message: `repeats the id of cases.${first}`,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const casesIssue = await casesProblem(directory);
|
|
205
|
+
if (casesIssue !== undefined) {
|
|
206
|
+
throw new ConfigError([
|
|
207
|
+
...issues,
|
|
208
|
+
{ path: `${label}/cases`, message: casesIssue },
|
|
209
|
+
]);
|
|
210
|
+
}
|
|
211
|
+
const patches = new Map();
|
|
212
|
+
const problems = new Map();
|
|
213
|
+
let total = 0;
|
|
214
|
+
for (const [index, { patch }] of header.cases.entries()) {
|
|
215
|
+
if (!patches.has(patch) && !problems.has(patch)) {
|
|
216
|
+
const patchBytes = await readBounded(join(directory, 'cases', patch), MAX_DIFF_BYTES, '16 MiB', { noFollow: true });
|
|
217
|
+
total += 'problem' in patchBytes
|
|
218
|
+
? patchBytes.counted ?? 0
|
|
219
|
+
: patchBytes.value.length;
|
|
220
|
+
if (total > MAX_PATCH_TOTAL_BYTES) {
|
|
221
|
+
issues.push({
|
|
222
|
+
path: at(index, 'patch'),
|
|
223
|
+
message: 'takes the patches over 64 MiB in total',
|
|
224
|
+
});
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
const read = 'problem' in patchBytes
|
|
228
|
+
? patchBytes
|
|
229
|
+
: await checkPatch(patchBytes.value);
|
|
230
|
+
if ('problem' in read)
|
|
231
|
+
problems.set(patch, read.problem);
|
|
232
|
+
else
|
|
233
|
+
patches.set(patch, read);
|
|
234
|
+
}
|
|
235
|
+
const problem = problems.get(patch);
|
|
236
|
+
if (problem !== undefined) {
|
|
237
|
+
issues.push({
|
|
238
|
+
path: at(index, 'patch'),
|
|
239
|
+
message: problem,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
for (const [index, entry] of header.cases.entries()) {
|
|
244
|
+
const read = patches.get(entry.patch);
|
|
245
|
+
if (entry.kind !== 'bug' || read === undefined)
|
|
246
|
+
continue;
|
|
247
|
+
for (const [position, { file: name, start, end }] of (entry.locations ?? []).entries()) {
|
|
248
|
+
if (!read.sections.has(name)) {
|
|
249
|
+
issues.push({
|
|
250
|
+
path: at(index, 'locations', position, 'file'),
|
|
251
|
+
message: 'is not a file of the patch, as its diff --git header names it',
|
|
252
|
+
});
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
for (let line = start; line <= end; line++) {
|
|
256
|
+
if (locate(read.sections, name, line) !== undefined)
|
|
257
|
+
continue;
|
|
258
|
+
issues.push({
|
|
259
|
+
path: at(index, 'locations', position),
|
|
260
|
+
message: `line ${line} is not in a hunk of the patch`,
|
|
261
|
+
});
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (issues.length > 0)
|
|
267
|
+
throw new ConfigError(issues);
|
|
268
|
+
const files = [
|
|
269
|
+
[
|
|
270
|
+
'suite.yaml',
|
|
271
|
+
sha256(bytes.value),
|
|
272
|
+
],
|
|
273
|
+
...[...patches].sort(([first], [second]) => first < second
|
|
274
|
+
? -1
|
|
275
|
+
: 1).map(([name, { hash }]) => [`cases/${name}`, hash]),
|
|
276
|
+
];
|
|
277
|
+
return {
|
|
278
|
+
name: header.name,
|
|
279
|
+
trials: header.trials,
|
|
280
|
+
window: header.window,
|
|
281
|
+
// The kind rules of `caseSchema` make each case a `BugCase` or a `ControlCase`.
|
|
282
|
+
cases: header.cases,
|
|
283
|
+
patches: new Map([...patches].map(([name, { text: patchText }]) => [name, patchText])),
|
|
284
|
+
hash: `sha256:${sha256(JSON.stringify(files))}`,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `quorum bench` subcommand: it reviews every case of a suite with every configuration, `--trials`
|
|
3
|
+
* times each, records each review in a run directory it can resume, and reports what each configuration
|
|
4
|
+
* caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
|
|
5
|
+
*/
|
|
6
|
+
import type { CliIO } from './io.ts';
|
|
7
|
+
export declare const BENCH_USAGE: string;
|
|
8
|
+
/**
|
|
9
|
+
* Run `quorum bench` with its arguments (those after `bench`) and return the exit code. Never throws.
|
|
10
|
+
*
|
|
11
|
+
* It reviews every case of the suite the argument names with every `--config` (the usual resolution when
|
|
12
|
+
* there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
|
|
13
|
+
* in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
|
|
14
|
+
* at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
|
|
15
|
+
* stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
|
|
16
|
+
* on stdout, which it also writes, with `report.json`, into the run directory.
|
|
17
|
+
*
|
|
18
|
+
* A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
|
|
19
|
+
* runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
|
|
20
|
+
* warning. A run directory of another suite, another quorum or other configurations, one that holds files
|
|
21
|
+
* of another kind, and a suite or a configuration that does not validate or cannot be built are refused
|
|
22
|
+
* before the first call.
|
|
23
|
+
*
|
|
24
|
+
* Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
|
|
25
|
+
* directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
|
|
26
|
+
* others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
|
|
27
|
+
* written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
|
|
28
|
+
* first review starts leaves the run directory untouched, even one that does not exist yet.
|
|
29
|
+
*
|
|
30
|
+
* Every line it writes is inert (invariant I3).
|
|
31
|
+
*/
|
|
32
|
+
export declare function runBench(io: CliIO, argv: readonly string[]): Promise<number>;
|