@tobi-or-not/quorum 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +37 -0
- package/dist/bench/fingerprint.js +80 -0
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +220 -0
- package/dist/bench/records.js +184 -0
- package/dist/bench/report.d.ts +143 -0
- package/dist/bench/report.js +538 -0
- package/dist/bench/run-dir.d.ts +127 -0
- package/dist/bench/run-dir.js +458 -0
- package/dist/bench/runner.d.ts +92 -0
- package/dist/bench/runner.js +195 -0
- package/dist/bench/score.d.ts +61 -0
- package/dist/bench/score.js +85 -0
- package/dist/bench/stats.d.ts +105 -0
- package/dist/bench/stats.js +158 -0
- package/dist/bench/suite.d.ts +132 -0
- package/dist/bench/suite.js +454 -0
- package/dist/cli/bench.d.ts +43 -0
- package/dist/cli/bench.js +537 -0
- package/dist/cli/io.d.ts +52 -0
- package/dist/cli/io.js +63 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +18 -85
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +20 -11
- package/dist/config/load.d.ts +46 -0
- package/dist/config/load.js +56 -14
- package/dist/core/diff.d.ts +20 -0
- package/dist/core/diff.js +102 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +8 -3
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +28 -4
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +2 -2
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
|
|
3
|
+
* patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
|
|
4
|
+
* file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
|
|
5
|
+
* YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
|
|
6
|
+
* deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
|
|
7
|
+
* `cases/` and every patch must not be a symbolic link.
|
|
8
|
+
*
|
|
9
|
+
* Everything is checked before any call, and no message quotes the file or a patch.
|
|
10
|
+
*/
|
|
11
|
+
import { z } from 'zod';
|
|
12
|
+
/** A suite name or a case id: a name for a directory and for a record, so it is never `.` or `..`, and holds no separator. */
|
|
13
|
+
export declare const SUITE_ID: RegExp;
|
|
14
|
+
/** A patch's file name: bare, and not starting with a dot. */
|
|
15
|
+
export declare const PATCH_NAME: RegExp;
|
|
16
|
+
/** The most `suite.yaml` may hold, in bytes. */
|
|
17
|
+
export declare const MAX_SUITE_BYTES: number;
|
|
18
|
+
export declare const MAX_CASES = 1000;
|
|
19
|
+
export declare const MAX_LOCATIONS = 32;
|
|
20
|
+
/** The most lines one location may span. */
|
|
21
|
+
export declare const MAX_RANGE_LINES = 500;
|
|
22
|
+
/** The most characters of a note or of one copyright line. */
|
|
23
|
+
export declare const MAX_TEXT = 500;
|
|
24
|
+
export declare const MAX_COPYRIGHT_LINES = 20;
|
|
25
|
+
export declare const MAX_TRIALS = 100;
|
|
26
|
+
export declare const MAX_WINDOW = 100;
|
|
27
|
+
/** How many lines either side of a location still count as a catch, when the suite does not say. */
|
|
28
|
+
export declare const DEFAULT_WINDOW = 3;
|
|
29
|
+
/** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
|
|
30
|
+
export declare const MAX_PATCH_TOTAL_BYTES: number;
|
|
31
|
+
/** The most characters of a suite name or a case id. */
|
|
32
|
+
export declare const MAX_ID = 64;
|
|
33
|
+
/** What an attack case tries to do to the review of its twin, in the security model's order. */
|
|
34
|
+
export declare const GOALS: readonly ["suppression", "forced-keep", "cost-bloat", "denial-of-review"];
|
|
35
|
+
/** Where an attack case plants its text in the patch. */
|
|
36
|
+
export declare const CARRIERS: readonly ["comment", "string-literal", "file-name", "documentation", "guideline-file"];
|
|
37
|
+
/** How an attack case writes its text. */
|
|
38
|
+
export declare const ENCODINGS: readonly ["plain", "multilingual", "hidden-unicode", "base64", "html-comment", "fake-delimiter"];
|
|
39
|
+
export type Goal = typeof GOALS[number];
|
|
40
|
+
export type Carrier = typeof CARRIERS[number];
|
|
41
|
+
export type Encoding = typeof ENCODINGS[number];
|
|
42
|
+
/** A span of lines of the new file, at the file's own path in the patch: where a bug is, or a decoy. */
|
|
43
|
+
export interface Location {
|
|
44
|
+
readonly file: string;
|
|
45
|
+
readonly start: number;
|
|
46
|
+
readonly end: number;
|
|
47
|
+
}
|
|
48
|
+
/** What a case says about where it comes from, whatever its kind. */
|
|
49
|
+
interface CaseOrigin {
|
|
50
|
+
readonly id: string;
|
|
51
|
+
/** The name of its patch in `cases/`. */
|
|
52
|
+
readonly patch: string;
|
|
53
|
+
readonly repo?: string;
|
|
54
|
+
readonly license?: string;
|
|
55
|
+
readonly copyright?: readonly string[];
|
|
56
|
+
readonly date?: string;
|
|
57
|
+
readonly note?: string;
|
|
58
|
+
}
|
|
59
|
+
/** A patch that introduces a vulnerability at its locations. */
|
|
60
|
+
export interface BugCase extends CaseOrigin {
|
|
61
|
+
readonly kind: 'bug';
|
|
62
|
+
readonly locations: readonly Location[];
|
|
63
|
+
readonly fixCommit?: string;
|
|
64
|
+
readonly introducingCommit?: string;
|
|
65
|
+
}
|
|
66
|
+
/** A patch that introduces none. */
|
|
67
|
+
export interface ControlCase extends CaseOrigin {
|
|
68
|
+
readonly kind: 'control';
|
|
69
|
+
readonly commit?: string;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* The patch of a bug case with text planted in it to turn its review: the case's twin. Its patch is
|
|
73
|
+
* the twin's own code, so it carries no commit: the twin holds them.
|
|
74
|
+
*/
|
|
75
|
+
export interface AttackCase extends CaseOrigin {
|
|
76
|
+
readonly kind: 'attack';
|
|
77
|
+
/** The id of the bug case of the suite whose patch this one poisons. */
|
|
78
|
+
readonly twin: string;
|
|
79
|
+
readonly goal: Goal;
|
|
80
|
+
readonly carrier: Carrier;
|
|
81
|
+
readonly encoding: Encoding;
|
|
82
|
+
/** Where the twin's bug is in this patch, whose lines the planted text may have moved. */
|
|
83
|
+
readonly locations: readonly Location[];
|
|
84
|
+
/** A forced keep's bait: lines of this patch a finding must not keep. */
|
|
85
|
+
readonly decoy?: Location;
|
|
86
|
+
/** The same code's lines in the twin's patch, so the twin's review is checked for the same finding. */
|
|
87
|
+
readonly twinDecoy?: Location;
|
|
88
|
+
}
|
|
89
|
+
export type SuiteCase = BugCase | ControlCase | AttackCase;
|
|
90
|
+
/** A suite once read and checked. */
|
|
91
|
+
export interface Suite {
|
|
92
|
+
readonly name: string;
|
|
93
|
+
/** How many trials a run takes when it is not told, if the suite says. */
|
|
94
|
+
readonly trials?: number;
|
|
95
|
+
/** How many lines either side of a location still count as a catch. */
|
|
96
|
+
readonly window: number;
|
|
97
|
+
readonly cases: readonly SuiteCase[];
|
|
98
|
+
/** The decoded text of each patch by name, in the order the cases first name them. */
|
|
99
|
+
readonly patches: ReadonlyMap<string, string>;
|
|
100
|
+
/**
|
|
101
|
+
* `sha256:` and the hex SHA-256 of what the reviews of the suite depend on and which of them a run holds:
|
|
102
|
+
* its `trials`, and each case's id with the hex SHA-256 of the bytes of its patch. A change of it plans
|
|
103
|
+
* or sends other reviews.
|
|
104
|
+
*/
|
|
105
|
+
readonly reviewHash: string;
|
|
106
|
+
/**
|
|
107
|
+
* `sha256:` and the hex SHA-256 of the bytes of `suite.yaml`, so it moves with any byte of the file: the
|
|
108
|
+
* locations and the window a score reads, and a comment too.
|
|
109
|
+
*/
|
|
110
|
+
readonly scoringHash: string;
|
|
111
|
+
}
|
|
112
|
+
/** A suite name or a case id, as a schema: at most `MAX_ID` characters matching `SUITE_ID`. */
|
|
113
|
+
export declare const idSchema: z.ZodString;
|
|
114
|
+
/**
|
|
115
|
+
* Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
|
|
116
|
+
* patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
|
|
117
|
+
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
118
|
+
* each location of a bug or an attack, and an attack's decoy, must name a file of its patch by the
|
|
119
|
+
* diff's own path, and each of its lines must sit in a hunk of the patch; an attack's twin must be a
|
|
120
|
+
* bug case of the suite, its twin decoy must do the same against the twin's patch, and no decoy may
|
|
121
|
+
* be within twice the window of a location it could be caught with. A file of `cases/` that no case
|
|
122
|
+
* names is not read. The review hash covers the case ids, `trials` and the bytes of each patch a case
|
|
123
|
+
* names; the scoring hash covers the bytes of `suite.yaml`.
|
|
124
|
+
*
|
|
125
|
+
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
126
|
+
* `<label>/cases`; no issue quotes a file.
|
|
127
|
+
*
|
|
128
|
+
* @throws ConfigError with the first problem of the bytes, the text or the YAML of `suite.yaml`, or every
|
|
129
|
+
* issue its data has; or the repeated ids and the problem of `cases/`; or every other issue found
|
|
130
|
+
*/
|
|
131
|
+
export declare function loadSuite(directory: string, label?: string): Promise<Suite>;
|
|
132
|
+
export {};
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a bench suite: a directory holding `suite.yaml` and a `cases/` directory of patches, each
|
|
3
|
+
* patch a diff the bench reviews. A suite is data, never configuration or code, and is read as a config
|
|
4
|
+
* file is read (see `src/config/load.ts`): `suite.yaml` is a regular file of at most 1 MiB, UTF-8, one
|
|
5
|
+
* YAML 1.2 core-schema document with no repeated key, alias or YAML 1.1 tag, plain data at most 64 keys
|
|
6
|
+
* deep and 10 000 values, with no `__proto__` key. Nothing in a suite reaches outside it: `suite.yaml`,
|
|
7
|
+
* `cases/` and every patch must not be a symbolic link.
|
|
8
|
+
*
|
|
9
|
+
* Everything is checked before any call, and no message quotes the file or a patch.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { z } from 'zod';
|
|
14
|
+
import { dataIssues, decode, isPlainObject, lstatOf, parseYaml, readBounded } from "../config/load.js";
|
|
15
|
+
import { configPath, parseConfig } from "../config/schema.js";
|
|
16
|
+
import { decodeDiff, firstLineOutside, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
|
|
17
|
+
import { ConfigError } from "../core/errors.js";
|
|
18
|
+
import { hiddenLabels } from "../core/sanitize.js";
|
|
19
|
+
/** A suite name or a case id: a name for a directory and for a record, so it is never `.` or `..`, and holds no separator. */
|
|
20
|
+
export const SUITE_ID = /^[a-z0-9][a-z0-9._-]*$/;
|
|
21
|
+
/** A patch's file name: bare, and not starting with a dot. */
|
|
22
|
+
export const PATCH_NAME = /^[A-Z0-9][\w.-]*$/i;
|
|
23
|
+
/** The most `suite.yaml` may hold, in bytes. */
|
|
24
|
+
export const MAX_SUITE_BYTES = 1024 * 1024;
|
|
25
|
+
export const MAX_CASES = 1000;
|
|
26
|
+
export const MAX_LOCATIONS = 32;
|
|
27
|
+
/** The most lines one location may span. */
|
|
28
|
+
export const MAX_RANGE_LINES = 500;
|
|
29
|
+
/** The most characters of a note or of one copyright line. */
|
|
30
|
+
export const MAX_TEXT = 500;
|
|
31
|
+
export const MAX_COPYRIGHT_LINES = 20;
|
|
32
|
+
export const MAX_TRIALS = 100;
|
|
33
|
+
export const MAX_WINDOW = 100;
|
|
34
|
+
/** How many lines either side of a location still count as a catch, when the suite does not say. */
|
|
35
|
+
export const DEFAULT_WINDOW = 3;
|
|
36
|
+
/** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
|
|
37
|
+
export const MAX_PATCH_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
38
|
+
/** The most characters of a suite name or a case id. */
|
|
39
|
+
export const MAX_ID = 64;
|
|
40
|
+
const MAX_PATCH_NAME = 128;
|
|
41
|
+
const MAX_FILE = 4096;
|
|
42
|
+
const MAX_REPO = 200;
|
|
43
|
+
/** A full commit hash: SHA-1 (40 digits) or SHA-256 (64). */
|
|
44
|
+
const COMMIT = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
|
|
45
|
+
/** One SPDX license id, not an expression. */
|
|
46
|
+
const LICENSE = /^[A-Z0-9][A-Z0-9.+-]{0,63}$/i;
|
|
47
|
+
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
48
|
+
/** What every diff the bench reads starts with. */
|
|
49
|
+
const DIFF_START = 'diff --git ';
|
|
50
|
+
/** What an attack case tries to do to the review of its twin, in the security model's order. */
|
|
51
|
+
export const GOALS = [
|
|
52
|
+
'suppression',
|
|
53
|
+
'forced-keep',
|
|
54
|
+
'cost-bloat',
|
|
55
|
+
'denial-of-review',
|
|
56
|
+
];
|
|
57
|
+
/** Where an attack case plants its text in the patch. */
|
|
58
|
+
export const CARRIERS = [
|
|
59
|
+
'comment',
|
|
60
|
+
'string-literal',
|
|
61
|
+
'file-name',
|
|
62
|
+
'documentation',
|
|
63
|
+
'guideline-file',
|
|
64
|
+
];
|
|
65
|
+
/** How an attack case writes its text. */
|
|
66
|
+
export const ENCODINGS = [
|
|
67
|
+
'plain',
|
|
68
|
+
'multilingual',
|
|
69
|
+
'hidden-unicode',
|
|
70
|
+
'base64',
|
|
71
|
+
'html-comment',
|
|
72
|
+
'fake-delimiter',
|
|
73
|
+
];
|
|
74
|
+
const sha256 = (data) => createHash('sha256').update(data).digest('hex');
|
|
75
|
+
/** Whether `text` is a real day written YYYY-MM-DD. */
|
|
76
|
+
function isDay(text) {
|
|
77
|
+
if (!DAY.test(text))
|
|
78
|
+
return false;
|
|
79
|
+
const time = Date.parse(`${text}T00:00:00Z`);
|
|
80
|
+
return !Number.isNaN(time) && new Date(time).toISOString().startsWith(text);
|
|
81
|
+
}
|
|
82
|
+
/** Whether `text` is an https:// URL with no credentials, query or fragment. */
|
|
83
|
+
function isRepository(text) {
|
|
84
|
+
if (text.length > MAX_REPO || /[\s?#]/u.test(text) || hiddenLabels(text).length > 0 || !URL.canParse(text))
|
|
85
|
+
return false;
|
|
86
|
+
const url = new URL(text);
|
|
87
|
+
return url.protocol === 'https:' && url.username === '' && url.password === '';
|
|
88
|
+
}
|
|
89
|
+
/** A suite name or a case id, as a schema: at most `MAX_ID` characters matching `SUITE_ID`. */
|
|
90
|
+
export const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
|
|
91
|
+
const textSchema = z.string().min(1, { error: 'must not be empty' }).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
|
|
92
|
+
const commitSchema = z.string().regex(COMMIT, { error: 'must be a full commit hash: 40 or 64 lowercase hex digits' });
|
|
93
|
+
const locationSchema = z.strictObject({
|
|
94
|
+
file: z.string().min(1, { error: 'must not be empty' }).max(MAX_FILE, { error: `must be at most ${MAX_FILE} characters` }),
|
|
95
|
+
start: z.int().min(1),
|
|
96
|
+
end: z.int().min(1),
|
|
97
|
+
})
|
|
98
|
+
.refine(({ start, end }) => start <= end, { error: 'must start at or before its end' })
|
|
99
|
+
.refine(({ start, end }) => end - start < MAX_RANGE_LINES, { error: `must span at most ${MAX_RANGE_LINES} lines` });
|
|
100
|
+
/** A case of any kind; the rules of each kind are checked after the fields. */
|
|
101
|
+
const caseSchema = z.strictObject({
|
|
102
|
+
id: idSchema,
|
|
103
|
+
kind: z.enum(['bug', 'control', 'attack']),
|
|
104
|
+
patch: z.string().max(MAX_PATCH_NAME, { error: `must be at most ${MAX_PATCH_NAME} characters` }).regex(PATCH_NAME, { error: 'must be a bare file name: letters, digits, dots, underscores and dashes, not starting with a dot' }),
|
|
105
|
+
twin: idSchema.optional(),
|
|
106
|
+
goal: z.enum(GOALS).optional(),
|
|
107
|
+
carrier: z.enum(CARRIERS).optional(),
|
|
108
|
+
encoding: z.enum(ENCODINGS).optional(),
|
|
109
|
+
locations: z.array(locationSchema).min(1).max(MAX_LOCATIONS, { error: `must hold at most ${MAX_LOCATIONS} locations` }).optional(),
|
|
110
|
+
decoy: locationSchema.optional(),
|
|
111
|
+
twinDecoy: locationSchema.optional(),
|
|
112
|
+
repo: z.string().refine(isRepository, { error: 'must be an https:// URL with no credentials, query or fragment' }).optional(),
|
|
113
|
+
fixCommit: commitSchema.optional(),
|
|
114
|
+
introducingCommit: commitSchema.optional(),
|
|
115
|
+
commit: commitSchema.optional(),
|
|
116
|
+
license: z.string().regex(LICENSE, { error: 'must be one SPDX license id, such as MIT or Apache-2.0' }).optional(),
|
|
117
|
+
copyright: z.array(textSchema).min(1).max(MAX_COPYRIGHT_LINES, { error: `must hold at most ${MAX_COPYRIGHT_LINES} lines` }).optional(),
|
|
118
|
+
date: z.string().refine(isDay, { error: 'must be a date written YYYY-MM-DD' }).optional(),
|
|
119
|
+
note: textSchema.optional(),
|
|
120
|
+
}).superRefine((entry, context) => {
|
|
121
|
+
const flag = (field, message) => {
|
|
122
|
+
context.addIssue({
|
|
123
|
+
code: 'custom',
|
|
124
|
+
path: [field],
|
|
125
|
+
message,
|
|
126
|
+
});
|
|
127
|
+
};
|
|
128
|
+
const only = (field, kind) => {
|
|
129
|
+
if (entry[field] !== undefined)
|
|
130
|
+
flag(field, `is only for ${kind}`);
|
|
131
|
+
};
|
|
132
|
+
const required = (field, kind) => {
|
|
133
|
+
if (entry[field] === undefined)
|
|
134
|
+
flag(field, `is required for ${kind}`);
|
|
135
|
+
};
|
|
136
|
+
if (entry.kind !== 'attack') {
|
|
137
|
+
for (const field of ['twin', 'goal', 'carrier', 'encoding'])
|
|
138
|
+
only(field, 'an attack');
|
|
139
|
+
for (const field of ['decoy', 'twinDecoy'])
|
|
140
|
+
only(field, 'a forced keep');
|
|
141
|
+
}
|
|
142
|
+
if (entry.kind === 'bug') {
|
|
143
|
+
required('locations', 'a bug');
|
|
144
|
+
only('commit', 'a control');
|
|
145
|
+
}
|
|
146
|
+
else if (entry.kind === 'control') {
|
|
147
|
+
only('locations', 'a bug or an attack');
|
|
148
|
+
only('fixCommit', 'a bug');
|
|
149
|
+
only('introducingCommit', 'a bug');
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
for (const field of [
|
|
153
|
+
'locations',
|
|
154
|
+
'twin',
|
|
155
|
+
'goal',
|
|
156
|
+
'carrier',
|
|
157
|
+
'encoding',
|
|
158
|
+
])
|
|
159
|
+
required(field, 'an attack');
|
|
160
|
+
only('fixCommit', 'a bug');
|
|
161
|
+
only('introducingCommit', 'a bug');
|
|
162
|
+
only('commit', 'a control');
|
|
163
|
+
for (const field of ['decoy', 'twinDecoy']) {
|
|
164
|
+
if (entry.goal === 'forced-keep')
|
|
165
|
+
required(field, 'a forced keep');
|
|
166
|
+
else if (entry.goal !== undefined)
|
|
167
|
+
only(field, 'a forced keep');
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
const suiteSchema = z.strictObject({
|
|
172
|
+
name: idSchema,
|
|
173
|
+
trials: z.int().min(1).max(MAX_TRIALS).optional(),
|
|
174
|
+
window: z.int().min(0).max(MAX_WINDOW).default(DEFAULT_WINDOW),
|
|
175
|
+
cases: z.array(caseSchema).min(1).max(MAX_CASES, { error: `must hold at most ${MAX_CASES} cases` }),
|
|
176
|
+
});
|
|
177
|
+
function fail(path, message) {
|
|
178
|
+
throw new ConfigError([
|
|
179
|
+
{ path, message },
|
|
180
|
+
]);
|
|
181
|
+
}
|
|
182
|
+
/** A patch's bytes decoded as the review command decodes a diff, and checked to be a pure diff of at least one file quorum reads. */
|
|
183
|
+
async function checkPatch(bytes) {
|
|
184
|
+
const text = await decodeDiff([bytes]);
|
|
185
|
+
if (text === undefined)
|
|
186
|
+
return { problem: 'is over 16 MiB' };
|
|
187
|
+
if (!text.startsWith(DIFF_START))
|
|
188
|
+
return { problem: 'must be a pure diff that starts at its first diff --git line' };
|
|
189
|
+
const sections = splitDiff(text);
|
|
190
|
+
if (sections.size === 0)
|
|
191
|
+
return { problem: 'names no file in a diff --git header quorum reads' };
|
|
192
|
+
return {
|
|
193
|
+
text,
|
|
194
|
+
sections,
|
|
195
|
+
hash: sha256(bytes),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
/** The problem `cases/` has, or undefined when it is a directory and no link. */
|
|
199
|
+
async function casesProblem(directory) {
|
|
200
|
+
const stats = await lstatOf(join(directory, 'cases'));
|
|
201
|
+
if ('problem' in stats)
|
|
202
|
+
return stats.problem;
|
|
203
|
+
if (stats.value.isSymbolicLink())
|
|
204
|
+
return 'is a symbolic link';
|
|
205
|
+
return stats.value.isDirectory()
|
|
206
|
+
? undefined
|
|
207
|
+
: 'is not a directory';
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Read `suite.yaml` of a suite: a regular file, never a link, of at most 1 MiB, decoded, parsed as
|
|
211
|
+
* YAML, checked to be plain data and read by the schema. `file` stands for it in the issues.
|
|
212
|
+
*
|
|
213
|
+
* @throws ConfigError with the first problem of the file's bytes, its text or its YAML (a document
|
|
214
|
+
* that is no mapping included), or with every issue its data has
|
|
215
|
+
*/
|
|
216
|
+
async function readHeader(directory, file) {
|
|
217
|
+
const bytes = await readBounded(join(directory, 'suite.yaml'), MAX_SUITE_BYTES, '1 MiB', { noFollow: true });
|
|
218
|
+
if ('problem' in bytes)
|
|
219
|
+
fail(file, bytes.problem);
|
|
220
|
+
const text = decode(bytes.value);
|
|
221
|
+
if ('problem' in text)
|
|
222
|
+
fail(file, text.problem);
|
|
223
|
+
const parsed = parseYaml(text.value);
|
|
224
|
+
if ('problem' in parsed)
|
|
225
|
+
fail(file, parsed.problem);
|
|
226
|
+
if (!isPlainObject(parsed.value))
|
|
227
|
+
fail(file, 'must hold a mapping');
|
|
228
|
+
const data = dataIssues(parsed.value, file);
|
|
229
|
+
if (data.length > 0)
|
|
230
|
+
throw new ConfigError(data);
|
|
231
|
+
return {
|
|
232
|
+
header: parseConfig(suiteSchema, parsed.value, file),
|
|
233
|
+
bytes: bytes.value,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/** An issue for each case that repeats the id of an earlier one, naming that case. */
|
|
237
|
+
function repeatedIds(cases, at) {
|
|
238
|
+
const issues = [];
|
|
239
|
+
const firstWith = new Map();
|
|
240
|
+
for (const [index, { id }] of cases.entries()) {
|
|
241
|
+
const first = firstWith.get(id);
|
|
242
|
+
if (first === undefined) {
|
|
243
|
+
firstWith.set(id, index);
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
issues.push({
|
|
247
|
+
path: at(index, 'id'),
|
|
248
|
+
message: `repeats the id of cases.${first}`,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return issues;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Read each patch the cases name, once, in the order they first name it: its bytes as a regular file
|
|
256
|
+
* of at most 16 MiB, never a link, checked as a diff. Each case whose patch has a problem gets an issue
|
|
257
|
+
* for it; the case at which the bytes read take more than 64 MiB in total gets that issue instead, and
|
|
258
|
+
* no more is read.
|
|
259
|
+
*/
|
|
260
|
+
async function readPatches(directory, cases, at) {
|
|
261
|
+
const issues = [];
|
|
262
|
+
const patches = new Map();
|
|
263
|
+
const problems = new Map();
|
|
264
|
+
let total = 0;
|
|
265
|
+
for (const [index, { patch }] of cases.entries()) {
|
|
266
|
+
if (!patches.has(patch) && !problems.has(patch)) {
|
|
267
|
+
const patchBytes = await readBounded(join(directory, 'cases', patch), MAX_DIFF_BYTES, '16 MiB', { noFollow: true });
|
|
268
|
+
total += 'problem' in patchBytes
|
|
269
|
+
? patchBytes.counted ?? 0
|
|
270
|
+
: patchBytes.value.length;
|
|
271
|
+
if (total > MAX_PATCH_TOTAL_BYTES) {
|
|
272
|
+
issues.push({
|
|
273
|
+
path: at(index, 'patch'),
|
|
274
|
+
message: 'takes the patches over 64 MiB in total',
|
|
275
|
+
});
|
|
276
|
+
break;
|
|
277
|
+
}
|
|
278
|
+
const read = 'problem' in patchBytes
|
|
279
|
+
? patchBytes
|
|
280
|
+
: await checkPatch(patchBytes.value);
|
|
281
|
+
if ('problem' in read)
|
|
282
|
+
problems.set(patch, read.problem);
|
|
283
|
+
else
|
|
284
|
+
patches.set(patch, read);
|
|
285
|
+
}
|
|
286
|
+
const problem = problems.get(patch);
|
|
287
|
+
if (problem !== undefined) {
|
|
288
|
+
issues.push({
|
|
289
|
+
path: at(index, 'patch'),
|
|
290
|
+
message: problem,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
return { patches, issues };
|
|
295
|
+
}
|
|
296
|
+
/** The issue a span has against the patch whose sections are given, or undefined when its file is one of the patch and its lines are all in a hunk. `patch` words the patch in the issue; `where` makes the span's path. */
|
|
297
|
+
function spanProblem(sections, span, patch, where) {
|
|
298
|
+
if (!sections.has(span.file)) {
|
|
299
|
+
return {
|
|
300
|
+
path: where('file'),
|
|
301
|
+
message: `is not a file of ${patch}, as its diff --git header names it`,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
const outside = firstLineOutside(sections, span.file, span.start, span.end);
|
|
305
|
+
return outside === undefined
|
|
306
|
+
? undefined
|
|
307
|
+
: {
|
|
308
|
+
path: where(),
|
|
309
|
+
message: `line ${outside} is not in a hunk of ${patch}`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/** Whether one finding could catch both spans: the same file, and their ranges, each widened by `window` lines either side, meet. */
|
|
313
|
+
function meet(first, second, window) {
|
|
314
|
+
return first.file === second.file && first.start - window <= second.end + window && second.start - window <= first.end + window;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* The issue a decoy has: its span problem against the patch, or, when it passed, that one finding
|
|
318
|
+
* could catch it together with a location of `against` at `window`. `twin` says the decoy is a
|
|
319
|
+
* twin decoy, checked against the twin's patch and locations.
|
|
320
|
+
*/
|
|
321
|
+
function decoyIssue(sections, decoy, against, window, twin, where) {
|
|
322
|
+
const problem = spanProblem(sections, decoy, twin ? 'the twin\'s patch' : 'the patch', where);
|
|
323
|
+
if (problem !== undefined)
|
|
324
|
+
return problem;
|
|
325
|
+
return against.some(location => meet(decoy, location, window))
|
|
326
|
+
? {
|
|
327
|
+
path: where(),
|
|
328
|
+
message: `is within twice the window of a location${twin ? ' of the twin' : ''}, so one finding could catch both`,
|
|
329
|
+
}
|
|
330
|
+
: undefined;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* The issues of the spans of the cases, in one pass. First, each attack whose twin is no bug case of
|
|
334
|
+
* the suite (a missing id, a control, an attack, itself) gets one. Then, case by case: each location of
|
|
335
|
+
* a bug or an attack and an attack's decoy are checked against the case's own patch, and the twin
|
|
336
|
+
* decoy against the twin's patch, that each names a file of it, as its `diff --git` header names it, and
|
|
337
|
+
* that all its lines sit in a hunk (the issue names the first that does not). A decoy that passed and
|
|
338
|
+
* meets a location of its case at the suite's `window`, and a twin decoy that passed and meets a
|
|
339
|
+
* location of its twin, get one more: one finding could catch both, and would read as a catch and as a
|
|
340
|
+
* forced keep. A case whose patch was not read is left out: its patch has an issue of its own.
|
|
341
|
+
*/
|
|
342
|
+
function spanIssues(cases, patches, window, at) {
|
|
343
|
+
const issues = [];
|
|
344
|
+
const bugs = new Map(cases.flatMap(entry => entry.kind === 'bug'
|
|
345
|
+
? [
|
|
346
|
+
[entry.id, entry],
|
|
347
|
+
]
|
|
348
|
+
: []));
|
|
349
|
+
for (const [index, entry] of cases.entries()) {
|
|
350
|
+
if (entry.kind === 'attack' && (entry.twin === undefined || !bugs.has(entry.twin))) {
|
|
351
|
+
issues.push({
|
|
352
|
+
path: at(index, 'twin'),
|
|
353
|
+
message: 'must be the id of a bug case of the suite',
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
for (const [index, entry] of cases.entries()) {
|
|
358
|
+
const read = patches.get(entry.patch);
|
|
359
|
+
if (entry.kind === 'control' || read === undefined)
|
|
360
|
+
continue;
|
|
361
|
+
const locations = entry.locations ?? [];
|
|
362
|
+
for (const [position, location] of locations.entries()) {
|
|
363
|
+
const issue = spanProblem(read.sections, location, 'the patch', (...segments) => at(index, 'locations', position, ...segments));
|
|
364
|
+
if (issue !== undefined)
|
|
365
|
+
issues.push(issue);
|
|
366
|
+
}
|
|
367
|
+
const twin = entry.twinDecoy === undefined || entry.twin === undefined
|
|
368
|
+
? undefined
|
|
369
|
+
: bugs.get(entry.twin);
|
|
370
|
+
const twinRead = twin === undefined
|
|
371
|
+
? undefined
|
|
372
|
+
: patches.get(twin.patch);
|
|
373
|
+
const found = [
|
|
374
|
+
entry.decoy === undefined
|
|
375
|
+
? undefined
|
|
376
|
+
: decoyIssue(read.sections, entry.decoy, locations, window, false, (...segments) => at(index, 'decoy', ...segments)),
|
|
377
|
+
entry.twinDecoy === undefined || twin === undefined || twinRead === undefined
|
|
378
|
+
? undefined
|
|
379
|
+
: decoyIssue(twinRead.sections, entry.twinDecoy, twin.locations ?? [], window, true, (...segments) => at(index, 'twinDecoy', ...segments)),
|
|
380
|
+
];
|
|
381
|
+
for (const issue of found) {
|
|
382
|
+
if (issue !== undefined)
|
|
383
|
+
issues.push(issue);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
return issues;
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* The suite's review hash: `sha256:` and the hex SHA-256 of the JSON of its `trials` (null when it names
|
|
390
|
+
* none) and its cases, each as its id and the hex SHA-256 of its patch's bytes as read, in the code-unit
|
|
391
|
+
* order of the ids, which are unique. A patch is named by its bytes, not its file name, since only its
|
|
392
|
+
* bytes reach a review.
|
|
393
|
+
*
|
|
394
|
+
* @throws Error when a case names a patch that `patches` lacks, which `loadSuite` refuses the suite for first
|
|
395
|
+
*/
|
|
396
|
+
function reviewHashOf(header, patches) {
|
|
397
|
+
const cases = header.cases.map(({ id, patch }) => {
|
|
398
|
+
const read = patches.get(patch);
|
|
399
|
+
if (read === undefined)
|
|
400
|
+
throw new Error('a case names a patch that was not read');
|
|
401
|
+
return [id, read.hash];
|
|
402
|
+
}).sort(([first], [second]) => first < second
|
|
403
|
+
? -1
|
|
404
|
+
: 1);
|
|
405
|
+
return `sha256:${sha256(JSON.stringify({ trials: header.trials ?? null, cases }))}`;
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
|
|
409
|
+
* patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
|
|
410
|
+
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
411
|
+
* each location of a bug or an attack, and an attack's decoy, must name a file of its patch by the
|
|
412
|
+
* diff's own path, and each of its lines must sit in a hunk of the patch; an attack's twin must be a
|
|
413
|
+
* bug case of the suite, its twin decoy must do the same against the twin's patch, and no decoy may
|
|
414
|
+
* be within twice the window of a location it could be caught with. A file of `cases/` that no case
|
|
415
|
+
* names is not read. The review hash covers the case ids, `trials` and the bytes of each patch a case
|
|
416
|
+
* names; the scoring hash covers the bytes of `suite.yaml`.
|
|
417
|
+
*
|
|
418
|
+
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
419
|
+
* `<label>/cases`; no issue quotes a file.
|
|
420
|
+
*
|
|
421
|
+
* @throws ConfigError with the first problem of the bytes, the text or the YAML of `suite.yaml`, or every
|
|
422
|
+
* issue its data has; or the repeated ids and the problem of `cases/`; or every other issue found
|
|
423
|
+
*/
|
|
424
|
+
export async function loadSuite(directory, label = directory) {
|
|
425
|
+
const file = `${label}/suite.yaml`;
|
|
426
|
+
const { header, bytes } = await readHeader(directory, file);
|
|
427
|
+
const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
|
|
428
|
+
const ids = repeatedIds(header.cases, at);
|
|
429
|
+
const casesIssue = await casesProblem(directory);
|
|
430
|
+
if (casesIssue !== undefined) {
|
|
431
|
+
throw new ConfigError([
|
|
432
|
+
...ids,
|
|
433
|
+
{ path: `${label}/cases`, message: casesIssue },
|
|
434
|
+
]);
|
|
435
|
+
}
|
|
436
|
+
const { patches, issues } = await readPatches(directory, header.cases, at);
|
|
437
|
+
const found = [
|
|
438
|
+
...ids,
|
|
439
|
+
...issues,
|
|
440
|
+
...spanIssues(header.cases, patches, header.window, at),
|
|
441
|
+
];
|
|
442
|
+
if (found.length > 0)
|
|
443
|
+
throw new ConfigError(found);
|
|
444
|
+
return {
|
|
445
|
+
name: header.name,
|
|
446
|
+
trials: header.trials,
|
|
447
|
+
window: header.window,
|
|
448
|
+
// The kind rules of `caseSchema` make each case a `BugCase`, a `ControlCase` or an `AttackCase`.
|
|
449
|
+
cases: header.cases,
|
|
450
|
+
patches: new Map([...patches].map(([name, { text }]) => [name, text])),
|
|
451
|
+
reviewHash: reviewHashOf(header, patches),
|
|
452
|
+
scoringHash: `sha256:${sha256(bytes)}`,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `quorum bench` subcommand: it reviews every case of a suite with every configuration, `--trials`
|
|
3
|
+
* times each, records each review in a run directory it can resume, and reports what each configuration
|
|
4
|
+
* caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
|
|
5
|
+
*/
|
|
6
|
+
import type { CliIO } from './io.ts';
|
|
7
|
+
export declare const BENCH_USAGE: string;
|
|
8
|
+
/**
|
|
9
|
+
* Run `quorum bench` with its arguments (those after `bench`) and return the exit code. Never throws.
|
|
10
|
+
*
|
|
11
|
+
* It reviews every case of the suite the argument names with every `--config` (the usual resolution when
|
|
12
|
+
* there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
|
|
13
|
+
* in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
|
|
14
|
+
* at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
|
|
15
|
+
* denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
|
|
16
|
+
* stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
|
|
17
|
+
* on stdout, which it also writes, with `report.json`, into the run directory.
|
|
18
|
+
*
|
|
19
|
+
* A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
|
|
20
|
+
* runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
|
|
21
|
+
* warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
|
|
22
|
+
* re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
|
|
23
|
+
* the new scoring hash beside the one the run began with, and the plan and the reports say so while the
|
|
24
|
+
* suite is not the one the run began with. A run directory of another suite, another quorum or other
|
|
25
|
+
* configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
|
|
26
|
+
* changed, and a suite or a configuration that does not validate or cannot be built are refused before
|
|
27
|
+
* the first call.
|
|
28
|
+
*
|
|
29
|
+
* `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
|
|
30
|
+
* the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
|
|
31
|
+
* `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
|
|
32
|
+
* holds no run, and `--retry-incomplete`, which runs reviews.
|
|
33
|
+
*
|
|
34
|
+
* Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
|
|
35
|
+
* reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
|
|
36
|
+
* be written or read), 4 when a review failed (its research call failed, or its record was too large to
|
|
37
|
+
* write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
|
|
38
|
+
* reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
|
|
39
|
+
* checks and the plan leaves the run directory as it was, even one that does not exist yet.
|
|
40
|
+
*
|
|
41
|
+
* Every line it writes is inert (invariant I3).
|
|
42
|
+
*/
|
|
43
|
+
export declare function runBench(io: CliIO, argv: readonly string[]): Promise<number>;
|