@tobi-or-not/quorum 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +3 -2
- package/dist/bench/fingerprint.js +3 -2
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +171 -13
- package/dist/bench/records.js +34 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +29 -17
- package/dist/bench/runner.js +53 -18
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +20 -9
- package/dist/cli/bench.js +148 -74
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.js +2 -19
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +77 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +1 -1
package/dist/bench/suite.js
CHANGED
|
@@ -9,15 +9,14 @@
|
|
|
9
9
|
* Everything is checked before any call, and no message quotes the file or a patch.
|
|
10
10
|
*/
|
|
11
11
|
import { createHash } from 'node:crypto';
|
|
12
|
-
import { lstat } from 'node:fs/promises';
|
|
13
12
|
import { join } from 'node:path';
|
|
14
13
|
import { z } from 'zod';
|
|
15
|
-
import { dataIssues, decode, isPlainObject, parseYaml, readBounded } from "../config/load.js";
|
|
14
|
+
import { dataIssues, decode, isPlainObject, lstatOf, parseYaml, readBounded } from "../config/load.js";
|
|
16
15
|
import { configPath, parseConfig } from "../config/schema.js";
|
|
17
|
-
import { decodeDiff,
|
|
16
|
+
import { decodeDiff, firstLineOutside, MAX_DIFF_BYTES, splitDiff } from "../core/diff.js";
|
|
18
17
|
import { ConfigError } from "../core/errors.js";
|
|
19
18
|
import { hiddenLabels } from "../core/sanitize.js";
|
|
20
|
-
/** A suite name or a case id: a name for a directory and for a record, so it
|
|
19
|
+
/** A suite name or a case id: a name for a directory and for a record, so it is never `.` or `..`, and holds no separator. */
|
|
21
20
|
export const SUITE_ID = /^[a-z0-9][a-z0-9._-]*$/;
|
|
22
21
|
/** A patch's file name: bare, and not starting with a dot. */
|
|
23
22
|
export const PATCH_NAME = /^[A-Z0-9][\w.-]*$/i;
|
|
@@ -36,7 +35,8 @@ export const MAX_WINDOW = 100;
|
|
|
36
35
|
export const DEFAULT_WINDOW = 3;
|
|
37
36
|
/** The most the patches of a suite may hold together, in bytes: a loaded suite holds them in memory. */
|
|
38
37
|
export const MAX_PATCH_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
39
|
-
|
|
38
|
+
/** The most characters of a suite name or a case id. */
|
|
39
|
+
export const MAX_ID = 64;
|
|
40
40
|
const MAX_PATCH_NAME = 128;
|
|
41
41
|
const MAX_FILE = 4096;
|
|
42
42
|
const MAX_REPO = 200;
|
|
@@ -47,6 +47,30 @@ const LICENSE = /^[A-Z0-9][A-Z0-9.+-]{0,63}$/i;
|
|
|
47
47
|
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
48
48
|
/** What every diff the bench reads starts with. */
|
|
49
49
|
const DIFF_START = 'diff --git ';
|
|
50
|
+
/** What an attack case tries to do to the review of its twin, in the security model's order. */
|
|
51
|
+
export const GOALS = [
|
|
52
|
+
'suppression',
|
|
53
|
+
'forced-keep',
|
|
54
|
+
'cost-bloat',
|
|
55
|
+
'denial-of-review',
|
|
56
|
+
];
|
|
57
|
+
/** Where an attack case plants its text in the patch. */
|
|
58
|
+
export const CARRIERS = [
|
|
59
|
+
'comment',
|
|
60
|
+
'string-literal',
|
|
61
|
+
'file-name',
|
|
62
|
+
'documentation',
|
|
63
|
+
'guideline-file',
|
|
64
|
+
];
|
|
65
|
+
/** How an attack case writes its text. */
|
|
66
|
+
export const ENCODINGS = [
|
|
67
|
+
'plain',
|
|
68
|
+
'multilingual',
|
|
69
|
+
'hidden-unicode',
|
|
70
|
+
'base64',
|
|
71
|
+
'html-comment',
|
|
72
|
+
'fake-delimiter',
|
|
73
|
+
];
|
|
50
74
|
const sha256 = (data) => createHash('sha256').update(data).digest('hex');
|
|
51
75
|
/** Whether `text` is a real day written YYYY-MM-DD. */
|
|
52
76
|
function isDay(text) {
|
|
@@ -62,8 +86,9 @@ function isRepository(text) {
|
|
|
62
86
|
const url = new URL(text);
|
|
63
87
|
return url.protocol === 'https:' && url.username === '' && url.password === '';
|
|
64
88
|
}
|
|
65
|
-
|
|
66
|
-
const
|
|
89
|
+
/** A suite name or a case id, as a schema: at most `MAX_ID` characters matching `SUITE_ID`. */
|
|
90
|
+
export const idSchema = z.string().max(MAX_ID, { error: `must be at most ${MAX_ID} characters` }).regex(SUITE_ID);
|
|
91
|
+
const textSchema = z.string().min(1, { error: 'must not be empty' }).max(MAX_TEXT, { error: `must be at most ${MAX_TEXT} characters` }).refine(text => hiddenLabels(text).length === 0, { error: 'must be one line of visible text' });
|
|
67
92
|
const commitSchema = z.string().regex(COMMIT, { error: 'must be a full commit hash: 40 or 64 lowercase hex digits' });
|
|
68
93
|
const locationSchema = z.strictObject({
|
|
69
94
|
file: z.string().min(1, { error: 'must not be empty' }).max(MAX_FILE, { error: `must be at most ${MAX_FILE} characters` }),
|
|
@@ -72,12 +97,18 @@ const locationSchema = z.strictObject({
|
|
|
72
97
|
})
|
|
73
98
|
.refine(({ start, end }) => start <= end, { error: 'must start at or before its end' })
|
|
74
99
|
.refine(({ start, end }) => end - start < MAX_RANGE_LINES, { error: `must span at most ${MAX_RANGE_LINES} lines` });
|
|
75
|
-
/** A case of
|
|
100
|
+
/** A case of any kind; the rules of each kind are checked after the fields. */
|
|
76
101
|
const caseSchema = z.strictObject({
|
|
77
102
|
id: idSchema,
|
|
78
|
-
kind: z.enum(['bug', 'control']),
|
|
103
|
+
kind: z.enum(['bug', 'control', 'attack']),
|
|
79
104
|
patch: z.string().max(MAX_PATCH_NAME, { error: `must be at most ${MAX_PATCH_NAME} characters` }).regex(PATCH_NAME, { error: 'must be a bare file name: letters, digits, dots, underscores and dashes, not starting with a dot' }),
|
|
105
|
+
twin: idSchema.optional(),
|
|
106
|
+
goal: z.enum(GOALS).optional(),
|
|
107
|
+
carrier: z.enum(CARRIERS).optional(),
|
|
108
|
+
encoding: z.enum(ENCODINGS).optional(),
|
|
80
109
|
locations: z.array(locationSchema).min(1).max(MAX_LOCATIONS, { error: `must hold at most ${MAX_LOCATIONS} locations` }).optional(),
|
|
110
|
+
decoy: locationSchema.optional(),
|
|
111
|
+
twinDecoy: locationSchema.optional(),
|
|
81
112
|
repo: z.string().refine(isRepository, { error: 'must be an https:// URL with no credentials, query or fragment' }).optional(),
|
|
82
113
|
fixCommit: commitSchema.optional(),
|
|
83
114
|
introducingCommit: commitSchema.optional(),
|
|
@@ -87,29 +118,54 @@ const caseSchema = z.strictObject({
|
|
|
87
118
|
date: z.string().refine(isDay, { error: 'must be a date written YYYY-MM-DD' }).optional(),
|
|
88
119
|
note: textSchema.optional(),
|
|
89
120
|
}).superRefine((entry, context) => {
|
|
121
|
+
const flag = (field, message) => {
|
|
122
|
+
context.addIssue({
|
|
123
|
+
code: 'custom',
|
|
124
|
+
path: [field],
|
|
125
|
+
message,
|
|
126
|
+
});
|
|
127
|
+
};
|
|
90
128
|
const only = (field, kind) => {
|
|
91
|
-
if (entry[field] !== undefined)
|
|
92
|
-
|
|
93
|
-
code: 'custom',
|
|
94
|
-
path: [field],
|
|
95
|
-
message: `is only for a ${kind}`,
|
|
96
|
-
});
|
|
97
|
-
}
|
|
129
|
+
if (entry[field] !== undefined)
|
|
130
|
+
flag(field, `is only for ${kind}`);
|
|
98
131
|
};
|
|
132
|
+
const required = (field, kind) => {
|
|
133
|
+
if (entry[field] === undefined)
|
|
134
|
+
flag(field, `is required for ${kind}`);
|
|
135
|
+
};
|
|
136
|
+
if (entry.kind !== 'attack') {
|
|
137
|
+
for (const field of ['twin', 'goal', 'carrier', 'encoding'])
|
|
138
|
+
only(field, 'an attack');
|
|
139
|
+
for (const field of ['decoy', 'twinDecoy'])
|
|
140
|
+
only(field, 'a forced keep');
|
|
141
|
+
}
|
|
99
142
|
if (entry.kind === 'bug') {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
only('commit', 'control');
|
|
143
|
+
required('locations', 'a bug');
|
|
144
|
+
only('commit', 'a control');
|
|
145
|
+
}
|
|
146
|
+
else if (entry.kind === 'control') {
|
|
147
|
+
only('locations', 'a bug or an attack');
|
|
148
|
+
only('fixCommit', 'a bug');
|
|
149
|
+
only('introducingCommit', 'a bug');
|
|
108
150
|
}
|
|
109
151
|
else {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
152
|
+
for (const field of [
|
|
153
|
+
'locations',
|
|
154
|
+
'twin',
|
|
155
|
+
'goal',
|
|
156
|
+
'carrier',
|
|
157
|
+
'encoding',
|
|
158
|
+
])
|
|
159
|
+
required(field, 'an attack');
|
|
160
|
+
only('fixCommit', 'a bug');
|
|
161
|
+
only('introducingCommit', 'a bug');
|
|
162
|
+
only('commit', 'a control');
|
|
163
|
+
for (const field of ['decoy', 'twinDecoy']) {
|
|
164
|
+
if (entry.goal === 'forced-keep')
|
|
165
|
+
required(field, 'a forced keep');
|
|
166
|
+
else if (entry.goal !== undefined)
|
|
167
|
+
only(field, 'a forced keep');
|
|
168
|
+
}
|
|
113
169
|
}
|
|
114
170
|
});
|
|
115
171
|
const suiteSchema = z.strictObject({
|
|
@@ -141,36 +197,23 @@ async function checkPatch(bytes) {
|
|
|
141
197
|
}
|
|
142
198
|
/** The problem `cases/` has, or undefined when it is a directory and no link. */
|
|
143
199
|
async function casesProblem(directory) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
catch (error) {
|
|
153
|
-
const code = error.code;
|
|
154
|
-
return code === 'ENOENT' || code === 'ENOTDIR'
|
|
155
|
-
? 'does not exist'
|
|
156
|
-
: 'cannot be read';
|
|
157
|
-
}
|
|
200
|
+
const stats = await lstatOf(join(directory, 'cases'));
|
|
201
|
+
if ('problem' in stats)
|
|
202
|
+
return stats.problem;
|
|
203
|
+
if (stats.value.isSymbolicLink())
|
|
204
|
+
return 'is a symbolic link';
|
|
205
|
+
return stats.value.isDirectory()
|
|
206
|
+
? undefined
|
|
207
|
+
: 'is not a directory';
|
|
158
208
|
}
|
|
159
209
|
/**
|
|
160
|
-
* Read
|
|
161
|
-
*
|
|
162
|
-
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
163
|
-
* each location of a bug must name a file of its patch by the diff's own path, and each of its lines
|
|
164
|
-
* must sit in a hunk of the patch. A file of `cases/` that no case names is not read. The hash covers
|
|
165
|
-
* `suite.yaml` and each patch a case names, once.
|
|
166
|
-
*
|
|
167
|
-
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
168
|
-
* `<label>/cases`; no issue quotes a file.
|
|
210
|
+
* Read `suite.yaml` of a suite: a regular file, never a link, of at most 1 MiB, decoded, parsed as
|
|
211
|
+
* YAML, checked to be plain data and read by the schema. `file` stands for it in the issues.
|
|
169
212
|
*
|
|
170
|
-
* @throws ConfigError with the first problem of
|
|
213
|
+
* @throws ConfigError with the first problem of the file's bytes, its text or its YAML (a document
|
|
214
|
+
* that is no mapping included), or with every issue its data has
|
|
171
215
|
*/
|
|
172
|
-
|
|
173
|
-
const file = `${label}/suite.yaml`;
|
|
216
|
+
async function readHeader(directory, file) {
|
|
174
217
|
const bytes = await readBounded(join(directory, 'suite.yaml'), MAX_SUITE_BYTES, '1 MiB', { noFollow: true });
|
|
175
218
|
if ('problem' in bytes)
|
|
176
219
|
fail(file, bytes.problem);
|
|
@@ -185,11 +228,16 @@ export async function loadSuite(directory, label = directory) {
|
|
|
185
228
|
const data = dataIssues(parsed.value, file);
|
|
186
229
|
if (data.length > 0)
|
|
187
230
|
throw new ConfigError(data);
|
|
188
|
-
|
|
189
|
-
|
|
231
|
+
return {
|
|
232
|
+
header: parseConfig(suiteSchema, parsed.value, file),
|
|
233
|
+
bytes: bytes.value,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/** An issue for each case that repeats the id of an earlier one, naming that case. */
|
|
237
|
+
function repeatedIds(cases, at) {
|
|
190
238
|
const issues = [];
|
|
191
239
|
const firstWith = new Map();
|
|
192
|
-
for (const [index, { id }] of
|
|
240
|
+
for (const [index, { id }] of cases.entries()) {
|
|
193
241
|
const first = firstWith.get(id);
|
|
194
242
|
if (first === undefined) {
|
|
195
243
|
firstWith.set(id, index);
|
|
@@ -201,17 +249,20 @@ export async function loadSuite(directory, label = directory) {
|
|
|
201
249
|
});
|
|
202
250
|
}
|
|
203
251
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
252
|
+
return issues;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Read each patch the cases name, once, in the order they first name it: its bytes as a regular file
|
|
256
|
+
* of at most 16 MiB, never a link, checked as a diff. Each case whose patch has a problem gets an issue
|
|
257
|
+
* for it; the case at which the bytes read take more than 64 MiB in total gets that issue instead, and
|
|
258
|
+
* no more is read.
|
|
259
|
+
*/
|
|
260
|
+
async function readPatches(directory, cases, at) {
|
|
261
|
+
const issues = [];
|
|
211
262
|
const patches = new Map();
|
|
212
263
|
const problems = new Map();
|
|
213
264
|
let total = 0;
|
|
214
|
-
for (const [index, { patch }] of
|
|
265
|
+
for (const [index, { patch }] of cases.entries()) {
|
|
215
266
|
if (!patches.has(patch) && !problems.has(patch)) {
|
|
216
267
|
const patchBytes = await readBounded(join(directory, 'cases', patch), MAX_DIFF_BYTES, '16 MiB', { noFollow: true });
|
|
217
268
|
total += 'problem' in patchBytes
|
|
@@ -240,47 +291,164 @@ export async function loadSuite(directory, label = directory) {
|
|
|
240
291
|
});
|
|
241
292
|
}
|
|
242
293
|
}
|
|
243
|
-
|
|
294
|
+
return { patches, issues };
|
|
295
|
+
}
|
|
296
|
+
/** The issue a span has against the patch whose sections are given, or undefined when its file is one of the patch and its lines are all in a hunk. `patch` words the patch in the issue; `where` makes the span's path. */
|
|
297
|
+
function spanProblem(sections, span, patch, where) {
|
|
298
|
+
if (!sections.has(span.file)) {
|
|
299
|
+
return {
|
|
300
|
+
path: where('file'),
|
|
301
|
+
message: `is not a file of ${patch}, as its diff --git header names it`,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
const outside = firstLineOutside(sections, span.file, span.start, span.end);
|
|
305
|
+
return outside === undefined
|
|
306
|
+
? undefined
|
|
307
|
+
: {
|
|
308
|
+
path: where(),
|
|
309
|
+
message: `line ${outside} is not in a hunk of ${patch}`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/** Whether one finding could catch both spans: the same file, and their ranges, each widened by `window` lines either side, meet. */
|
|
313
|
+
function meet(first, second, window) {
|
|
314
|
+
return first.file === second.file && first.start - window <= second.end + window && second.start - window <= first.end + window;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* The issue a decoy has: its span problem against the patch, or, when it passed, that one finding
|
|
318
|
+
* could catch it together with a location of `against` at `window`. `twin` says the decoy is a
|
|
319
|
+
* twin decoy, checked against the twin's patch and locations.
|
|
320
|
+
*/
|
|
321
|
+
function decoyIssue(sections, decoy, against, window, twin, where) {
|
|
322
|
+
const problem = spanProblem(sections, decoy, twin ? 'the twin\'s patch' : 'the patch', where);
|
|
323
|
+
if (problem !== undefined)
|
|
324
|
+
return problem;
|
|
325
|
+
return against.some(location => meet(decoy, location, window))
|
|
326
|
+
? {
|
|
327
|
+
path: where(),
|
|
328
|
+
message: `is within twice the window of a location${twin ? ' of the twin' : ''}, so one finding could catch both`,
|
|
329
|
+
}
|
|
330
|
+
: undefined;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* The issues of the spans of the cases, in one pass. First, each attack whose twin is no bug case of
|
|
334
|
+
* the suite (a missing id, a control, an attack, itself) gets one. Then, case by case: each location of
|
|
335
|
+
* a bug or an attack and an attack's decoy are checked against the case's own patch, and the twin
|
|
336
|
+
* decoy against the twin's patch, that each names a file of it, as its `diff --git` header names it, and
|
|
337
|
+
* that all its lines sit in a hunk (the issue names the first that does not). A decoy that passed and
|
|
338
|
+
* meets a location of its case at the suite's `window`, and a twin decoy that passed and meets a
|
|
339
|
+
* location of its twin, get one more: one finding could catch both, and would read as a catch and as a
|
|
340
|
+
* forced keep. A case whose patch was not read is left out: its patch has an issue of its own.
|
|
341
|
+
*/
|
|
342
|
+
function spanIssues(cases, patches, window, at) {
|
|
343
|
+
const issues = [];
|
|
344
|
+
const bugs = new Map(cases.flatMap(entry => entry.kind === 'bug'
|
|
345
|
+
? [
|
|
346
|
+
[entry.id, entry],
|
|
347
|
+
]
|
|
348
|
+
: []));
|
|
349
|
+
for (const [index, entry] of cases.entries()) {
|
|
350
|
+
if (entry.kind === 'attack' && (entry.twin === undefined || !bugs.has(entry.twin))) {
|
|
351
|
+
issues.push({
|
|
352
|
+
path: at(index, 'twin'),
|
|
353
|
+
message: 'must be the id of a bug case of the suite',
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
for (const [index, entry] of cases.entries()) {
|
|
244
358
|
const read = patches.get(entry.patch);
|
|
245
|
-
if (entry.kind
|
|
359
|
+
if (entry.kind === 'control' || read === undefined)
|
|
246
360
|
continue;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
361
|
+
const locations = entry.locations ?? [];
|
|
362
|
+
for (const [position, location] of locations.entries()) {
|
|
363
|
+
const issue = spanProblem(read.sections, location, 'the patch', (...segments) => at(index, 'locations', position, ...segments));
|
|
364
|
+
if (issue !== undefined)
|
|
365
|
+
issues.push(issue);
|
|
366
|
+
}
|
|
367
|
+
const twin = entry.twinDecoy === undefined || entry.twin === undefined
|
|
368
|
+
? undefined
|
|
369
|
+
: bugs.get(entry.twin);
|
|
370
|
+
const twinRead = twin === undefined
|
|
371
|
+
? undefined
|
|
372
|
+
: patches.get(twin.patch);
|
|
373
|
+
const found = [
|
|
374
|
+
entry.decoy === undefined
|
|
375
|
+
? undefined
|
|
376
|
+
: decoyIssue(read.sections, entry.decoy, locations, window, false, (...segments) => at(index, 'decoy', ...segments)),
|
|
377
|
+
entry.twinDecoy === undefined || twin === undefined || twinRead === undefined
|
|
378
|
+
? undefined
|
|
379
|
+
: decoyIssue(twinRead.sections, entry.twinDecoy, twin.locations ?? [], window, true, (...segments) => at(index, 'twinDecoy', ...segments)),
|
|
380
|
+
];
|
|
381
|
+
for (const issue of found) {
|
|
382
|
+
if (issue !== undefined)
|
|
383
|
+
issues.push(issue);
|
|
264
384
|
}
|
|
265
385
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
386
|
+
return issues;
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* The suite's review hash: `sha256:` and the hex SHA-256 of the JSON of its `trials` (null when it names
|
|
390
|
+
* none) and its cases, each as its id and the hex SHA-256 of its patch's bytes as read, in the code-unit
|
|
391
|
+
* order of the ids, which are unique. A patch is named by its bytes, not its file name, since only its
|
|
392
|
+
* bytes reach a review.
|
|
393
|
+
*
|
|
394
|
+
* @throws Error when a case names a patch that `patches` lacks, which `loadSuite` refuses the suite for first
|
|
395
|
+
*/
|
|
396
|
+
function reviewHashOf(header, patches) {
|
|
397
|
+
const cases = header.cases.map(({ id, patch }) => {
|
|
398
|
+
const read = patches.get(patch);
|
|
399
|
+
if (read === undefined)
|
|
400
|
+
throw new Error('a case names a patch that was not read');
|
|
401
|
+
return [id, read.hash];
|
|
402
|
+
}).sort(([first], [second]) => first < second
|
|
403
|
+
? -1
|
|
404
|
+
: 1);
|
|
405
|
+
return `sha256:${sha256(JSON.stringify({ trials: header.trials ?? null, cases }))}`;
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Read the suite in `directory`: `suite.yaml` and the patches in `cases/` that its cases name. Each
|
|
409
|
+
* patch is read once, as a regular file of at most 16 MiB (all together at most 64 MiB), decoded as
|
|
410
|
+
* the review command decodes a diff, and must start with `diff --git ` and name at least one file;
|
|
411
|
+
* each location of a bug or an attack, and an attack's decoy, must name a file of its patch by the
|
|
412
|
+
* diff's own path, and each of its lines must sit in a hunk of the patch; an attack's twin must be a
|
|
413
|
+
* bug case of the suite, its twin decoy must do the same against the twin's patch, and no decoy may
|
|
414
|
+
* be within twice the window of a location it could be caught with. A file of `cases/` that no case
|
|
415
|
+
* names is not read. The review hash covers the case ids, `trials` and the bytes of each patch a case
|
|
416
|
+
* names; the scoring hash covers the bytes of `suite.yaml`.
|
|
417
|
+
*
|
|
418
|
+
* `label` stands for the directory in the issues, which name `<label>/suite.yaml` and a path in it, or
|
|
419
|
+
* `<label>/cases`; no issue quotes a file.
|
|
420
|
+
*
|
|
421
|
+
* @throws ConfigError with the first problem of the bytes, the text or the YAML of `suite.yaml`, or every
|
|
422
|
+
* issue its data has; or the repeated ids and the problem of `cases/`; or every other issue found
|
|
423
|
+
*/
|
|
424
|
+
export async function loadSuite(directory, label = directory) {
|
|
425
|
+
const file = `${label}/suite.yaml`;
|
|
426
|
+
const { header, bytes } = await readHeader(directory, file);
|
|
427
|
+
const at = (index, ...segments) => configPath(['cases', index, ...segments], file);
|
|
428
|
+
const ids = repeatedIds(header.cases, at);
|
|
429
|
+
const casesIssue = await casesProblem(directory);
|
|
430
|
+
if (casesIssue !== undefined) {
|
|
431
|
+
throw new ConfigError([
|
|
432
|
+
...ids,
|
|
433
|
+
{ path: `${label}/cases`, message: casesIssue },
|
|
434
|
+
]);
|
|
435
|
+
}
|
|
436
|
+
const { patches, issues } = await readPatches(directory, header.cases, at);
|
|
437
|
+
const found = [
|
|
438
|
+
...ids,
|
|
439
|
+
...issues,
|
|
440
|
+
...spanIssues(header.cases, patches, header.window, at),
|
|
276
441
|
];
|
|
442
|
+
if (found.length > 0)
|
|
443
|
+
throw new ConfigError(found);
|
|
277
444
|
return {
|
|
278
445
|
name: header.name,
|
|
279
446
|
trials: header.trials,
|
|
280
447
|
window: header.window,
|
|
281
|
-
// The kind rules of `caseSchema` make each case a `BugCase` or
|
|
448
|
+
// The kind rules of `caseSchema` make each case a `BugCase`, a `ControlCase` or an `AttackCase`.
|
|
282
449
|
cases: header.cases,
|
|
283
|
-
patches: new Map([...patches].map(([name, { text
|
|
284
|
-
|
|
450
|
+
patches: new Map([...patches].map(([name, { text }]) => [name, text])),
|
|
451
|
+
reviewHash: reviewHashOf(header, patches),
|
|
452
|
+
scoringHash: `sha256:${sha256(bytes)}`,
|
|
285
453
|
};
|
|
286
454
|
}
|
package/dist/cli/bench.d.ts
CHANGED
|
@@ -11,21 +11,32 @@ export declare const BENCH_USAGE: string;
|
|
|
11
11
|
* It reviews every case of the suite the argument names with every `--config` (the usual resolution when
|
|
12
12
|
* there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
|
|
13
13
|
* in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
|
|
14
|
-
* at a time
|
|
14
|
+
* at a time, and a review that a model's reply denied (an unusable, refused or cut off research reply) as
|
|
15
|
+
* denied. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
|
|
15
16
|
* stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
|
|
16
17
|
* on stdout, which it also writes, with `report.json`, into the run directory.
|
|
17
18
|
*
|
|
18
19
|
* A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
|
|
19
20
|
* runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
|
|
20
|
-
* warning. A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
21
|
+
* warning. A suite whose scoring hash alone changed (`suite.yaml`, but not a case id, `trials` or a patch)
|
|
22
|
+
* re-scores the run: its records are scored under the suite as it is now, with no call, `run.json` records
|
|
23
|
+
* the new scoring hash beside the one the run began with, and the plan and the reports say so while the
|
|
24
|
+
* suite is not the one the run began with. A run directory of another suite, another quorum or other
|
|
25
|
+
* configurations, one that holds files of another kind, a suite whose case ids, `trials` or patches
|
|
26
|
+
* changed, and a suite or a configuration that does not validate or cannot be built are refused before
|
|
27
|
+
* the first call.
|
|
23
28
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
+
* `--report-only` runs no review: it scores what the run directory records, writes `run.json` again when
|
|
30
|
+
* the scoring hash changed, writes the reports and exits 0, whatever is recorded. It never opens
|
|
31
|
+
* `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
|
|
32
|
+
* holds no run, and `--retry-incomplete`, which runs reviews.
|
|
33
|
+
*
|
|
34
|
+
* Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
|
|
35
|
+
* reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
|
|
36
|
+
* be written or read), 4 when a review failed (its research call failed, or its record was too large to
|
|
37
|
+
* write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
|
|
38
|
+
* reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
|
|
39
|
+
* checks and the plan leaves the run directory as it was, even one that does not exist yet.
|
|
29
40
|
*
|
|
30
41
|
* Every line it writes is inert (invariant I3).
|
|
31
42
|
*/
|