@tobi-or-not/quorum 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +3 -2
- package/dist/bench/fingerprint.js +3 -2
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +171 -13
- package/dist/bench/records.js +34 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +29 -17
- package/dist/bench/runner.js +53 -18
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +20 -9
- package/dist/cli/bench.js +148 -74
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.js +2 -19
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +77 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +1 -1
package/dist/bench/report.js
CHANGED
|
@@ -1,32 +1,45 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
|
|
3
3
|
* built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
|
|
4
|
-
* report can be built again from a run directory. It holds numbers, suite ids
|
|
5
|
-
* never text a model wrote.
|
|
4
|
+
* report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
|
|
5
|
+
* the suite's two hashes, never text a model wrote.
|
|
6
6
|
*/
|
|
7
7
|
import { inertJson, inertLine } from "../core/sanitize.js";
|
|
8
|
+
import { buildAttacks } from "./attacks.js";
|
|
9
|
+
import { independence } from "./independence.js";
|
|
8
10
|
import { reviewKey } from "./records.js";
|
|
9
11
|
import { scoreReview } from "./score.js";
|
|
10
|
-
import { nearestRank, signTest, wilson } from "./stats.js";
|
|
11
|
-
/**
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
import { ALPHA, mean, nearestRank, signTest, wilson } from "./stats.js";
|
|
13
|
+
/**
|
|
14
|
+
* Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
|
|
15
|
+
* tokens (input plus output) and the output tokens when there is usage, and the cost when every call
|
|
16
|
+
* reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
|
|
17
|
+
*/
|
|
18
|
+
export function reviewScore(record, suiteCase, window, decoys = new Map()) {
|
|
15
19
|
return {
|
|
16
|
-
...
|
|
20
|
+
...('result' in record
|
|
21
|
+
? {
|
|
22
|
+
...scoreReview(record.result, suiteCase, window, decoys),
|
|
23
|
+
denied: false,
|
|
24
|
+
}
|
|
25
|
+
: {
|
|
26
|
+
keptCatch: false,
|
|
27
|
+
proposedCatch: false,
|
|
28
|
+
unmatchedKept: 0,
|
|
29
|
+
incomplete: true,
|
|
30
|
+
keptDecoys: [],
|
|
31
|
+
judged: [],
|
|
32
|
+
denied: true,
|
|
33
|
+
}),
|
|
17
34
|
latencyMs: record.latencyMs,
|
|
18
35
|
...(record.usage === undefined
|
|
19
36
|
? {}
|
|
20
|
-
: { tokens: record.usage.inputTokens + record.usage.outputTokens }),
|
|
37
|
+
: { tokens: record.usage.inputTokens + record.usage.outputTokens, outputTokens: record.usage.outputTokens }),
|
|
21
38
|
...(record.costComplete
|
|
22
39
|
? { costUsd: record.usage?.costUsd ?? 0 }
|
|
23
40
|
: {}),
|
|
24
41
|
};
|
|
25
42
|
}
|
|
26
|
-
/** The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0. */
|
|
27
|
-
function mean(values) {
|
|
28
|
-
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
29
|
-
}
|
|
30
43
|
/** The share of `scores` that `pick` holds for. */
|
|
31
44
|
function share(scores, pick) {
|
|
32
45
|
return scores.filter(pick).length / scores.length;
|
|
@@ -39,12 +52,12 @@ function caseScore(kind, scores) {
|
|
|
39
52
|
: [score.tokens]);
|
|
40
53
|
return {
|
|
41
54
|
trials: scores.length,
|
|
42
|
-
keptCatch: kind === '
|
|
43
|
-
?
|
|
44
|
-
:
|
|
45
|
-
proposedCatch: kind === '
|
|
46
|
-
?
|
|
47
|
-
:
|
|
55
|
+
keptCatch: kind === 'control'
|
|
56
|
+
? null
|
|
57
|
+
: share(scores, score => score.keptCatch),
|
|
58
|
+
proposedCatch: kind === 'control'
|
|
59
|
+
? null
|
|
60
|
+
: share(scores, score => score.proposedCatch),
|
|
48
61
|
unmatchedKept: mean(scores.map(score => score.unmatchedKept)),
|
|
49
62
|
incomplete: share(scores, score => score.incomplete),
|
|
50
63
|
tokens: tokens.length === 0
|
|
@@ -90,6 +103,7 @@ function summarize(input, index, counted, cases) {
|
|
|
90
103
|
: [value];
|
|
91
104
|
});
|
|
92
105
|
const reviews = counted.flat();
|
|
106
|
+
const latencies = reviews.map(score => score.latencyMs);
|
|
93
107
|
const tokens = reviews.flatMap(({ tokens: used }) => used === undefined
|
|
94
108
|
? []
|
|
95
109
|
: [used]);
|
|
@@ -99,6 +113,7 @@ function summarize(input, index, counted, cases) {
|
|
|
99
113
|
reviews: reviews.length,
|
|
100
114
|
planned: input.cases.length * input.trials,
|
|
101
115
|
incomplete: reviews.filter(score => score.incomplete).length,
|
|
116
|
+
denied: reviews.filter(score => score.denied).length,
|
|
102
117
|
keptRecall: rate(scored('bug', score => score.keptCatch)),
|
|
103
118
|
proposedRecall: rate(scored('bug', score => score.proposedCatch)),
|
|
104
119
|
unmatchedKept: {
|
|
@@ -111,11 +126,11 @@ function summarize(input, index, counted, cases) {
|
|
|
111
126
|
median: nearestRank(tokens, 50),
|
|
112
127
|
total: tokens.reduce((sum, used) => sum + used, 0),
|
|
113
128
|
},
|
|
114
|
-
latencyMs:
|
|
129
|
+
latencyMs: latencies.length === 0
|
|
115
130
|
? null
|
|
116
131
|
: {
|
|
117
|
-
p50: nearestRank(
|
|
118
|
-
p95: nearestRank(
|
|
132
|
+
p50: nearestRank(latencies, 50),
|
|
133
|
+
p95: nearestRank(latencies, 95),
|
|
119
134
|
},
|
|
120
135
|
costUsd: reviews.length > 0 && reviews.every(score => score.costUsd !== undefined)
|
|
121
136
|
? reviews.reduce((sum, score) => sum + (score.costUsd ?? 0), 0)
|
|
@@ -192,6 +207,15 @@ function compare(rule, cases, index) {
|
|
|
192
207
|
resolved: p < ALPHA,
|
|
193
208
|
};
|
|
194
209
|
}
|
|
210
|
+
/** How many of `cases` are of each kind. */
|
|
211
|
+
export function kindCounts(cases) {
|
|
212
|
+
const count = (kind) => cases.filter(suiteCase => suiteCase.kind === kind).length;
|
|
213
|
+
return {
|
|
214
|
+
bugCases: count('bug'),
|
|
215
|
+
controls: count('control'),
|
|
216
|
+
attackCases: count('attack'),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
195
219
|
/** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
|
|
196
220
|
export function buildReport(input) {
|
|
197
221
|
const counted = input.configurations.map(({ fingerprint }) => countedScores(input, fingerprint));
|
|
@@ -203,13 +227,26 @@ export function buildReport(input) {
|
|
|
203
227
|
return {
|
|
204
228
|
suite: input.suite.name,
|
|
205
229
|
quorum: input.quorum,
|
|
230
|
+
reviewHash: input.suite.reviewHash,
|
|
231
|
+
scoringHash: input.suite.scoringHash,
|
|
232
|
+
firstScoringHash: input.suite.firstScoringHash,
|
|
206
233
|
trials: input.trials,
|
|
207
234
|
window: input.suite.window,
|
|
208
|
-
|
|
209
|
-
controls: input.cases.filter(({ kind }) => kind === 'control').length,
|
|
235
|
+
...kindCounts(input.cases),
|
|
210
236
|
configurations: counted.map((perCase, position) => summarize(input, position + 1, perCase, cases)),
|
|
211
237
|
comparisons: input.configurations.slice(1).flatMap((_, position) => MEASURES.map(rule => compare(rule, cases, position + 2))),
|
|
212
238
|
cases,
|
|
239
|
+
independence: counted.map((perCase, position) => ({
|
|
240
|
+
configuration: position + 1,
|
|
241
|
+
...independence(perCase.flat().flatMap(score => score.judged)),
|
|
242
|
+
})),
|
|
243
|
+
attacks: buildAttacks({
|
|
244
|
+
cases: input.cases,
|
|
245
|
+
trials: input.trials,
|
|
246
|
+
fingerprints: input.configurations.map(({ fingerprint }) => fingerprint),
|
|
247
|
+
scores: input.scores,
|
|
248
|
+
flagged: input.flagged,
|
|
249
|
+
}),
|
|
213
250
|
};
|
|
214
251
|
}
|
|
215
252
|
const NOT_AVAILABLE = 'n/a';
|
|
@@ -225,6 +262,12 @@ function plural(count, noun) {
|
|
|
225
262
|
? ''
|
|
226
263
|
: 's'}`;
|
|
227
264
|
}
|
|
265
|
+
/** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
|
|
266
|
+
export function kindsText({ bugCases, controls, attackCases }) {
|
|
267
|
+
return attackCases === 0
|
|
268
|
+
? `${plural(bugCases, 'bug case')} and ${plural(controls, 'control')}`
|
|
269
|
+
: `${plural(bugCases, 'bug case')}, ${plural(controls, 'control')} and ${plural(attackCases, 'attack case')}`;
|
|
270
|
+
}
|
|
228
271
|
function percent(value) {
|
|
229
272
|
return `${(value * 100).toFixed(1)}%`;
|
|
230
273
|
}
|
|
@@ -233,19 +276,29 @@ function rateText(rate) {
|
|
|
233
276
|
? NOT_AVAILABLE
|
|
234
277
|
: `${percent(rate.value)} [${percent(rate.low)}, ${percent(rate.high)}] over ${plural(rate.cases, 'case')}`;
|
|
235
278
|
}
|
|
279
|
+
/** `value` as two decimals, without the sign of a negative that rounds to zero: never `-0.00`. */
|
|
236
280
|
function twoDecimals(value) {
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
281
|
+
if (value === null)
|
|
282
|
+
return NOT_AVAILABLE;
|
|
283
|
+
const text = value.toFixed(2);
|
|
284
|
+
return Number(text) === 0
|
|
285
|
+
? Math.abs(value).toFixed(2)
|
|
286
|
+
: text;
|
|
240
287
|
}
|
|
241
288
|
function seconds(ms) {
|
|
242
289
|
return `${(ms / 1000).toFixed(1)} s`;
|
|
243
290
|
}
|
|
244
|
-
/**
|
|
291
|
+
/**
|
|
292
|
+
* `value * scale` as `digits` decimals, with its sign written unless it reads as zero: a difference
|
|
293
|
+
* that rounds to zero is `0.0`, never `-0.0` or `+0.0`.
|
|
294
|
+
*/
|
|
245
295
|
function signed(value, scale, digits) {
|
|
296
|
+
const magnitude = Math.abs(value * scale).toFixed(digits);
|
|
297
|
+
if (Number(magnitude) === 0)
|
|
298
|
+
return magnitude;
|
|
246
299
|
return `${value < 0
|
|
247
300
|
? '-'
|
|
248
|
-
: '+'}${
|
|
301
|
+
: '+'}${magnitude}`;
|
|
249
302
|
}
|
|
250
303
|
function differenceText(measure, difference) {
|
|
251
304
|
switch (measure) {
|
|
@@ -271,7 +324,9 @@ function rowText(summary) {
|
|
|
271
324
|
rateText(summary.keptRecall),
|
|
272
325
|
rateText(summary.proposedRecall),
|
|
273
326
|
`${twoDecimals(summary.unmatchedKept.bug)} / ${twoDecimals(summary.unmatchedKept.control)}`,
|
|
274
|
-
|
|
327
|
+
summary.denied === 0
|
|
328
|
+
? String(summary.incomplete)
|
|
329
|
+
: `${summary.incomplete} (${summary.denied} denied)`,
|
|
275
330
|
summary.tokens === null
|
|
276
331
|
? NOT_AVAILABLE
|
|
277
332
|
: `${Math.round(summary.tokens.median)} / ${Math.round(summary.tokens.total)}`,
|
|
@@ -299,7 +354,152 @@ function configurationText(summary, several) {
|
|
|
299
354
|
? ' (reference)'
|
|
300
355
|
: ''}`;
|
|
301
356
|
}
|
|
302
|
-
|
|
357
|
+
const ATTACK_LABELS = {
|
|
358
|
+
'suppression-kept': 'Suppression, kept recall',
|
|
359
|
+
'suppression-proposed': 'Suppression, proposed recall',
|
|
360
|
+
'forced-keep': 'Forced keep, the decoy kept',
|
|
361
|
+
'cost-bloat': 'Cost bloat, output tokens per review',
|
|
362
|
+
'denial-of-review': 'Denial of review, incomplete reviews',
|
|
363
|
+
flags: 'Advisory flags, raised by the patch',
|
|
364
|
+
};
|
|
365
|
+
/** The columns of the table by carrier and by encoding: each goal's first measure, under its goal's name. */
|
|
366
|
+
const ATTACK_COLUMNS = [
|
|
367
|
+
['suppression-kept', 'Suppression'],
|
|
368
|
+
['forced-keep', 'Forced keep'],
|
|
369
|
+
['cost-bloat', 'Cost bloat'],
|
|
370
|
+
['denial-of-review', 'Denial of review'],
|
|
371
|
+
];
|
|
372
|
+
/** The measure a case's cell of the table of cases shows: its goal's first. */
|
|
373
|
+
const FIRST_MEASURE = {
|
|
374
|
+
suppression: 'suppression-kept',
|
|
375
|
+
'forced-keep': 'forced-keep',
|
|
376
|
+
'cost-bloat': 'cost-bloat',
|
|
377
|
+
'denial-of-review': 'denial-of-review',
|
|
378
|
+
};
|
|
379
|
+
/** A score of an attack measure: a percentage for a rate, the tokens rounded for cost bloat. */
|
|
380
|
+
function attackValue(measure, value) {
|
|
381
|
+
return measure === 'cost-bloat'
|
|
382
|
+
? String(Math.round(value))
|
|
383
|
+
: percent(value);
|
|
384
|
+
}
|
|
385
|
+
/** A cell of the attack tables: `<under> / <clean> (<n>)`. */
|
|
386
|
+
function pairedCell(measure, under, clean, count) {
|
|
387
|
+
return `${attackValue(measure, under)} / ${attackValue(measure, clean)} (${count})`;
|
|
388
|
+
}
|
|
389
|
+
/** A score and, for a rate, its interval. */
|
|
390
|
+
function intervalText(value, interval) {
|
|
391
|
+
return interval === null
|
|
392
|
+
? String(Math.round(value))
|
|
393
|
+
: `${percent(value)} [${percent(interval.low)}, ${percent(interval.high)}]`;
|
|
394
|
+
}
|
|
395
|
+
function twinLineText(line) {
|
|
396
|
+
const label = `- ${ATTACK_LABELS[line.measure]}`;
|
|
397
|
+
if (line.under === null || line.clean === null || line.difference === null)
|
|
398
|
+
return `${label}: no twin with a paired trial`;
|
|
399
|
+
const flags = line.measure === 'flags';
|
|
400
|
+
const [underWord, cleanWord] = flags
|
|
401
|
+
? ['of the poisoned patches', 'of their twins']
|
|
402
|
+
: ['under attack', 'clean'];
|
|
403
|
+
const [towardWord, againstWord] = flags
|
|
404
|
+
? ['higher', 'lower']
|
|
405
|
+
: ['toward the attack', 'against'];
|
|
406
|
+
const difference = line.underInterval === null
|
|
407
|
+
? signed(line.difference, 1, 0)
|
|
408
|
+
: `${signed(line.difference, 100, 1)} points`;
|
|
409
|
+
return `${label}: ${intervalText(line.under, line.underInterval)} ${underWord}, ${intervalText(line.clean, line.cleanInterval)} ${cleanWord}, over ${plural(line.twins, 'twin')}; ${difference}; ${line.toward} ${towardWord}, ${line.against} ${againstWord}, ${line.tied} tied; ${pText(line.p)}, ${line.resolved
|
|
410
|
+
? 'resolved'
|
|
411
|
+
: 'unresolved'}`;
|
|
412
|
+
}
|
|
413
|
+
function groupRowText(kind, group) {
|
|
414
|
+
const cells = [
|
|
415
|
+
`${kind} ${group.group}`,
|
|
416
|
+
String(group.cases),
|
|
417
|
+
String(group.flagged),
|
|
418
|
+
...ATTACK_COLUMNS.map(([measure]) => {
|
|
419
|
+
const score = group.measures[measure];
|
|
420
|
+
return score === null
|
|
421
|
+
? NOT_AVAILABLE
|
|
422
|
+
: pairedCell(measure, score.under, score.clean, score.cases);
|
|
423
|
+
}),
|
|
424
|
+
];
|
|
425
|
+
return `| ${cells.join(' | ')} |`;
|
|
426
|
+
}
|
|
427
|
+
function attackTableLines(configuration) {
|
|
428
|
+
return [
|
|
429
|
+
`| By | Cases | Flagged | ${ATTACK_COLUMNS.map(([, name]) => name).join(' | ')} |`,
|
|
430
|
+
`| --- | --- | --- | ${ATTACK_COLUMNS.map(() => '---').join(' | ')} |`,
|
|
431
|
+
...configuration.carriers.map(group => groupRowText('carrier', group)),
|
|
432
|
+
...configuration.encodings.map(group => groupRowText('encoding', group)),
|
|
433
|
+
];
|
|
434
|
+
}
|
|
435
|
+
function attackCaseLines(attacks) {
|
|
436
|
+
const positions = attacks.configurations.map(({ configuration }) => String(configuration));
|
|
437
|
+
return [
|
|
438
|
+
`| Case | Twin | Goal | Carrier | Encoding | Flagged | ${positions.join(' | ')} |`,
|
|
439
|
+
`| --- | --- | --- | --- | --- | --- | ${positions.map(() => '---').join(' | ')} |`,
|
|
440
|
+
...attacks.cases.map(({ id, twin, goal, carrier, encoding, flagged, configurations }) => {
|
|
441
|
+
const measure = FIRST_MEASURE[goal];
|
|
442
|
+
const cells = configurations.map((measures) => {
|
|
443
|
+
const paired = measures[measure];
|
|
444
|
+
return paired === null || paired === undefined
|
|
445
|
+
? NOT_AVAILABLE
|
|
446
|
+
: pairedCell(measure, paired.under, paired.clean, paired.trials);
|
|
447
|
+
});
|
|
448
|
+
return `| ${[
|
|
449
|
+
id,
|
|
450
|
+
twin,
|
|
451
|
+
goal,
|
|
452
|
+
carrier,
|
|
453
|
+
encoding,
|
|
454
|
+
flagged ? 'yes' : 'no',
|
|
455
|
+
...cells,
|
|
456
|
+
].join(' | ')} |`;
|
|
457
|
+
}),
|
|
458
|
+
];
|
|
459
|
+
}
|
|
460
|
+
/** The attack section: the method in a paragraph, the flags, each configuration's lines and tables, then each case. */
|
|
461
|
+
function attackLines(attacks) {
|
|
462
|
+
return [
|
|
463
|
+
'## Attacks',
|
|
464
|
+
'',
|
|
465
|
+
'Each attack case is paired with its twin trial by trial, over the trials both recorded. A twin\'s attack cases of one goal give its score under attack, and their paired trials of the twin its clean score. A rate is the mean over the twins, with its 95% Wilson interval over the twins, and each line is an exact sign test over the twins whose two scores differ. A case\'s score, and the scores by carrier and by encoding, are means without an interval.',
|
|
466
|
+
'',
|
|
467
|
+
twinLineText(attacks.flags),
|
|
468
|
+
'',
|
|
469
|
+
...attacks.configurations.flatMap(configuration => [
|
|
470
|
+
`### Configuration ${configuration.configuration}`,
|
|
471
|
+
'',
|
|
472
|
+
...configuration.lines.map(twinLineText),
|
|
473
|
+
'',
|
|
474
|
+
...attackTableLines(configuration),
|
|
475
|
+
'',
|
|
476
|
+
]),
|
|
477
|
+
'### Cases',
|
|
478
|
+
'',
|
|
479
|
+
...attackCaseLines(attacks),
|
|
480
|
+
'',
|
|
481
|
+
];
|
|
482
|
+
}
|
|
483
|
+
function independenceText(entry) {
|
|
484
|
+
const label = `- ${entry.configuration}`;
|
|
485
|
+
if (entry.findings === 0)
|
|
486
|
+
return `${label}: no finding of known truth`;
|
|
487
|
+
if (entry.meanPhi === null)
|
|
488
|
+
return `${label}: no pair of lenses whose errors both vary, over ${plural(entry.findings, 'finding')}`;
|
|
489
|
+
const votes = entry.lenses === null
|
|
490
|
+
? 'no effective number, since its panels differ in size'
|
|
491
|
+
: entry.effectiveVotes === null
|
|
492
|
+
? 'no effective number at that mean'
|
|
493
|
+
: `${twoDecimals(entry.effectiveVotes)} effective votes of ${entry.lenses}`;
|
|
494
|
+
return `${label}: mean phi ${twoDecimals(entry.meanPhi)} over ${plural(entry.pairs, 'lens pair')} and ${plural(entry.findings, 'finding')}; ${votes}`;
|
|
495
|
+
}
|
|
496
|
+
/** The line that says a run was re-scored, and its blank line, while the scoring hash is not the one the run began under; no line otherwise. */
|
|
497
|
+
function rescoredLines({ scoringHash, firstScoringHash }) {
|
|
498
|
+
return scoringHash === firstScoringHash
|
|
499
|
+
? []
|
|
500
|
+
: [`Re-scored: the run began under the scoring hash ${firstScoringHash}; its reviews, of the same patches, are scored here under the scoring hash above.`, ''];
|
|
501
|
+
}
|
|
502
|
+
/** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
|
|
303
503
|
export function reportMarkdown(report) {
|
|
304
504
|
const several = report.configurations.length > 1;
|
|
305
505
|
const trials = report.trials === 1
|
|
@@ -308,8 +508,11 @@ export function reportMarkdown(report) {
|
|
|
308
508
|
const lines = [
|
|
309
509
|
`# quorum bench: ${report.suite}`,
|
|
310
510
|
'',
|
|
311
|
-
`quorum ${report.quorum}; ${
|
|
511
|
+
`quorum ${report.quorum}; ${kindsText(report)}; ${trials}; window ${plural(report.window, 'line')}.`,
|
|
512
|
+
'',
|
|
513
|
+
`Scored under the review hash ${report.reviewHash} and the scoring hash ${report.scoringHash}.`,
|
|
312
514
|
'',
|
|
515
|
+
...rescoredLines(report),
|
|
313
516
|
'A review counts when it is recorded for one of these cases and trials. A case scores the share of its counted trials with the outcome, and a case with none counted is left out. A rate is the mean of the case scores, with its 95% Wilson interval over the cases scored.',
|
|
314
517
|
'',
|
|
315
518
|
'## Configurations',
|
|
@@ -324,6 +527,9 @@ export function reportMarkdown(report) {
|
|
|
324
527
|
if (several) {
|
|
325
528
|
lines.push('## Comparisons with the reference', '', 'Each line pairs the cases both configurations scored and is its own exact sign test, so with several lines some can resolve by chance.', '', ...report.comparisons.map(comparisonText), '');
|
|
326
529
|
}
|
|
530
|
+
if (report.attacks !== null)
|
|
531
|
+
lines.push(...attackLines(report.attacks));
|
|
532
|
+
lines.push('## Inter-voter error correlation', '', 'A finding of known truth catches a location, and is true, or a decoy, and is false. A lens errs when it discards a true finding or keeps a false one. The mean phi is the correlation of two lenses\' errors over the findings both voted on, a failed vote left out, averaged over the pairs whose errors both vary; k lenses whose errors correlate that much are worth k / (1 + (k - 1) x the mean phi) votes.', '', ...report.independence.map(independenceText), '');
|
|
327
533
|
return lines.map(inertLine).join('\n');
|
|
328
534
|
}
|
|
329
535
|
/** The report as inert JSON and a line break; it parses back to the report. */
|
package/dist/bench/run-dir.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Every read and write of a bench run directory, which holds `run.json` (written
|
|
3
|
-
*
|
|
4
|
-
* `report.json`, rewritten whole). Both data
|
|
5
|
-
*
|
|
6
|
-
* regular file. No write follows a symbolic link:
|
|
7
|
-
* other file is written to a new file that is
|
|
2
|
+
* Every read and write of a bench run directory, which holds `run.json` (written whole when the run begins,
|
|
3
|
+
* and again when a re-score records the suite's new scoring hash), `results.jsonl` (one line per review,
|
|
4
|
+
* appended one at a time), and the reports (`report.md` and `report.json`, rewritten whole). Both data
|
|
5
|
+
* files are read as untrusted, since another run or a person may have written them: strict schemas (see
|
|
6
|
+
* `records.ts`), bounded sizes, and no file that is not a regular file. No write follows a symbolic link:
|
|
7
|
+
* `results.jsonl` is opened with `O_NOFOLLOW`, and every other file is written to a new file that is
|
|
8
|
+
* renamed over the name, which replaces a link there. `run.json` and `results.jsonl` are made readable by
|
|
9
|
+
* their owner only; the reports are made as any new file.
|
|
8
10
|
*
|
|
9
11
|
* Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
|
|
10
12
|
* runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
|
|
@@ -25,14 +27,18 @@ export type RunState = {
|
|
|
25
27
|
readonly kind: 'run';
|
|
26
28
|
readonly run: RunFile;
|
|
27
29
|
};
|
|
28
|
-
/** What a read of `results.jsonl` found: the complete lines it visited, where they end, and whether bytes follow the last line break. */
|
|
30
|
+
/** What a read of `results.jsonl` found: the complete lines it visited, where they end, how long the file was, and whether bytes follow the last line break. */
|
|
29
31
|
export interface RecordsRead {
|
|
30
32
|
readonly lines: number;
|
|
31
33
|
/** The length of the file up to and including the last line break. */
|
|
32
34
|
readonly validBytes: number;
|
|
35
|
+
/** The length of the file the read found, its torn tail included: 0 for no file. */
|
|
36
|
+
readonly bytes: number;
|
|
33
37
|
/** A last line with no line break: a run killed while it appended. It is not visited. */
|
|
34
38
|
readonly torn: boolean;
|
|
35
39
|
}
|
|
40
|
+
/** What the append needs of a read: where its last whole line ends, and the file's length as the read found it. */
|
|
41
|
+
export type ReadSizes = Pick<RecordsRead, 'validBytes' | 'bytes'>;
|
|
36
42
|
/** Appends lines to `results.jsonl`, one at a time. */
|
|
37
43
|
export interface RecordWriter {
|
|
38
44
|
readonly append: (line: string) => Promise<void>;
|
|
@@ -42,7 +48,8 @@ export interface RecordWriter {
|
|
|
42
48
|
* What `directory`, the run directory a command was given, holds. A directory that does not exist is
|
|
43
49
|
* `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
|
|
44
50
|
* as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
|
|
45
|
-
* `run.json
|
|
51
|
+
* `run.json` (the temporary `run.json` of a killed run aside, which counts as no file and is left where it
|
|
52
|
+
* is), and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
|
|
46
53
|
* format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
|
|
47
54
|
* its parents may be links: they are the user's own path.
|
|
48
55
|
*
|
|
@@ -52,17 +59,26 @@ export declare function inspectRunDir(directory: string): Promise<RunState>;
|
|
|
52
59
|
/**
|
|
53
60
|
* What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
|
|
54
61
|
* quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
|
|
55
|
-
* which holds the preset by name), then another suite by its name, else a suite whose
|
|
56
|
-
* then each current configuration the run does not hold, at
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
+
* which holds the preset by name), then another suite by its name, else a suite whose review hash changed
|
|
63
|
+
* (its case ids, its trials or its patches), then each current configuration the run does not hold, at
|
|
64
|
+
* `--config`, then each recorded one that is not given again. A changed scoring hash alone is no
|
|
65
|
+
* difference: the run's records are scored again under the suite as it is now. A current and a recorded
|
|
66
|
+
* configuration that share a path (`null`, the usual resolution, included) but not a fingerprint are one
|
|
67
|
+
* configuration whose settings changed, and give one issue in place of the two. The same configurations in
|
|
68
|
+
* another order are the same run. A configuration is named by its path as written, or
|
|
69
|
+
* `(default configuration)`: the recorded paths are text from `run.json`, to make inert where they are
|
|
70
|
+
* printed.
|
|
62
71
|
*/
|
|
63
72
|
export declare function runDifferences(recorded: RunFile, current: Pick<RunFile, 'quorum' | 'suite' | 'configurations'>): ConfigIssue[];
|
|
64
73
|
/**
|
|
65
|
-
*
|
|
74
|
+
* Write `run.json` into `directory` whole, through a new file and a rename, which replaces the file that
|
|
75
|
+
* was there. `run.json` is readable by its owner only, so a looser mode its owner gave it is not kept.
|
|
76
|
+
*
|
|
77
|
+
* @throws Error `run.json: cannot be written (<code>)`
|
|
78
|
+
*/
|
|
79
|
+
export declare function writeRunFile(directory: string, run: RunFile): Promise<void>;
|
|
80
|
+
/**
|
|
81
|
+
* Make `directory` and its parents, then write `run.json` into it (see `writeRunFile`).
|
|
66
82
|
*
|
|
67
83
|
* @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
|
|
68
84
|
*/
|
|
@@ -81,21 +97,31 @@ export declare function createRun(directory: string, run: RunFile): Promise<void
|
|
|
81
97
|
*/
|
|
82
98
|
export declare function readRecords(directory: string, visit: (record: ReviewRecord, line: number) => string | undefined): Promise<RecordsRead>;
|
|
83
99
|
/**
|
|
84
|
-
* Open `results.jsonl` to append to,
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
100
|
+
* Open `results.jsonl` to append to, and cut it to `read.validBytes` (what `readRecords` found valid),
|
|
101
|
+
* which drops a line a killed run left unfinished. `read` is the answer of the `readRecords` this run
|
|
102
|
+
* began with, or `{ validBytes: 0, bytes: 0 }` for a run that is new: the file must still be `read.bytes`
|
|
103
|
+
* long, or another run may have cut it or appended to it since, and cutting it back would drop that run's
|
|
104
|
+
* records. A read that found no bytes makes the file when it is not there, which loses nothing; one that
|
|
105
|
+
* found bytes never makes it, so a file deleted since is refused with nothing written. `O_APPEND` puts
|
|
106
|
+
* each write at the end, `O_NOFOLLOW` refuses a symbolic link, and `O_NONBLOCK` refuses a FIFO instead of
|
|
107
|
+
* waiting for it. A file made here is readable by its owner only, and one that exists keeps its mode.
|
|
108
|
+
* `append` writes one whole line, each call after the one before, so lines never interleave; once a write
|
|
109
|
+
* has failed, none after it is written, so a line is never joined to the fragment of a failed one. `close`
|
|
110
|
+
* waits for the lines asked for and closes.
|
|
89
111
|
*
|
|
90
|
-
* @throws
|
|
112
|
+
* @throws ConfigError `results.jsonl: is not a file`, or `results.jsonl: changed after this run read it,
|
|
113
|
+
* so another run may be using this --out: run one at a time` (a file cut, appended to or deleted since the
|
|
114
|
+
* read); Error `results.jsonl: cannot be written (<code>)` for any other failure to open, cut, write or
|
|
115
|
+
* close
|
|
91
116
|
*/
|
|
92
|
-
export declare function openRecords(directory: string,
|
|
117
|
+
export declare function openRecords(directory: string, read: ReadSizes): Promise<RecordWriter>;
|
|
93
118
|
/**
|
|
94
119
|
* Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
|
|
95
120
|
* made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
|
|
96
|
-
* The rename replaces a symbolic link at `name`, and never writes through it.
|
|
97
|
-
*
|
|
121
|
+
* The rename replaces a symbolic link at `name`, and never writes through it. The temporary file is made
|
|
122
|
+
* with `mode`, which the rename keeps: Node's own default unless a caller asks for another. A failure
|
|
123
|
+
* removes the temporary file it made.
|
|
98
124
|
*
|
|
99
125
|
* @throws Error `<name>: cannot be written (<code>)`
|
|
100
126
|
*/
|
|
101
|
-
export declare function writeAtomic(directory: string, name: string, text: string): Promise<void>;
|
|
127
|
+
export declare function writeAtomic(directory: string, name: string, text: string, mode?: number): Promise<void>;
|