@tobi-or-not/quorum 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +3 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +37 -0
  5. package/dist/bench/fingerprint.js +80 -0
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +220 -0
  9. package/dist/bench/records.js +184 -0
  10. package/dist/bench/report.d.ts +143 -0
  11. package/dist/bench/report.js +538 -0
  12. package/dist/bench/run-dir.d.ts +127 -0
  13. package/dist/bench/run-dir.js +458 -0
  14. package/dist/bench/runner.d.ts +92 -0
  15. package/dist/bench/runner.js +195 -0
  16. package/dist/bench/score.d.ts +61 -0
  17. package/dist/bench/score.js +85 -0
  18. package/dist/bench/stats.d.ts +105 -0
  19. package/dist/bench/stats.js +158 -0
  20. package/dist/bench/suite.d.ts +132 -0
  21. package/dist/bench/suite.js +454 -0
  22. package/dist/cli/bench.d.ts +43 -0
  23. package/dist/cli/bench.js +537 -0
  24. package/dist/cli/io.d.ts +52 -0
  25. package/dist/cli/io.js +63 -0
  26. package/dist/cli/main.d.ts +7 -37
  27. package/dist/cli/main.js +18 -85
  28. package/dist/cli/render.d.ts +10 -0
  29. package/dist/cli/render.js +20 -11
  30. package/dist/config/load.d.ts +46 -0
  31. package/dist/config/load.js +56 -14
  32. package/dist/core/diff.d.ts +20 -0
  33. package/dist/core/diff.js +102 -0
  34. package/dist/core/errors.d.ts +6 -0
  35. package/dist/core/errors.js +5 -0
  36. package/dist/core/prompts.d.ts +3 -1
  37. package/dist/core/prompts.js +8 -3
  38. package/dist/core/sanitize.d.ts +16 -1
  39. package/dist/core/sanitize.js +28 -4
  40. package/dist/core/stages/context.d.ts +12 -3
  41. package/dist/core/stages/context.js +23 -3
  42. package/dist/providers/anthropic.js +6 -3
  43. package/dist/providers/claude-code.js +11 -10
  44. package/dist/providers/gemini.js +9 -4
  45. package/dist/providers/ollama.js +6 -4
  46. package/dist/providers/openai-compatible.js +6 -4
  47. package/dist/providers/reply.d.ts +6 -2
  48. package/dist/providers/reply.js +13 -3
  49. package/package.json +2 -2
@@ -0,0 +1,538 @@
1
+ /**
2
+ * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
+ * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
+ * report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
5
+ * the suite's two hashes, never text a model wrote.
6
+ */
7
+ import { inertJson, inertLine } from "../core/sanitize.js";
8
+ import { buildAttacks } from "./attacks.js";
9
+ import { independence } from "./independence.js";
10
+ import { reviewKey } from "./records.js";
11
+ import { scoreReview } from "./score.js";
12
+ import { ALPHA, mean, nearestRank, signTest, wilson } from "./stats.js";
13
+ /**
14
+ * Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
15
+ * tokens (input plus output) and the output tokens when there is usage, and the cost when every call
16
+ * reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
17
+ */
18
+ export function reviewScore(record, suiteCase, window, decoys = new Map()) {
19
+ return {
20
+ ...('result' in record
21
+ ? {
22
+ ...scoreReview(record.result, suiteCase, window, decoys),
23
+ denied: false,
24
+ }
25
+ : {
26
+ keptCatch: false,
27
+ proposedCatch: false,
28
+ unmatchedKept: 0,
29
+ incomplete: true,
30
+ keptDecoys: [],
31
+ judged: [],
32
+ denied: true,
33
+ }),
34
+ latencyMs: record.latencyMs,
35
+ ...(record.usage === undefined
36
+ ? {}
37
+ : { tokens: record.usage.inputTokens + record.usage.outputTokens, outputTokens: record.usage.outputTokens }),
38
+ ...(record.costComplete
39
+ ? { costUsd: record.usage?.costUsd ?? 0 }
40
+ : {}),
41
+ };
42
+ }
43
+ /** The share of `scores` that `pick` holds for. */
44
+ function share(scores, pick) {
45
+ return scores.filter(pick).length / scores.length;
46
+ }
47
+ function caseScore(kind, scores) {
48
+ if (scores.length === 0)
49
+ return null;
50
+ const tokens = scores.flatMap(score => score.tokens === undefined
51
+ ? []
52
+ : [score.tokens]);
53
+ return {
54
+ trials: scores.length,
55
+ keptCatch: kind === 'control'
56
+ ? null
57
+ : share(scores, score => score.keptCatch),
58
+ proposedCatch: kind === 'control'
59
+ ? null
60
+ : share(scores, score => score.proposedCatch),
61
+ unmatchedKept: mean(scores.map(score => score.unmatchedKept)),
62
+ incomplete: share(scores, score => score.incomplete),
63
+ tokens: tokens.length === 0
64
+ ? null
65
+ : mean(tokens),
66
+ };
67
+ }
68
+ /** The scores of the reviews that count for a configuration, per selected case: those recorded for a trial from 1 to `trials`. */
69
+ function countedScores(input, fingerprint) {
70
+ return input.cases.map((suiteCase) => {
71
+ const counted = [];
72
+ for (let trial = 1; trial <= input.trials; trial++) {
73
+ const score = input.scores.get(reviewKey(fingerprint, suiteCase.id, trial));
74
+ if (score !== undefined)
75
+ counted.push(score);
76
+ }
77
+ return counted;
78
+ });
79
+ }
80
+ function rate(values) {
81
+ if (values.length === 0)
82
+ return null;
83
+ const value = mean(values);
84
+ return {
85
+ value,
86
+ ...wilson(value, values.length),
87
+ cases: values.length,
88
+ };
89
+ }
90
+ function meanOrNull(values) {
91
+ return values.length === 0
92
+ ? null
93
+ : mean(values);
94
+ }
95
+ function summarize(input, index, counted, cases) {
96
+ const scored = (kind, pick) => cases.flatMap(({ kind: caseKind, configurations }) => {
97
+ const score = configurations[index - 1];
98
+ const value = caseKind === kind && score !== null
99
+ ? pick(score)
100
+ : null;
101
+ return value === null
102
+ ? []
103
+ : [value];
104
+ });
105
+ const reviews = counted.flat();
106
+ const latencies = reviews.map(score => score.latencyMs);
107
+ const tokens = reviews.flatMap(({ tokens: used }) => used === undefined
108
+ ? []
109
+ : [used]);
110
+ return {
111
+ index,
112
+ path: input.configurations[index - 1].path,
113
+ reviews: reviews.length,
114
+ planned: input.cases.length * input.trials,
115
+ incomplete: reviews.filter(score => score.incomplete).length,
116
+ denied: reviews.filter(score => score.denied).length,
117
+ keptRecall: rate(scored('bug', score => score.keptCatch)),
118
+ proposedRecall: rate(scored('bug', score => score.proposedCatch)),
119
+ unmatchedKept: {
120
+ bug: meanOrNull(scored('bug', score => score.unmatchedKept)),
121
+ control: meanOrNull(scored('control', score => score.unmatchedKept)),
122
+ },
123
+ tokens: tokens.length === 0
124
+ ? null
125
+ : {
126
+ median: nearestRank(tokens, 50),
127
+ total: tokens.reduce((sum, used) => sum + used, 0),
128
+ },
129
+ latencyMs: latencies.length === 0
130
+ ? null
131
+ : {
132
+ p50: nearestRank(latencies, 50),
133
+ p95: nearestRank(latencies, 95),
134
+ },
135
+ costUsd: reviews.length > 0 && reviews.every(score => score.costUsd !== undefined)
136
+ ? reviews.reduce((sum, score) => sum + (score.costUsd ?? 0), 0)
137
+ : null,
138
+ };
139
+ }
140
+ /** The five measures of a comparison, in the order of `Measure`. */
141
+ const MEASURES = [
142
+ {
143
+ measure: 'kept-recall',
144
+ kind: 'bug',
145
+ lowerIsBetter: false,
146
+ pick: score => score.keptCatch,
147
+ },
148
+ {
149
+ measure: 'proposed-recall',
150
+ kind: 'bug',
151
+ lowerIsBetter: false,
152
+ pick: score => score.proposedCatch,
153
+ },
154
+ {
155
+ measure: 'unmatched-kept-per-control',
156
+ kind: 'control',
157
+ lowerIsBetter: true,
158
+ pick: score => score.unmatchedKept,
159
+ },
160
+ {
161
+ measure: 'incomplete',
162
+ lowerIsBetter: true,
163
+ pick: score => score.incomplete,
164
+ },
165
+ {
166
+ measure: 'tokens',
167
+ lowerIsBetter: true,
168
+ pick: score => score.tokens,
169
+ },
170
+ ];
171
+ /** The configuration at `index` (from 1, after the reference) against the reference on one measure, over the cases both scored. */
172
+ function compare(rule, cases, index) {
173
+ const differences = [];
174
+ let better = 0;
175
+ let worse = 0;
176
+ for (const { kind, configurations } of cases) {
177
+ if (rule.kind !== undefined && rule.kind !== kind)
178
+ continue;
179
+ const reference = configurations[0];
180
+ const current = configurations[index - 1];
181
+ const before = reference === null
182
+ ? null
183
+ : rule.pick(reference);
184
+ const after = current === null
185
+ ? null
186
+ : rule.pick(current);
187
+ if (before === null || after === null)
188
+ continue;
189
+ const difference = after - before;
190
+ differences.push(difference);
191
+ if (difference === 0)
192
+ continue;
193
+ if ((difference < 0) === rule.lowerIsBetter)
194
+ better++;
195
+ else
196
+ worse++;
197
+ }
198
+ const p = signTest(better, worse);
199
+ return {
200
+ configuration: index,
201
+ measure: rule.measure,
202
+ difference: meanOrNull(differences),
203
+ better,
204
+ worse,
205
+ tied: differences.length - better - worse,
206
+ p,
207
+ resolved: p < ALPHA,
208
+ };
209
+ }
210
+ /** How many of `cases` are of each kind. */
211
+ export function kindCounts(cases) {
212
+ const count = (kind) => cases.filter(suiteCase => suiteCase.kind === kind).length;
213
+ return {
214
+ bugCases: count('bug'),
215
+ controls: count('control'),
216
+ attackCases: count('attack'),
217
+ };
218
+ }
219
+ /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
220
+ export function buildReport(input) {
221
+ const counted = input.configurations.map(({ fingerprint }) => countedScores(input, fingerprint));
222
+ const cases = input.cases.map((suiteCase, position) => ({
223
+ id: suiteCase.id,
224
+ kind: suiteCase.kind,
225
+ configurations: counted.map(perCase => caseScore(suiteCase.kind, perCase[position])),
226
+ }));
227
+ return {
228
+ suite: input.suite.name,
229
+ quorum: input.quorum,
230
+ reviewHash: input.suite.reviewHash,
231
+ scoringHash: input.suite.scoringHash,
232
+ firstScoringHash: input.suite.firstScoringHash,
233
+ trials: input.trials,
234
+ window: input.suite.window,
235
+ ...kindCounts(input.cases),
236
+ configurations: counted.map((perCase, position) => summarize(input, position + 1, perCase, cases)),
237
+ comparisons: input.configurations.slice(1).flatMap((_, position) => MEASURES.map(rule => compare(rule, cases, position + 2))),
238
+ cases,
239
+ independence: counted.map((perCase, position) => ({
240
+ configuration: position + 1,
241
+ ...independence(perCase.flat().flatMap(score => score.judged)),
242
+ })),
243
+ attacks: buildAttacks({
244
+ cases: input.cases,
245
+ trials: input.trials,
246
+ fingerprints: input.configurations.map(({ fingerprint }) => fingerprint),
247
+ scores: input.scores,
248
+ flagged: input.flagged,
249
+ }),
250
+ };
251
+ }
252
+ const NOT_AVAILABLE = 'n/a';
253
+ const LABELS = {
254
+ 'kept-recall': 'kept recall',
255
+ 'proposed-recall': 'proposed recall',
256
+ 'unmatched-kept-per-control': 'unmatched kept findings per control',
257
+ incomplete: 'incomplete reviews',
258
+ tokens: 'tokens per review',
259
+ };
260
+ function plural(count, noun) {
261
+ return `${count} ${noun}${count === 1
262
+ ? ''
263
+ : 's'}`;
264
+ }
265
+ /** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
266
+ export function kindsText({ bugCases, controls, attackCases }) {
267
+ return attackCases === 0
268
+ ? `${plural(bugCases, 'bug case')} and ${plural(controls, 'control')}`
269
+ : `${plural(bugCases, 'bug case')}, ${plural(controls, 'control')} and ${plural(attackCases, 'attack case')}`;
270
+ }
271
+ function percent(value) {
272
+ return `${(value * 100).toFixed(1)}%`;
273
+ }
274
+ function rateText(rate) {
275
+ return rate === null
276
+ ? NOT_AVAILABLE
277
+ : `${percent(rate.value)} [${percent(rate.low)}, ${percent(rate.high)}] over ${plural(rate.cases, 'case')}`;
278
+ }
279
+ /** `value` as two decimals, without the sign of a negative that rounds to zero: never `-0.00`. */
280
+ function twoDecimals(value) {
281
+ if (value === null)
282
+ return NOT_AVAILABLE;
283
+ const text = value.toFixed(2);
284
+ return Number(text) === 0
285
+ ? Math.abs(value).toFixed(2)
286
+ : text;
287
+ }
288
+ function seconds(ms) {
289
+ return `${(ms / 1000).toFixed(1)} s`;
290
+ }
291
+ /**
292
+ * `value * scale` as `digits` decimals, with its sign written unless it reads as zero: a difference
293
+ * that rounds to zero is `0.0`, never `-0.0` or `+0.0`.
294
+ */
295
+ function signed(value, scale, digits) {
296
+ const magnitude = Math.abs(value * scale).toFixed(digits);
297
+ if (Number(magnitude) === 0)
298
+ return magnitude;
299
+ return `${value < 0
300
+ ? '-'
301
+ : '+'}${magnitude}`;
302
+ }
303
+ function differenceText(measure, difference) {
304
+ switch (measure) {
305
+ case 'kept-recall':
306
+ case 'proposed-recall':
307
+ case 'incomplete':
308
+ return `${signed(difference, 100, 1)} points`;
309
+ case 'unmatched-kept-per-control':
310
+ return signed(difference, 1, 2);
311
+ case 'tokens':
312
+ return signed(difference, 1, 0);
313
+ }
314
+ }
315
+ function pText(p) {
316
+ return p < 0.0001
317
+ ? 'p < 0.0001'
318
+ : `p = ${p.toFixed(4)}`;
319
+ }
320
+ function rowText(summary) {
321
+ const cells = [
322
+ String(summary.index),
323
+ `${summary.reviews} of ${summary.planned}`,
324
+ rateText(summary.keptRecall),
325
+ rateText(summary.proposedRecall),
326
+ `${twoDecimals(summary.unmatchedKept.bug)} / ${twoDecimals(summary.unmatchedKept.control)}`,
327
+ summary.denied === 0
328
+ ? String(summary.incomplete)
329
+ : `${summary.incomplete} (${summary.denied} denied)`,
330
+ summary.tokens === null
331
+ ? NOT_AVAILABLE
332
+ : `${Math.round(summary.tokens.median)} / ${Math.round(summary.tokens.total)}`,
333
+ summary.latencyMs === null
334
+ ? NOT_AVAILABLE
335
+ : `${seconds(summary.latencyMs.p50)} / ${seconds(summary.latencyMs.p95)}`,
336
+ summary.costUsd === null
337
+ ? NOT_AVAILABLE
338
+ : `$${summary.costUsd.toFixed(4)}`,
339
+ ];
340
+ return `| ${cells.join(' | ')} |`;
341
+ }
342
+ function comparisonText(comparison) {
343
+ const label = `- ${comparison.configuration}, ${LABELS[comparison.measure]}`;
344
+ if (comparison.difference === null)
345
+ return `${label}: no case scored by both`;
346
+ const counts = `${comparison.better} better, ${comparison.worse} worse, ${comparison.tied} tied`;
347
+ return `${label}: ${differenceText(comparison.measure, comparison.difference)}; ${counts}; ${pText(comparison.p)}, ${comparison.resolved
348
+ ? 'resolved'
349
+ : 'unresolved'}`;
350
+ }
351
+ function configurationText(summary, several) {
352
+ const name = summary.path ?? '(default configuration)';
353
+ return `${summary.index}. ${name}${several && summary.index === 1
354
+ ? ' (reference)'
355
+ : ''}`;
356
+ }
357
+ const ATTACK_LABELS = {
358
+ 'suppression-kept': 'Suppression, kept recall',
359
+ 'suppression-proposed': 'Suppression, proposed recall',
360
+ 'forced-keep': 'Forced keep, the decoy kept',
361
+ 'cost-bloat': 'Cost bloat, output tokens per review',
362
+ 'denial-of-review': 'Denial of review, incomplete reviews',
363
+ flags: 'Advisory flags, raised by the patch',
364
+ };
365
+ /** The columns of the table by carrier and by encoding: each goal's first measure, under its goal's name. */
366
+ const ATTACK_COLUMNS = [
367
+ ['suppression-kept', 'Suppression'],
368
+ ['forced-keep', 'Forced keep'],
369
+ ['cost-bloat', 'Cost bloat'],
370
+ ['denial-of-review', 'Denial of review'],
371
+ ];
372
+ /** The measure a case's cell of the table of cases shows: its goal's first. */
373
+ const FIRST_MEASURE = {
374
+ suppression: 'suppression-kept',
375
+ 'forced-keep': 'forced-keep',
376
+ 'cost-bloat': 'cost-bloat',
377
+ 'denial-of-review': 'denial-of-review',
378
+ };
379
+ /** A score of an attack measure: a percentage for a rate, the tokens rounded for cost bloat. */
380
+ function attackValue(measure, value) {
381
+ return measure === 'cost-bloat'
382
+ ? String(Math.round(value))
383
+ : percent(value);
384
+ }
385
+ /** A cell of the attack tables: `<under> / <clean> (<n>)`. */
386
+ function pairedCell(measure, under, clean, count) {
387
+ return `${attackValue(measure, under)} / ${attackValue(measure, clean)} (${count})`;
388
+ }
389
+ /** A score and, for a rate, its interval. */
390
+ function intervalText(value, interval) {
391
+ return interval === null
392
+ ? String(Math.round(value))
393
+ : `${percent(value)} [${percent(interval.low)}, ${percent(interval.high)}]`;
394
+ }
395
+ function twinLineText(line) {
396
+ const label = `- ${ATTACK_LABELS[line.measure]}`;
397
+ if (line.under === null || line.clean === null || line.difference === null)
398
+ return `${label}: no twin with a paired trial`;
399
+ const flags = line.measure === 'flags';
400
+ const [underWord, cleanWord] = flags
401
+ ? ['of the poisoned patches', 'of their twins']
402
+ : ['under attack', 'clean'];
403
+ const [towardWord, againstWord] = flags
404
+ ? ['higher', 'lower']
405
+ : ['toward the attack', 'against'];
406
+ const difference = line.underInterval === null
407
+ ? signed(line.difference, 1, 0)
408
+ : `${signed(line.difference, 100, 1)} points`;
409
+ return `${label}: ${intervalText(line.under, line.underInterval)} ${underWord}, ${intervalText(line.clean, line.cleanInterval)} ${cleanWord}, over ${plural(line.twins, 'twin')}; ${difference}; ${line.toward} ${towardWord}, ${line.against} ${againstWord}, ${line.tied} tied; ${pText(line.p)}, ${line.resolved
410
+ ? 'resolved'
411
+ : 'unresolved'}`;
412
+ }
413
+ function groupRowText(kind, group) {
414
+ const cells = [
415
+ `${kind} ${group.group}`,
416
+ String(group.cases),
417
+ String(group.flagged),
418
+ ...ATTACK_COLUMNS.map(([measure]) => {
419
+ const score = group.measures[measure];
420
+ return score === null
421
+ ? NOT_AVAILABLE
422
+ : pairedCell(measure, score.under, score.clean, score.cases);
423
+ }),
424
+ ];
425
+ return `| ${cells.join(' | ')} |`;
426
+ }
427
+ function attackTableLines(configuration) {
428
+ return [
429
+ `| By | Cases | Flagged | ${ATTACK_COLUMNS.map(([, name]) => name).join(' | ')} |`,
430
+ `| --- | --- | --- | ${ATTACK_COLUMNS.map(() => '---').join(' | ')} |`,
431
+ ...configuration.carriers.map(group => groupRowText('carrier', group)),
432
+ ...configuration.encodings.map(group => groupRowText('encoding', group)),
433
+ ];
434
+ }
435
+ function attackCaseLines(attacks) {
436
+ const positions = attacks.configurations.map(({ configuration }) => String(configuration));
437
+ return [
438
+ `| Case | Twin | Goal | Carrier | Encoding | Flagged | ${positions.join(' | ')} |`,
439
+ `| --- | --- | --- | --- | --- | --- | ${positions.map(() => '---').join(' | ')} |`,
440
+ ...attacks.cases.map(({ id, twin, goal, carrier, encoding, flagged, configurations }) => {
441
+ const measure = FIRST_MEASURE[goal];
442
+ const cells = configurations.map((measures) => {
443
+ const paired = measures[measure];
444
+ return paired === null || paired === undefined
445
+ ? NOT_AVAILABLE
446
+ : pairedCell(measure, paired.under, paired.clean, paired.trials);
447
+ });
448
+ return `| ${[
449
+ id,
450
+ twin,
451
+ goal,
452
+ carrier,
453
+ encoding,
454
+ flagged ? 'yes' : 'no',
455
+ ...cells,
456
+ ].join(' | ')} |`;
457
+ }),
458
+ ];
459
+ }
460
+ /** The attack section: the method in a paragraph, the flags, each configuration's lines and tables, then each case. */
461
+ function attackLines(attacks) {
462
+ return [
463
+ '## Attacks',
464
+ '',
465
+ 'Each attack case is paired with its twin trial by trial, over the trials both recorded. A twin\'s attack cases of one goal give its score under attack, and their paired trials of the twin its clean score. A rate is the mean over the twins, with its 95% Wilson interval over the twins, and each line is an exact sign test over the twins whose two scores differ. A case\'s score, and the scores by carrier and by encoding, are means without an interval.',
466
+ '',
467
+ twinLineText(attacks.flags),
468
+ '',
469
+ ...attacks.configurations.flatMap(configuration => [
470
+ `### Configuration ${configuration.configuration}`,
471
+ '',
472
+ ...configuration.lines.map(twinLineText),
473
+ '',
474
+ ...attackTableLines(configuration),
475
+ '',
476
+ ]),
477
+ '### Cases',
478
+ '',
479
+ ...attackCaseLines(attacks),
480
+ '',
481
+ ];
482
+ }
483
+ function independenceText(entry) {
484
+ const label = `- ${entry.configuration}`;
485
+ if (entry.findings === 0)
486
+ return `${label}: no finding of known truth`;
487
+ if (entry.meanPhi === null)
488
+ return `${label}: no pair of lenses whose errors both vary, over ${plural(entry.findings, 'finding')}`;
489
+ const votes = entry.lenses === null
490
+ ? 'no effective number, since its panels differ in size'
491
+ : entry.effectiveVotes === null
492
+ ? 'no effective number at that mean'
493
+ : `${twoDecimals(entry.effectiveVotes)} effective votes of ${entry.lenses}`;
494
+ return `${label}: mean phi ${twoDecimals(entry.meanPhi)} over ${plural(entry.pairs, 'lens pair')} and ${plural(entry.findings, 'finding')}; ${votes}`;
495
+ }
496
+ /** The line that says a run was re-scored, and its blank line, while the scoring hash is not the one the run began under; no line otherwise. */
497
+ function rescoredLines({ scoringHash, firstScoringHash }) {
498
+ return scoringHash === firstScoringHash
499
+ ? []
500
+ : [`Re-scored: the run began under the scoring hash ${firstScoringHash}; its reviews, of the same patches, are scored here under the scoring hash above.`, ''];
501
+ }
502
+ /** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
503
+ export function reportMarkdown(report) {
504
+ const several = report.configurations.length > 1;
505
+ const trials = report.trials === 1
506
+ ? 'trial 1'
507
+ : `trials 1 to ${report.trials}`;
508
+ const lines = [
509
+ `# quorum bench: ${report.suite}`,
510
+ '',
511
+ `quorum ${report.quorum}; ${kindsText(report)}; ${trials}; window ${plural(report.window, 'line')}.`,
512
+ '',
513
+ `Scored under the review hash ${report.reviewHash} and the scoring hash ${report.scoringHash}.`,
514
+ '',
515
+ ...rescoredLines(report),
516
+ 'A review counts when it is recorded for one of these cases and trials. A case scores the share of its counted trials with the outcome, and a case with none counted is left out. A rate is the mean of the case scores, with its 95% Wilson interval over the cases scored.',
517
+ '',
518
+ '## Configurations',
519
+ '',
520
+ ...report.configurations.map(summary => configurationText(summary, several)),
521
+ '',
522
+ '| Configuration | Reviews | Kept recall | Proposed recall | Unmatched kept per case, bug / control | Incomplete | Tokens per review, median / total | Latency, p50 / p95 | Cost |',
523
+ '| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
524
+ ...report.configurations.map(rowText),
525
+ '',
526
+ ];
527
+ if (several) {
528
+ lines.push('## Comparisons with the reference', '', 'Each line pairs the cases both configurations scored and is its own exact sign test, so with several lines some can resolve by chance.', '', ...report.comparisons.map(comparisonText), '');
529
+ }
530
+ if (report.attacks !== null)
531
+ lines.push(...attackLines(report.attacks));
532
+ lines.push('## Inter-voter error correlation', '', 'A finding of known truth catches a location, and is true, or a decoy, and is false. A lens errs when it discards a true finding or keeps a false one. The mean phi is the correlation of two lenses\' errors over the findings both voted on, a failed vote left out, averaged over the pairs whose errors both vary; k lenses whose errors correlate that much are worth k / (1 + (k - 1) x the mean phi) votes.', '', ...report.independence.map(independenceText), '');
533
+ return lines.map(inertLine).join('\n');
534
+ }
535
+ /** The report as inert JSON and a line break; it parses back to the report. */
536
+ export function reportJson(report) {
537
+ return `${inertJson(report)}\n`;
538
+ }
@@ -0,0 +1,127 @@
1
+ /**
2
+ * Every read and write of a bench run directory, which holds `run.json` (written whole when the run begins,
3
+ * and again when a re-score records the suite's new scoring hash), `results.jsonl` (one line per review,
4
+ * appended one at a time), and the reports (`report.md` and `report.json`, rewritten whole). Both data
5
+ * files are read as untrusted, since another run or a person may have written them: strict schemas (see
6
+ * `records.ts`), bounded sizes, and no file that is not a regular file. No write follows a symbolic link:
7
+ * `results.jsonl` is opened with `O_NOFOLLOW`, and every other file is written to a new file that is
8
+ * renamed over the name, which replaces a link there. `run.json` and `results.jsonl` are made readable by
9
+ * their owner only; the reports are made as any new file.
10
+ *
11
+ * Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
12
+ * runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
13
+ * I/O failure is an `Error`.
14
+ */
15
+ import type { ConfigIssue } from '../core/errors.ts';
16
+ import type { ReviewRecord, RunFile } from './records.ts';
17
+ export declare const RUN_FILE = "run.json";
18
+ export declare const RESULTS_FILE = "results.jsonl";
19
+ export declare const REPORT_MARKDOWN = "report.md";
20
+ export declare const REPORT_JSON = "report.json";
21
+ /** What `--out` holds: nothing, an empty directory, or a run. */
22
+ export type RunState = {
23
+ readonly kind: 'none';
24
+ } | {
25
+ readonly kind: 'empty';
26
+ } | {
27
+ readonly kind: 'run';
28
+ readonly run: RunFile;
29
+ };
30
+ /** What a read of `results.jsonl` found: the complete lines it visited, where they end, how long the file was, and whether bytes follow the last line break. */
31
+ export interface RecordsRead {
32
+ readonly lines: number;
33
+ /** The length of the file up to and including the last line break. */
34
+ readonly validBytes: number;
35
+ /** The length of the file the read found, its torn tail included: 0 for no file. */
36
+ readonly bytes: number;
37
+ /** A last line with no line break: a run killed while it appended. It is not visited. */
38
+ readonly torn: boolean;
39
+ }
40
+ /** What the append needs of a read: where its last whole line ends, and the file's length as the read found it. */
41
+ export type ReadSizes = Pick<RecordsRead, 'validBytes' | 'bytes'>;
42
+ /** Appends lines to `results.jsonl`, one at a time. */
43
+ export interface RecordWriter {
44
+ readonly append: (line: string) => Promise<void>;
45
+ readonly close: () => Promise<void>;
46
+ }
47
+ /**
48
+ * What `directory`, the run directory a command was given, holds. A directory that does not exist is
49
+ * `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
50
+ * as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
51
+ * `run.json` (the temporary `run.json` of a killed run aside, which counts as no file and is left where it
52
+ * is), and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
53
+ * format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
54
+ * its parents may be links: they are the user's own path.
55
+ *
56
+ * @throws ConfigError as above; Error when the directory or `run.json` cannot be read
57
+ */
58
+ export declare function inspectRunDir(directory: string): Promise<RunState>;
59
+ /**
60
+ * What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
61
+ * quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
62
+ * which holds the preset by name), then another suite by its name, else a suite whose review hash changed
63
+ * (its case ids, its trials or its patches), then each current configuration the run does not hold, at
64
+ * `--config`, then each recorded one that is not given again. A changed scoring hash alone is no
65
+ * difference: the run's records are scored again under the suite as it is now. A current and a recorded
66
+ * configuration that share a path (`null`, the usual resolution, included) but not a fingerprint are one
67
+ * configuration whose settings changed, and give one issue in place of the two. The same configurations in
68
+ * another order are the same run. A configuration is named by its path as written, or
69
+ * `(default configuration)`: the recorded paths are text from `run.json`, to make inert where they are
70
+ * printed.
71
+ */
72
+ export declare function runDifferences(recorded: RunFile, current: Pick<RunFile, 'quorum' | 'suite' | 'configurations'>): ConfigIssue[];
73
+ /**
74
+ * Write `run.json` into `directory` whole, through a new file and a rename, which replaces the file that
75
+ * was there. `run.json` is readable by its owner only, so a looser mode its owner gave it is not kept.
76
+ *
77
+ * @throws Error `run.json: cannot be written (<code>)`
78
+ */
79
+ export declare function writeRunFile(directory: string, run: RunFile): Promise<void>;
80
+ /**
81
+ * Make `directory` and its parents, then write `run.json` into it (see `writeRunFile`).
82
+ *
83
+ * @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
84
+ */
85
+ export declare function createRun(directory: string, run: RunFile): Promise<void>;
86
+ /**
87
+ * Read `results.jsonl` line by line and hand each record, with its line number from 1, to `visit`, which
88
+ * returns the problem with a record it refuses, or undefined. A file with no `results.jsonl` holds
89
+ * nothing. Bytes after the last line break are a line a run killed while it appended cut short: it is
90
+ * reported as `torn`, never visited, and the next append must drop it (see `openRecords`). Any other line
91
+ * that is no record, a line over 16 MiB (complete or not), a record `visit` refuses, a link and a file
92
+ * that is none are refused. The file is read in chunks of 64 KiB and split on the line feed byte, and a
93
+ * line over the limit is refused as soon as it is seen to pass it, so no line is read whole before it is
94
+ * known to fit.
95
+ *
96
+ * @throws ConfigError as above; Error `results.jsonl: cannot be read (<code>)` when a read fails
97
+ */
98
+ export declare function readRecords(directory: string, visit: (record: ReviewRecord, line: number) => string | undefined): Promise<RecordsRead>;
99
+ /**
100
+ * Open `results.jsonl` to append to, and cut it to `read.validBytes` (what `readRecords` found valid),
101
+ * which drops a line a killed run left unfinished. `read` is the answer of the `readRecords` this run
102
+ * began with, or `{ validBytes: 0, bytes: 0 }` for a run that is new: the file must still be `read.bytes`
103
+ * long, or another run may have cut it or appended to it since, and cutting it back would drop that run's
104
+ * records. A read that found no bytes makes the file when it is not there, which loses nothing; one that
105
+ * found bytes never makes it, so a file deleted since is refused with nothing written. `O_APPEND` puts
106
+ * each write at the end, `O_NOFOLLOW` refuses a symbolic link, and `O_NONBLOCK` refuses a FIFO instead of
107
+ * waiting for it. A file made here is readable by its owner only, and one that exists keeps its mode.
108
+ * `append` writes one whole line, each call after the one before, so lines never interleave; once a write
109
+ * has failed, none after it is written, so a line is never joined to the fragment of a failed one. `close`
110
+ * waits for the lines asked for and closes.
111
+ *
112
+ * @throws ConfigError `results.jsonl: is not a file`, or `results.jsonl: changed after this run read it,
113
+ * so another run may be using this --out: run one at a time` (a file cut, appended to or deleted since the
114
+ * read); Error `results.jsonl: cannot be written (<code>)` for any other failure to open, cut, write or
115
+ * close
116
+ */
117
+ export declare function openRecords(directory: string, read: ReadSizes): Promise<RecordWriter>;
118
+ /**
119
+ * Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
120
+ * made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
121
+ * The rename replaces a symbolic link at `name`, and never writes through it. The temporary file is made
122
+ * with `mode`, which the rename keeps: Node's own default unless a caller asks for another. A failure
123
+ * removes the temporary file it made.
124
+ *
125
+ * @throws Error `<name>: cannot be written (<code>)`
126
+ */
127
+ export declare function writeAtomic(directory: string, name: string, text: string, mode?: number): Promise<void>;