@tobi-or-not/quorum 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,332 @@
1
+ /**
2
+ * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
+ * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
+ * report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
5
+ * never text a model wrote.
6
+ */
7
+ import { inertJson, inertLine } from "../core/sanitize.js";
8
+ import { reviewKey } from "./records.js";
9
+ import { scoreReview } from "./score.js";
10
+ import { nearestRank, signTest, wilson } from "./stats.js";
11
+ /** A comparison is resolved when its p is below this. */
12
+ const ALPHA = 0.05;
13
+ /** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
14
+ export function reviewScore(record, suiteCase, window) {
15
+ return {
16
+ ...scoreReview(record.result, suiteCase, window),
17
+ latencyMs: record.latencyMs,
18
+ ...(record.usage === undefined
19
+ ? {}
20
+ : { tokens: record.usage.inputTokens + record.usage.outputTokens }),
21
+ ...(record.costComplete
22
+ ? { costUsd: record.usage?.costUsd ?? 0 }
23
+ : {}),
24
+ };
25
+ }
26
+ /** The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0. */
27
+ function mean(values) {
28
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
29
+ }
30
+ /** The share of `scores` that `pick` holds for. */
31
+ function share(scores, pick) {
32
+ return scores.filter(pick).length / scores.length;
33
+ }
34
+ function caseScore(kind, scores) {
35
+ if (scores.length === 0)
36
+ return null;
37
+ const tokens = scores.flatMap(score => score.tokens === undefined
38
+ ? []
39
+ : [score.tokens]);
40
+ return {
41
+ trials: scores.length,
42
+ keptCatch: kind === 'bug'
43
+ ? share(scores, score => score.keptCatch)
44
+ : null,
45
+ proposedCatch: kind === 'bug'
46
+ ? share(scores, score => score.proposedCatch)
47
+ : null,
48
+ unmatchedKept: mean(scores.map(score => score.unmatchedKept)),
49
+ incomplete: share(scores, score => score.incomplete),
50
+ tokens: tokens.length === 0
51
+ ? null
52
+ : mean(tokens),
53
+ };
54
+ }
55
+ /** The scores of the reviews that count for a configuration, per selected case: those recorded for a trial from 1 to `trials`. */
56
+ function countedScores(input, fingerprint) {
57
+ return input.cases.map((suiteCase) => {
58
+ const counted = [];
59
+ for (let trial = 1; trial <= input.trials; trial++) {
60
+ const score = input.scores.get(reviewKey(fingerprint, suiteCase.id, trial));
61
+ if (score !== undefined)
62
+ counted.push(score);
63
+ }
64
+ return counted;
65
+ });
66
+ }
67
+ function rate(values) {
68
+ if (values.length === 0)
69
+ return null;
70
+ const value = mean(values);
71
+ return {
72
+ value,
73
+ ...wilson(value, values.length),
74
+ cases: values.length,
75
+ };
76
+ }
77
+ function meanOrNull(values) {
78
+ return values.length === 0
79
+ ? null
80
+ : mean(values);
81
+ }
82
+ function summarize(input, index, counted, cases) {
83
+ const scored = (kind, pick) => cases.flatMap(({ kind: caseKind, configurations }) => {
84
+ const score = configurations[index - 1];
85
+ const value = caseKind === kind && score !== null
86
+ ? pick(score)
87
+ : null;
88
+ return value === null
89
+ ? []
90
+ : [value];
91
+ });
92
+ const reviews = counted.flat();
93
+ const tokens = reviews.flatMap(({ tokens: used }) => used === undefined
94
+ ? []
95
+ : [used]);
96
+ return {
97
+ index,
98
+ path: input.configurations[index - 1].path,
99
+ reviews: reviews.length,
100
+ planned: input.cases.length * input.trials,
101
+ incomplete: reviews.filter(score => score.incomplete).length,
102
+ keptRecall: rate(scored('bug', score => score.keptCatch)),
103
+ proposedRecall: rate(scored('bug', score => score.proposedCatch)),
104
+ unmatchedKept: {
105
+ bug: meanOrNull(scored('bug', score => score.unmatchedKept)),
106
+ control: meanOrNull(scored('control', score => score.unmatchedKept)),
107
+ },
108
+ tokens: tokens.length === 0
109
+ ? null
110
+ : {
111
+ median: nearestRank(tokens, 50),
112
+ total: tokens.reduce((sum, used) => sum + used, 0),
113
+ },
114
+ latencyMs: reviews.length === 0
115
+ ? null
116
+ : {
117
+ p50: nearestRank(reviews.map(score => score.latencyMs), 50),
118
+ p95: nearestRank(reviews.map(score => score.latencyMs), 95),
119
+ },
120
+ costUsd: reviews.length > 0 && reviews.every(score => score.costUsd !== undefined)
121
+ ? reviews.reduce((sum, score) => sum + (score.costUsd ?? 0), 0)
122
+ : null,
123
+ };
124
+ }
125
+ /** The five measures of a comparison, in the order of `Measure`. */
126
+ const MEASURES = [
127
+ {
128
+ measure: 'kept-recall',
129
+ kind: 'bug',
130
+ lowerIsBetter: false,
131
+ pick: score => score.keptCatch,
132
+ },
133
+ {
134
+ measure: 'proposed-recall',
135
+ kind: 'bug',
136
+ lowerIsBetter: false,
137
+ pick: score => score.proposedCatch,
138
+ },
139
+ {
140
+ measure: 'unmatched-kept-per-control',
141
+ kind: 'control',
142
+ lowerIsBetter: true,
143
+ pick: score => score.unmatchedKept,
144
+ },
145
+ {
146
+ measure: 'incomplete',
147
+ lowerIsBetter: true,
148
+ pick: score => score.incomplete,
149
+ },
150
+ {
151
+ measure: 'tokens',
152
+ lowerIsBetter: true,
153
+ pick: score => score.tokens,
154
+ },
155
+ ];
156
+ /** The configuration at `index` (from 1, after the reference) against the reference on one measure, over the cases both scored. */
157
+ function compare(rule, cases, index) {
158
+ const differences = [];
159
+ let better = 0;
160
+ let worse = 0;
161
+ for (const { kind, configurations } of cases) {
162
+ if (rule.kind !== undefined && rule.kind !== kind)
163
+ continue;
164
+ const reference = configurations[0];
165
+ const current = configurations[index - 1];
166
+ const before = reference === null
167
+ ? null
168
+ : rule.pick(reference);
169
+ const after = current === null
170
+ ? null
171
+ : rule.pick(current);
172
+ if (before === null || after === null)
173
+ continue;
174
+ const difference = after - before;
175
+ differences.push(difference);
176
+ if (difference === 0)
177
+ continue;
178
+ if ((difference < 0) === rule.lowerIsBetter)
179
+ better++;
180
+ else
181
+ worse++;
182
+ }
183
+ const p = signTest(better, worse);
184
+ return {
185
+ configuration: index,
186
+ measure: rule.measure,
187
+ difference: meanOrNull(differences),
188
+ better,
189
+ worse,
190
+ tied: differences.length - better - worse,
191
+ p,
192
+ resolved: p < ALPHA,
193
+ };
194
+ }
195
+ /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
196
+ export function buildReport(input) {
197
+ const counted = input.configurations.map(({ fingerprint }) => countedScores(input, fingerprint));
198
+ const cases = input.cases.map((suiteCase, position) => ({
199
+ id: suiteCase.id,
200
+ kind: suiteCase.kind,
201
+ configurations: counted.map(perCase => caseScore(suiteCase.kind, perCase[position])),
202
+ }));
203
+ return {
204
+ suite: input.suite.name,
205
+ quorum: input.quorum,
206
+ trials: input.trials,
207
+ window: input.suite.window,
208
+ bugCases: input.cases.filter(({ kind }) => kind === 'bug').length,
209
+ controls: input.cases.filter(({ kind }) => kind === 'control').length,
210
+ configurations: counted.map((perCase, position) => summarize(input, position + 1, perCase, cases)),
211
+ comparisons: input.configurations.slice(1).flatMap((_, position) => MEASURES.map(rule => compare(rule, cases, position + 2))),
212
+ cases,
213
+ };
214
+ }
215
+ const NOT_AVAILABLE = 'n/a';
216
+ const LABELS = {
217
+ 'kept-recall': 'kept recall',
218
+ 'proposed-recall': 'proposed recall',
219
+ 'unmatched-kept-per-control': 'unmatched kept findings per control',
220
+ incomplete: 'incomplete reviews',
221
+ tokens: 'tokens per review',
222
+ };
223
+ function plural(count, noun) {
224
+ return `${count} ${noun}${count === 1
225
+ ? ''
226
+ : 's'}`;
227
+ }
228
+ function percent(value) {
229
+ return `${(value * 100).toFixed(1)}%`;
230
+ }
231
+ function rateText(rate) {
232
+ return rate === null
233
+ ? NOT_AVAILABLE
234
+ : `${percent(rate.value)} [${percent(rate.low)}, ${percent(rate.high)}] over ${plural(rate.cases, 'case')}`;
235
+ }
236
+ function twoDecimals(value) {
237
+ return value === null
238
+ ? NOT_AVAILABLE
239
+ : value.toFixed(2);
240
+ }
241
+ function seconds(ms) {
242
+ return `${(ms / 1000).toFixed(1)} s`;
243
+ }
244
+ /** `value` with its sign always written, its magnitude as `digits` decimals of `value * scale`. */
245
+ function signed(value, scale, digits) {
246
+ return `${value < 0
247
+ ? '-'
248
+ : '+'}${Math.abs(value * scale).toFixed(digits)}`;
249
+ }
250
+ function differenceText(measure, difference) {
251
+ switch (measure) {
252
+ case 'kept-recall':
253
+ case 'proposed-recall':
254
+ case 'incomplete':
255
+ return `${signed(difference, 100, 1)} points`;
256
+ case 'unmatched-kept-per-control':
257
+ return signed(difference, 1, 2);
258
+ case 'tokens':
259
+ return signed(difference, 1, 0);
260
+ }
261
+ }
262
+ function pText(p) {
263
+ return p < 0.0001
264
+ ? 'p < 0.0001'
265
+ : `p = ${p.toFixed(4)}`;
266
+ }
267
+ function rowText(summary) {
268
+ const cells = [
269
+ String(summary.index),
270
+ `${summary.reviews} of ${summary.planned}`,
271
+ rateText(summary.keptRecall),
272
+ rateText(summary.proposedRecall),
273
+ `${twoDecimals(summary.unmatchedKept.bug)} / ${twoDecimals(summary.unmatchedKept.control)}`,
274
+ String(summary.incomplete),
275
+ summary.tokens === null
276
+ ? NOT_AVAILABLE
277
+ : `${Math.round(summary.tokens.median)} / ${Math.round(summary.tokens.total)}`,
278
+ summary.latencyMs === null
279
+ ? NOT_AVAILABLE
280
+ : `${seconds(summary.latencyMs.p50)} / ${seconds(summary.latencyMs.p95)}`,
281
+ summary.costUsd === null
282
+ ? NOT_AVAILABLE
283
+ : `$${summary.costUsd.toFixed(4)}`,
284
+ ];
285
+ return `| ${cells.join(' | ')} |`;
286
+ }
287
+ function comparisonText(comparison) {
288
+ const label = `- ${comparison.configuration}, ${LABELS[comparison.measure]}`;
289
+ if (comparison.difference === null)
290
+ return `${label}: no case scored by both`;
291
+ const counts = `${comparison.better} better, ${comparison.worse} worse, ${comparison.tied} tied`;
292
+ return `${label}: ${differenceText(comparison.measure, comparison.difference)}; ${counts}; ${pText(comparison.p)}, ${comparison.resolved
293
+ ? 'resolved'
294
+ : 'unresolved'}`;
295
+ }
296
+ function configurationText(summary, several) {
297
+ const name = summary.path ?? '(default configuration)';
298
+ return `${summary.index}. ${name}${several && summary.index === 1
299
+ ? ' (reference)'
300
+ : ''}`;
301
+ }
302
+ /** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
303
+ export function reportMarkdown(report) {
304
+ const several = report.configurations.length > 1;
305
+ const trials = report.trials === 1
306
+ ? 'trial 1'
307
+ : `trials 1 to ${report.trials}`;
308
+ const lines = [
309
+ `# quorum bench: ${report.suite}`,
310
+ '',
311
+ `quorum ${report.quorum}; ${plural(report.bugCases, 'bug case')} and ${plural(report.controls, 'control')}; ${trials}; window ${plural(report.window, 'line')}.`,
312
+ '',
313
+ 'A review counts when it is recorded for one of these cases and trials. A case scores the share of its counted trials with the outcome, and a case with none counted is left out. A rate is the mean of the case scores, with its 95% Wilson interval over the cases scored.',
314
+ '',
315
+ '## Configurations',
316
+ '',
317
+ ...report.configurations.map(summary => configurationText(summary, several)),
318
+ '',
319
+ '| Configuration | Reviews | Kept recall | Proposed recall | Unmatched kept per case, bug / control | Incomplete | Tokens per review, median / total | Latency, p50 / p95 | Cost |',
320
+ '| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
321
+ ...report.configurations.map(rowText),
322
+ '',
323
+ ];
324
+ if (several) {
325
+ lines.push('## Comparisons with the reference', '', 'Each line pairs the cases both configurations scored and is its own exact sign test, so with several lines some can resolve by chance.', '', ...report.comparisons.map(comparisonText), '');
326
+ }
327
+ return lines.map(inertLine).join('\n');
328
+ }
329
+ /** The report as inert JSON and a line break; it parses back to the report. */
330
+ export function reportJson(report) {
331
+ return `${inertJson(report)}\n`;
332
+ }
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Every read and write of a bench run directory, which holds `run.json` (written once, whole),
3
+ * `results.jsonl` (one line per review, appended one at a time), and the reports (`report.md` and
4
+ * `report.json`, rewritten whole). Both data files are read as untrusted, since another run or a person
5
+ * may have written them: strict schemas (see `records.ts`), bounded sizes, and no file that is not a
6
+ * regular file. No write follows a symbolic link: `results.jsonl` is opened with `O_NOFOLLOW`, and every
7
+ * other file is written to a new file that is renamed over the name, which replaces a link there.
8
+ *
9
+ * Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
10
+ * runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
11
+ * I/O failure is an `Error`.
12
+ */
13
+ import type { ConfigIssue } from '../core/errors.ts';
14
+ import type { ReviewRecord, RunFile } from './records.ts';
15
+ export declare const RUN_FILE = "run.json";
16
+ export declare const RESULTS_FILE = "results.jsonl";
17
+ export declare const REPORT_MARKDOWN = "report.md";
18
+ export declare const REPORT_JSON = "report.json";
19
+ /** What `--out` holds: nothing, an empty directory, or a run. */
20
+ export type RunState = {
21
+ readonly kind: 'none';
22
+ } | {
23
+ readonly kind: 'empty';
24
+ } | {
25
+ readonly kind: 'run';
26
+ readonly run: RunFile;
27
+ };
28
+ /** What a read of `results.jsonl` found: the complete lines it visited, where they end, and whether bytes follow the last line break. */
29
+ export interface RecordsRead {
30
+ readonly lines: number;
31
+ /** The length of the file up to and including the last line break. */
32
+ readonly validBytes: number;
33
+ /** A last line with no line break: a run killed while it appended. It is not visited. */
34
+ readonly torn: boolean;
35
+ }
36
+ /** Appends lines to `results.jsonl`, one at a time. */
37
+ export interface RecordWriter {
38
+ readonly append: (line: string) => Promise<void>;
39
+ readonly close: () => Promise<void>;
40
+ }
41
+ /**
42
+ * What `directory`, the run directory a command was given, holds. A directory that does not exist is
43
+ * `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
44
+ * as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
45
+ * `run.json`, and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
46
+ * format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
47
+ * its parents may be links: they are the user's own path.
48
+ *
49
+ * @throws ConfigError as above; Error when the directory or `run.json` cannot be read
50
+ */
51
+ export declare function inspectRunDir(directory: string): Promise<RunState>;
52
+ /**
53
+ * What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
54
+ * quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
55
+ * which holds the preset by name), then another suite by its name, else a suite whose content changed,
56
+ * then each current configuration the run does not hold, at `--config`, then each recorded one that is
57
+ * not given again. A current and a recorded configuration that share a path (`null`, the usual
58
+ * resolution, included) but not a fingerprint are one configuration whose settings changed, and give one
59
+ * issue in place of the two. The same configurations in another order are the same run. A configuration
60
+ * is named by its path as written, or `(default configuration)`: the recorded paths are text from
61
+ * `run.json`, to make inert where they are printed.
62
+ */
63
+ export declare function runDifferences(recorded: RunFile, current: Pick<RunFile, 'quorum' | 'suite' | 'configurations'>): ConfigIssue[];
64
+ /**
65
+ * Make `directory` and its parents, then write `run.json` into it whole, through a new file and a rename.
66
+ *
67
+ * @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
68
+ */
69
+ export declare function createRun(directory: string, run: RunFile): Promise<void>;
70
+ /**
71
+ * Read `results.jsonl` line by line and hand each record, with its line number from 1, to `visit`, which
72
+ * returns the problem with a record it refuses, or undefined. A file with no `results.jsonl` holds
73
+ * nothing. Bytes after the last line break are a line a run killed while it appended cut short: it is
74
+ * reported as `torn`, never visited, and the next append must drop it (see `openRecords`). Any other line
75
+ * that is no record, a line over 16 MiB (complete or not), a record `visit` refuses, a link and a file
76
+ * that is none are refused. The file is read in chunks of 64 KiB and split on the line feed byte, and a
77
+ * line over the limit is refused as soon as it is seen to pass it, so no line is read whole before it is
78
+ * known to fit.
79
+ *
80
+ * @throws ConfigError as above; Error `results.jsonl: cannot be read (<code>)` when a read fails
81
+ */
82
+ export declare function readRecords(directory: string, visit: (record: ReviewRecord, line: number) => string | undefined): Promise<RecordsRead>;
83
+ /**
84
+ * Open `results.jsonl` to append to, making it when it is not there, and cut it to `validBytes` (what
85
+ * `readRecords` found valid), which drops a line a killed run left unfinished. `O_APPEND` puts each write at
86
+ * the end, and `O_NOFOLLOW` refuses a symbolic link. `append` writes one whole line, each call after the
87
+ * one before, so lines never interleave; once a write has failed, none after it is written, so a line
88
+ * is never joined to the fragment of a failed one. `close` waits for the lines asked for and closes.
89
+ *
90
+ * @throws Error `results.jsonl: cannot be written (<code>)` for any failure to open, cut, write or close
91
+ */
92
+ export declare function openRecords(directory: string, validBytes: number): Promise<RecordWriter>;
93
+ /**
94
+ * Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
95
+ * made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
96
+ * The rename replaces a symbolic link at `name`, and never writes through it. A failure removes the
97
+ * temporary file it made.
98
+ *
99
+ * @throws Error `<name>: cannot be written (<code>)`
100
+ */
101
+ export declare function writeAtomic(directory: string, name: string, text: string): Promise<void>;