@tobi-or-not/quorum 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +5 -0
- package/dist/bench/fingerprint.d.ts +36 -0
- package/dist/bench/fingerprint.js +79 -0
- package/dist/bench/records.d.ts +62 -0
- package/dist/bench/records.js +161 -0
- package/dist/bench/report.d.ts +110 -0
- package/dist/bench/report.js +332 -0
- package/dist/bench/run-dir.d.ts +101 -0
- package/dist/bench/run-dir.js +379 -0
- package/dist/bench/runner.d.ts +80 -0
- package/dist/bench/runner.js +160 -0
- package/dist/bench/score.d.ts +30 -0
- package/dist/bench/score.js +38 -0
- package/dist/bench/stats.d.ts +51 -0
- package/dist/bench/stats.js +83 -0
- package/dist/bench/suite.d.ts +87 -0
- package/dist/bench/suite.js +286 -0
- package/dist/cli/bench.d.ts +32 -0
- package/dist/cli/bench.js +463 -0
- package/dist/cli/bin.js +9 -1
- package/dist/cli/io.d.ts +45 -0
- package/dist/cli/io.js +45 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +47 -60
- package/dist/cli/print-config.js +38 -9
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +37 -16
- package/dist/config/claude-code.js +42 -11
- package/dist/config/env.js +135 -32
- package/dist/config/from-config.js +43 -9
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.js +205 -46
- package/dist/config/profiles.js +87 -20
- package/dist/config/resolve.js +194 -47
- package/dist/config/schema.js +98 -23
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +90 -17
- package/dist/core/errors.js +14 -5
- package/dist/core/flags.js +30 -6
- package/dist/core/output.js +49 -13
- package/dist/core/policy.js +6 -2
- package/dist/core/profile.js +42 -15
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +28 -8
- package/dist/core/quorum.js +128 -33
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +31 -5
- package/dist/core/stages/context.js +30 -5
- package/dist/core/stages/panel.js +55 -9
- package/dist/core/stages/research.js +13 -3
- package/dist/core/verify.js +6 -2
- package/dist/index.js +5 -1
- package/dist/providers/anthropic.js +28 -5
- package/dist/providers/build.js +57 -16
- package/dist/providers/claude-code.js +206 -51
- package/dist/providers/gemini.js +61 -14
- package/dist/providers/http.js +32 -9
- package/dist/providers/messages.js +14 -2
- package/dist/providers/ollama.js +31 -8
- package/dist/providers/openai-compatible.js +51 -15
- package/dist/providers/options.js +120 -40
- package/dist/providers/presets.js +143 -16
- package/dist/providers/reply.js +15 -5
- package/dist/providers/size.js +11 -3
- package/package.json +6 -5
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
|
|
3
|
+
* built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
|
|
4
|
+
* report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
|
|
5
|
+
* never text a model wrote.
|
|
6
|
+
*/
|
|
7
|
+
import { inertJson, inertLine } from "../core/sanitize.js";
|
|
8
|
+
import { reviewKey } from "./records.js";
|
|
9
|
+
import { scoreReview } from "./score.js";
|
|
10
|
+
import { nearestRank, signTest, wilson } from "./stats.js";
|
|
11
|
+
/** A comparison is resolved when its p is below this. */
|
|
12
|
+
const ALPHA = 0.05;
|
|
13
|
+
/** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
|
|
14
|
+
export function reviewScore(record, suiteCase, window) {
|
|
15
|
+
return {
|
|
16
|
+
...scoreReview(record.result, suiteCase, window),
|
|
17
|
+
latencyMs: record.latencyMs,
|
|
18
|
+
...(record.usage === undefined
|
|
19
|
+
? {}
|
|
20
|
+
: { tokens: record.usage.inputTokens + record.usage.outputTokens }),
|
|
21
|
+
...(record.costComplete
|
|
22
|
+
? { costUsd: record.usage?.costUsd ?? 0 }
|
|
23
|
+
: {}),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
/** The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0. */
|
|
27
|
+
function mean(values) {
|
|
28
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
29
|
+
}
|
|
30
|
+
/** The share of `scores` that `pick` holds for. */
|
|
31
|
+
function share(scores, pick) {
|
|
32
|
+
return scores.filter(pick).length / scores.length;
|
|
33
|
+
}
|
|
34
|
+
function caseScore(kind, scores) {
|
|
35
|
+
if (scores.length === 0)
|
|
36
|
+
return null;
|
|
37
|
+
const tokens = scores.flatMap(score => score.tokens === undefined
|
|
38
|
+
? []
|
|
39
|
+
: [score.tokens]);
|
|
40
|
+
return {
|
|
41
|
+
trials: scores.length,
|
|
42
|
+
keptCatch: kind === 'bug'
|
|
43
|
+
? share(scores, score => score.keptCatch)
|
|
44
|
+
: null,
|
|
45
|
+
proposedCatch: kind === 'bug'
|
|
46
|
+
? share(scores, score => score.proposedCatch)
|
|
47
|
+
: null,
|
|
48
|
+
unmatchedKept: mean(scores.map(score => score.unmatchedKept)),
|
|
49
|
+
incomplete: share(scores, score => score.incomplete),
|
|
50
|
+
tokens: tokens.length === 0
|
|
51
|
+
? null
|
|
52
|
+
: mean(tokens),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/** The scores of the reviews that count for a configuration, per selected case: those recorded for a trial from 1 to `trials`. */
|
|
56
|
+
function countedScores(input, fingerprint) {
|
|
57
|
+
return input.cases.map((suiteCase) => {
|
|
58
|
+
const counted = [];
|
|
59
|
+
for (let trial = 1; trial <= input.trials; trial++) {
|
|
60
|
+
const score = input.scores.get(reviewKey(fingerprint, suiteCase.id, trial));
|
|
61
|
+
if (score !== undefined)
|
|
62
|
+
counted.push(score);
|
|
63
|
+
}
|
|
64
|
+
return counted;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
function rate(values) {
|
|
68
|
+
if (values.length === 0)
|
|
69
|
+
return null;
|
|
70
|
+
const value = mean(values);
|
|
71
|
+
return {
|
|
72
|
+
value,
|
|
73
|
+
...wilson(value, values.length),
|
|
74
|
+
cases: values.length,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function meanOrNull(values) {
|
|
78
|
+
return values.length === 0
|
|
79
|
+
? null
|
|
80
|
+
: mean(values);
|
|
81
|
+
}
|
|
82
|
+
function summarize(input, index, counted, cases) {
|
|
83
|
+
const scored = (kind, pick) => cases.flatMap(({ kind: caseKind, configurations }) => {
|
|
84
|
+
const score = configurations[index - 1];
|
|
85
|
+
const value = caseKind === kind && score !== null
|
|
86
|
+
? pick(score)
|
|
87
|
+
: null;
|
|
88
|
+
return value === null
|
|
89
|
+
? []
|
|
90
|
+
: [value];
|
|
91
|
+
});
|
|
92
|
+
const reviews = counted.flat();
|
|
93
|
+
const tokens = reviews.flatMap(({ tokens: used }) => used === undefined
|
|
94
|
+
? []
|
|
95
|
+
: [used]);
|
|
96
|
+
return {
|
|
97
|
+
index,
|
|
98
|
+
path: input.configurations[index - 1].path,
|
|
99
|
+
reviews: reviews.length,
|
|
100
|
+
planned: input.cases.length * input.trials,
|
|
101
|
+
incomplete: reviews.filter(score => score.incomplete).length,
|
|
102
|
+
keptRecall: rate(scored('bug', score => score.keptCatch)),
|
|
103
|
+
proposedRecall: rate(scored('bug', score => score.proposedCatch)),
|
|
104
|
+
unmatchedKept: {
|
|
105
|
+
bug: meanOrNull(scored('bug', score => score.unmatchedKept)),
|
|
106
|
+
control: meanOrNull(scored('control', score => score.unmatchedKept)),
|
|
107
|
+
},
|
|
108
|
+
tokens: tokens.length === 0
|
|
109
|
+
? null
|
|
110
|
+
: {
|
|
111
|
+
median: nearestRank(tokens, 50),
|
|
112
|
+
total: tokens.reduce((sum, used) => sum + used, 0),
|
|
113
|
+
},
|
|
114
|
+
latencyMs: reviews.length === 0
|
|
115
|
+
? null
|
|
116
|
+
: {
|
|
117
|
+
p50: nearestRank(reviews.map(score => score.latencyMs), 50),
|
|
118
|
+
p95: nearestRank(reviews.map(score => score.latencyMs), 95),
|
|
119
|
+
},
|
|
120
|
+
costUsd: reviews.length > 0 && reviews.every(score => score.costUsd !== undefined)
|
|
121
|
+
? reviews.reduce((sum, score) => sum + (score.costUsd ?? 0), 0)
|
|
122
|
+
: null,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/** The five measures of a comparison, in the order of `Measure`. */
|
|
126
|
+
const MEASURES = [
|
|
127
|
+
{
|
|
128
|
+
measure: 'kept-recall',
|
|
129
|
+
kind: 'bug',
|
|
130
|
+
lowerIsBetter: false,
|
|
131
|
+
pick: score => score.keptCatch,
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
measure: 'proposed-recall',
|
|
135
|
+
kind: 'bug',
|
|
136
|
+
lowerIsBetter: false,
|
|
137
|
+
pick: score => score.proposedCatch,
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
measure: 'unmatched-kept-per-control',
|
|
141
|
+
kind: 'control',
|
|
142
|
+
lowerIsBetter: true,
|
|
143
|
+
pick: score => score.unmatchedKept,
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
measure: 'incomplete',
|
|
147
|
+
lowerIsBetter: true,
|
|
148
|
+
pick: score => score.incomplete,
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
measure: 'tokens',
|
|
152
|
+
lowerIsBetter: true,
|
|
153
|
+
pick: score => score.tokens,
|
|
154
|
+
},
|
|
155
|
+
];
|
|
156
|
+
/** The configuration at `index` (from 1, after the reference) against the reference on one measure, over the cases both scored. */
|
|
157
|
+
function compare(rule, cases, index) {
|
|
158
|
+
const differences = [];
|
|
159
|
+
let better = 0;
|
|
160
|
+
let worse = 0;
|
|
161
|
+
for (const { kind, configurations } of cases) {
|
|
162
|
+
if (rule.kind !== undefined && rule.kind !== kind)
|
|
163
|
+
continue;
|
|
164
|
+
const reference = configurations[0];
|
|
165
|
+
const current = configurations[index - 1];
|
|
166
|
+
const before = reference === null
|
|
167
|
+
? null
|
|
168
|
+
: rule.pick(reference);
|
|
169
|
+
const after = current === null
|
|
170
|
+
? null
|
|
171
|
+
: rule.pick(current);
|
|
172
|
+
if (before === null || after === null)
|
|
173
|
+
continue;
|
|
174
|
+
const difference = after - before;
|
|
175
|
+
differences.push(difference);
|
|
176
|
+
if (difference === 0)
|
|
177
|
+
continue;
|
|
178
|
+
if ((difference < 0) === rule.lowerIsBetter)
|
|
179
|
+
better++;
|
|
180
|
+
else
|
|
181
|
+
worse++;
|
|
182
|
+
}
|
|
183
|
+
const p = signTest(better, worse);
|
|
184
|
+
return {
|
|
185
|
+
configuration: index,
|
|
186
|
+
measure: rule.measure,
|
|
187
|
+
difference: meanOrNull(differences),
|
|
188
|
+
better,
|
|
189
|
+
worse,
|
|
190
|
+
tied: differences.length - better - worse,
|
|
191
|
+
p,
|
|
192
|
+
resolved: p < ALPHA,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
|
|
196
|
+
export function buildReport(input) {
|
|
197
|
+
const counted = input.configurations.map(({ fingerprint }) => countedScores(input, fingerprint));
|
|
198
|
+
const cases = input.cases.map((suiteCase, position) => ({
|
|
199
|
+
id: suiteCase.id,
|
|
200
|
+
kind: suiteCase.kind,
|
|
201
|
+
configurations: counted.map(perCase => caseScore(suiteCase.kind, perCase[position])),
|
|
202
|
+
}));
|
|
203
|
+
return {
|
|
204
|
+
suite: input.suite.name,
|
|
205
|
+
quorum: input.quorum,
|
|
206
|
+
trials: input.trials,
|
|
207
|
+
window: input.suite.window,
|
|
208
|
+
bugCases: input.cases.filter(({ kind }) => kind === 'bug').length,
|
|
209
|
+
controls: input.cases.filter(({ kind }) => kind === 'control').length,
|
|
210
|
+
configurations: counted.map((perCase, position) => summarize(input, position + 1, perCase, cases)),
|
|
211
|
+
comparisons: input.configurations.slice(1).flatMap((_, position) => MEASURES.map(rule => compare(rule, cases, position + 2))),
|
|
212
|
+
cases,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
const NOT_AVAILABLE = 'n/a';
|
|
216
|
+
const LABELS = {
|
|
217
|
+
'kept-recall': 'kept recall',
|
|
218
|
+
'proposed-recall': 'proposed recall',
|
|
219
|
+
'unmatched-kept-per-control': 'unmatched kept findings per control',
|
|
220
|
+
incomplete: 'incomplete reviews',
|
|
221
|
+
tokens: 'tokens per review',
|
|
222
|
+
};
|
|
223
|
+
function plural(count, noun) {
|
|
224
|
+
return `${count} ${noun}${count === 1
|
|
225
|
+
? ''
|
|
226
|
+
: 's'}`;
|
|
227
|
+
}
|
|
228
|
+
function percent(value) {
|
|
229
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
230
|
+
}
|
|
231
|
+
function rateText(rate) {
|
|
232
|
+
return rate === null
|
|
233
|
+
? NOT_AVAILABLE
|
|
234
|
+
: `${percent(rate.value)} [${percent(rate.low)}, ${percent(rate.high)}] over ${plural(rate.cases, 'case')}`;
|
|
235
|
+
}
|
|
236
|
+
function twoDecimals(value) {
|
|
237
|
+
return value === null
|
|
238
|
+
? NOT_AVAILABLE
|
|
239
|
+
: value.toFixed(2);
|
|
240
|
+
}
|
|
241
|
+
function seconds(ms) {
|
|
242
|
+
return `${(ms / 1000).toFixed(1)} s`;
|
|
243
|
+
}
|
|
244
|
+
/** `value` with its sign always written, its magnitude as `digits` decimals of `value * scale`. */
|
|
245
|
+
function signed(value, scale, digits) {
|
|
246
|
+
return `${value < 0
|
|
247
|
+
? '-'
|
|
248
|
+
: '+'}${Math.abs(value * scale).toFixed(digits)}`;
|
|
249
|
+
}
|
|
250
|
+
function differenceText(measure, difference) {
|
|
251
|
+
switch (measure) {
|
|
252
|
+
case 'kept-recall':
|
|
253
|
+
case 'proposed-recall':
|
|
254
|
+
case 'incomplete':
|
|
255
|
+
return `${signed(difference, 100, 1)} points`;
|
|
256
|
+
case 'unmatched-kept-per-control':
|
|
257
|
+
return signed(difference, 1, 2);
|
|
258
|
+
case 'tokens':
|
|
259
|
+
return signed(difference, 1, 0);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
function pText(p) {
|
|
263
|
+
return p < 0.0001
|
|
264
|
+
? 'p < 0.0001'
|
|
265
|
+
: `p = ${p.toFixed(4)}`;
|
|
266
|
+
}
|
|
267
|
+
function rowText(summary) {
|
|
268
|
+
const cells = [
|
|
269
|
+
String(summary.index),
|
|
270
|
+
`${summary.reviews} of ${summary.planned}`,
|
|
271
|
+
rateText(summary.keptRecall),
|
|
272
|
+
rateText(summary.proposedRecall),
|
|
273
|
+
`${twoDecimals(summary.unmatchedKept.bug)} / ${twoDecimals(summary.unmatchedKept.control)}`,
|
|
274
|
+
String(summary.incomplete),
|
|
275
|
+
summary.tokens === null
|
|
276
|
+
? NOT_AVAILABLE
|
|
277
|
+
: `${Math.round(summary.tokens.median)} / ${Math.round(summary.tokens.total)}`,
|
|
278
|
+
summary.latencyMs === null
|
|
279
|
+
? NOT_AVAILABLE
|
|
280
|
+
: `${seconds(summary.latencyMs.p50)} / ${seconds(summary.latencyMs.p95)}`,
|
|
281
|
+
summary.costUsd === null
|
|
282
|
+
? NOT_AVAILABLE
|
|
283
|
+
: `$${summary.costUsd.toFixed(4)}`,
|
|
284
|
+
];
|
|
285
|
+
return `| ${cells.join(' | ')} |`;
|
|
286
|
+
}
|
|
287
|
+
function comparisonText(comparison) {
|
|
288
|
+
const label = `- ${comparison.configuration}, ${LABELS[comparison.measure]}`;
|
|
289
|
+
if (comparison.difference === null)
|
|
290
|
+
return `${label}: no case scored by both`;
|
|
291
|
+
const counts = `${comparison.better} better, ${comparison.worse} worse, ${comparison.tied} tied`;
|
|
292
|
+
return `${label}: ${differenceText(comparison.measure, comparison.difference)}; ${counts}; ${pText(comparison.p)}, ${comparison.resolved
|
|
293
|
+
? 'resolved'
|
|
294
|
+
: 'unresolved'}`;
|
|
295
|
+
}
|
|
296
|
+
function configurationText(summary, several) {
|
|
297
|
+
const name = summary.path ?? '(default configuration)';
|
|
298
|
+
return `${summary.index}. ${name}${several && summary.index === 1
|
|
299
|
+
? ' (reference)'
|
|
300
|
+
: ''}`;
|
|
301
|
+
}
|
|
302
|
+
/** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
|
|
303
|
+
export function reportMarkdown(report) {
|
|
304
|
+
const several = report.configurations.length > 1;
|
|
305
|
+
const trials = report.trials === 1
|
|
306
|
+
? 'trial 1'
|
|
307
|
+
: `trials 1 to ${report.trials}`;
|
|
308
|
+
const lines = [
|
|
309
|
+
`# quorum bench: ${report.suite}`,
|
|
310
|
+
'',
|
|
311
|
+
`quorum ${report.quorum}; ${plural(report.bugCases, 'bug case')} and ${plural(report.controls, 'control')}; ${trials}; window ${plural(report.window, 'line')}.`,
|
|
312
|
+
'',
|
|
313
|
+
'A review counts when it is recorded for one of these cases and trials. A case scores the share of its counted trials with the outcome, and a case with none counted is left out. A rate is the mean of the case scores, with its 95% Wilson interval over the cases scored.',
|
|
314
|
+
'',
|
|
315
|
+
'## Configurations',
|
|
316
|
+
'',
|
|
317
|
+
...report.configurations.map(summary => configurationText(summary, several)),
|
|
318
|
+
'',
|
|
319
|
+
'| Configuration | Reviews | Kept recall | Proposed recall | Unmatched kept per case, bug / control | Incomplete | Tokens per review, median / total | Latency, p50 / p95 | Cost |',
|
|
320
|
+
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
|
|
321
|
+
...report.configurations.map(rowText),
|
|
322
|
+
'',
|
|
323
|
+
];
|
|
324
|
+
if (several) {
|
|
325
|
+
lines.push('## Comparisons with the reference', '', 'Each line pairs the cases both configurations scored and is its own exact sign test, so with several lines some can resolve by chance.', '', ...report.comparisons.map(comparisonText), '');
|
|
326
|
+
}
|
|
327
|
+
return lines.map(inertLine).join('\n');
|
|
328
|
+
}
|
|
329
|
+
/** The report as inert JSON and a line break; it parses back to the report. */
|
|
330
|
+
export function reportJson(report) {
|
|
331
|
+
return `${inertJson(report)}\n`;
|
|
332
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every read and write of a bench run directory, which holds `run.json` (written once, whole),
|
|
3
|
+
* `results.jsonl` (one line per review, appended one at a time), and the reports (`report.md` and
|
|
4
|
+
* `report.json`, rewritten whole). Both data files are read as untrusted, since another run or a person
|
|
5
|
+
* may have written them: strict schemas (see `records.ts`), bounded sizes, and no file that is not a
|
|
6
|
+
* regular file. No write follows a symbolic link: `results.jsonl` is opened with `O_NOFOLLOW`, and every
|
|
7
|
+
* other file is written to a new file that is renamed over the name, which replaces a link there.
|
|
8
|
+
*
|
|
9
|
+
* Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
|
|
10
|
+
* runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
|
|
11
|
+
* I/O failure is an `Error`.
|
|
12
|
+
*/
|
|
13
|
+
import type { ConfigIssue } from '../core/errors.ts';
|
|
14
|
+
import type { ReviewRecord, RunFile } from './records.ts';
|
|
15
|
+
export declare const RUN_FILE = "run.json";
|
|
16
|
+
export declare const RESULTS_FILE = "results.jsonl";
|
|
17
|
+
export declare const REPORT_MARKDOWN = "report.md";
|
|
18
|
+
export declare const REPORT_JSON = "report.json";
|
|
19
|
+
/** What `--out` holds: nothing, an empty directory, or a run. */
|
|
20
|
+
export type RunState = {
|
|
21
|
+
readonly kind: 'none';
|
|
22
|
+
} | {
|
|
23
|
+
readonly kind: 'empty';
|
|
24
|
+
} | {
|
|
25
|
+
readonly kind: 'run';
|
|
26
|
+
readonly run: RunFile;
|
|
27
|
+
};
|
|
28
|
+
/** What a read of `results.jsonl` found: the complete lines it visited, where they end, and whether bytes follow the last line break. */
|
|
29
|
+
export interface RecordsRead {
|
|
30
|
+
readonly lines: number;
|
|
31
|
+
/** The length of the file up to and including the last line break. */
|
|
32
|
+
readonly validBytes: number;
|
|
33
|
+
/** A last line with no line break: a run killed while it appended. It is not visited. */
|
|
34
|
+
readonly torn: boolean;
|
|
35
|
+
}
|
|
36
|
+
/** Appends lines to `results.jsonl`, one at a time. */
|
|
37
|
+
export interface RecordWriter {
|
|
38
|
+
readonly append: (line: string) => Promise<void>;
|
|
39
|
+
readonly close: () => Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* What `directory`, the run directory a command was given, holds. A directory that does not exist is
|
|
43
|
+
* `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
|
|
44
|
+
* as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
|
|
45
|
+
* `run.json`, and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
|
|
46
|
+
* format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
|
|
47
|
+
* its parents may be links: they are the user's own path.
|
|
48
|
+
*
|
|
49
|
+
* @throws ConfigError as above; Error when the directory or `run.json` cannot be read
|
|
50
|
+
*/
|
|
51
|
+
export declare function inspectRunDir(directory: string): Promise<RunState>;
|
|
52
|
+
/**
|
|
53
|
+
* What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
|
|
54
|
+
* quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
|
|
55
|
+
* which holds the preset by name), then another suite by its name, else a suite whose content changed,
|
|
56
|
+
* then each current configuration the run does not hold, at `--config`, then each recorded one that is
|
|
57
|
+
* not given again. A current and a recorded configuration that share a path (`null`, the usual
|
|
58
|
+
* resolution, included) but not a fingerprint are one configuration whose settings changed, and give one
|
|
59
|
+
* issue in place of the two. The same configurations in another order are the same run. A configuration
|
|
60
|
+
* is named by its path as written, or `(default configuration)`: the recorded paths are text from
|
|
61
|
+
* `run.json`, to make inert where they are printed.
|
|
62
|
+
*/
|
|
63
|
+
export declare function runDifferences(recorded: RunFile, current: Pick<RunFile, 'quorum' | 'suite' | 'configurations'>): ConfigIssue[];
|
|
64
|
+
/**
|
|
65
|
+
* Make `directory` and its parents, then write `run.json` into it whole, through a new file and a rename.
|
|
66
|
+
*
|
|
67
|
+
* @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
|
|
68
|
+
*/
|
|
69
|
+
export declare function createRun(directory: string, run: RunFile): Promise<void>;
|
|
70
|
+
/**
|
|
71
|
+
* Read `results.jsonl` line by line and hand each record, with its line number from 1, to `visit`, which
|
|
72
|
+
* returns the problem with a record it refuses, or undefined. A file with no `results.jsonl` holds
|
|
73
|
+
* nothing. Bytes after the last line break are a line a run killed while it appended cut short: it is
|
|
74
|
+
* reported as `torn`, never visited, and the next append must drop it (see `openRecords`). Any other line
|
|
75
|
+
* that is no record, a line over 16 MiB (complete or not), a record `visit` refuses, a link and a file
|
|
76
|
+
* that is none are refused. The file is read in chunks of 64 KiB and split on the line feed byte, and a
|
|
77
|
+
* line over the limit is refused as soon as it is seen to pass it, so no line is read whole before it is
|
|
78
|
+
* known to fit.
|
|
79
|
+
*
|
|
80
|
+
* @throws ConfigError as above; Error `results.jsonl: cannot be read (<code>)` when a read fails
|
|
81
|
+
*/
|
|
82
|
+
export declare function readRecords(directory: string, visit: (record: ReviewRecord, line: number) => string | undefined): Promise<RecordsRead>;
|
|
83
|
+
/**
|
|
84
|
+
* Open `results.jsonl` to append to, making it when it is not there, and cut it to `validBytes` (what
|
|
85
|
+
* `readRecords` found valid), which drops a line a killed run left unfinished. `O_APPEND` puts each write at
|
|
86
|
+
* the end, and `O_NOFOLLOW` refuses a symbolic link. `append` writes one whole line, each call after the
|
|
87
|
+
* one before, so lines never interleave; once a write has failed, none after it is written, so a line
|
|
88
|
+
* is never joined to the fragment of a failed one. `close` waits for the lines asked for and closes.
|
|
89
|
+
*
|
|
90
|
+
* @throws Error `results.jsonl: cannot be written (<code>)` for any failure to open, cut, write or close
|
|
91
|
+
*/
|
|
92
|
+
export declare function openRecords(directory: string, validBytes: number): Promise<RecordWriter>;
|
|
93
|
+
/**
|
|
94
|
+
* Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
|
|
95
|
+
* made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
|
|
96
|
+
* The rename replaces a symbolic link at `name`, and never writes through it. A failure removes the
|
|
97
|
+
* temporary file it made.
|
|
98
|
+
*
|
|
99
|
+
* @throws Error `<name>: cannot be written (<code>)`
|
|
100
|
+
*/
|
|
101
|
+
export declare function writeAtomic(directory: string, name: string, text: string): Promise<void>;
|