@tobi-or-not/quorum 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +3 -2
  5. package/dist/bench/fingerprint.js +3 -2
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +171 -13
  9. package/dist/bench/records.js +34 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +29 -17
  15. package/dist/bench/runner.js +53 -18
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +20 -9
  23. package/dist/cli/bench.js +148 -74
  24. package/dist/cli/io.d.ts +9 -2
  25. package/dist/cli/io.js +20 -2
  26. package/dist/cli/main.js +2 -19
  27. package/dist/config/load.d.ts +3 -0
  28. package/dist/config/load.js +17 -8
  29. package/dist/core/diff.d.ts +10 -0
  30. package/dist/core/diff.js +77 -0
  31. package/dist/core/errors.d.ts +6 -0
  32. package/dist/core/errors.js +5 -0
  33. package/dist/core/stages/context.d.ts +12 -3
  34. package/dist/core/stages/context.js +23 -3
  35. package/dist/providers/anthropic.js +6 -3
  36. package/dist/providers/claude-code.js +11 -10
  37. package/dist/providers/gemini.js +9 -4
  38. package/dist/providers/ollama.js +6 -4
  39. package/dist/providers/openai-compatible.js +6 -4
  40. package/dist/providers/reply.d.ts +6 -2
  41. package/dist/providers/reply.js +13 -3
  42. package/package.json +1 -1
@@ -1,32 +1,45 @@
1
1
  /**
2
2
  * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
3
  * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
- * report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
5
- * never text a model wrote.
4
+ * report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
5
+ * the suite's two hashes, never text a model wrote.
6
6
  */
7
7
  import { inertJson, inertLine } from "../core/sanitize.js";
8
+ import { buildAttacks } from "./attacks.js";
9
+ import { independence } from "./independence.js";
8
10
  import { reviewKey } from "./records.js";
9
11
  import { scoreReview } from "./score.js";
10
- import { nearestRank, signTest, wilson } from "./stats.js";
11
- /** A comparison is resolved when its p is below this. */
12
- const ALPHA = 0.05;
13
- /** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
14
- export function reviewScore(record, suiteCase, window) {
12
+ import { ALPHA, mean, nearestRank, signTest, wilson } from "./stats.js";
13
+ /**
14
+ * Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
15
+ * tokens (input plus output) and the output tokens when there is usage, and the cost when every call
16
+ * reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
17
+ */
18
+ export function reviewScore(record, suiteCase, window, decoys = new Map()) {
15
19
  return {
16
- ...scoreReview(record.result, suiteCase, window),
20
+ ...('result' in record
21
+ ? {
22
+ ...scoreReview(record.result, suiteCase, window, decoys),
23
+ denied: false,
24
+ }
25
+ : {
26
+ keptCatch: false,
27
+ proposedCatch: false,
28
+ unmatchedKept: 0,
29
+ incomplete: true,
30
+ keptDecoys: [],
31
+ judged: [],
32
+ denied: true,
33
+ }),
17
34
  latencyMs: record.latencyMs,
18
35
  ...(record.usage === undefined
19
36
  ? {}
20
- : { tokens: record.usage.inputTokens + record.usage.outputTokens }),
37
+ : { tokens: record.usage.inputTokens + record.usage.outputTokens, outputTokens: record.usage.outputTokens }),
21
38
  ...(record.costComplete
22
39
  ? { costUsd: record.usage?.costUsd ?? 0 }
23
40
  : {}),
24
41
  };
25
42
  }
26
- /** The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0. */
27
- function mean(values) {
28
- return values.reduce((sum, value) => sum + value, 0) / values.length;
29
- }
30
43
  /** The share of `scores` that `pick` holds for. */
31
44
  function share(scores, pick) {
32
45
  return scores.filter(pick).length / scores.length;
@@ -39,12 +52,12 @@ function caseScore(kind, scores) {
39
52
  : [score.tokens]);
40
53
  return {
41
54
  trials: scores.length,
42
- keptCatch: kind === 'bug'
43
- ? share(scores, score => score.keptCatch)
44
- : null,
45
- proposedCatch: kind === 'bug'
46
- ? share(scores, score => score.proposedCatch)
47
- : null,
55
+ keptCatch: kind === 'control'
56
+ ? null
57
+ : share(scores, score => score.keptCatch),
58
+ proposedCatch: kind === 'control'
59
+ ? null
60
+ : share(scores, score => score.proposedCatch),
48
61
  unmatchedKept: mean(scores.map(score => score.unmatchedKept)),
49
62
  incomplete: share(scores, score => score.incomplete),
50
63
  tokens: tokens.length === 0
@@ -90,6 +103,7 @@ function summarize(input, index, counted, cases) {
90
103
  : [value];
91
104
  });
92
105
  const reviews = counted.flat();
106
+ const latencies = reviews.map(score => score.latencyMs);
93
107
  const tokens = reviews.flatMap(({ tokens: used }) => used === undefined
94
108
  ? []
95
109
  : [used]);
@@ -99,6 +113,7 @@ function summarize(input, index, counted, cases) {
99
113
  reviews: reviews.length,
100
114
  planned: input.cases.length * input.trials,
101
115
  incomplete: reviews.filter(score => score.incomplete).length,
116
+ denied: reviews.filter(score => score.denied).length,
102
117
  keptRecall: rate(scored('bug', score => score.keptCatch)),
103
118
  proposedRecall: rate(scored('bug', score => score.proposedCatch)),
104
119
  unmatchedKept: {
@@ -111,11 +126,11 @@ function summarize(input, index, counted, cases) {
111
126
  median: nearestRank(tokens, 50),
112
127
  total: tokens.reduce((sum, used) => sum + used, 0),
113
128
  },
114
- latencyMs: reviews.length === 0
129
+ latencyMs: latencies.length === 0
115
130
  ? null
116
131
  : {
117
- p50: nearestRank(reviews.map(score => score.latencyMs), 50),
118
- p95: nearestRank(reviews.map(score => score.latencyMs), 95),
132
+ p50: nearestRank(latencies, 50),
133
+ p95: nearestRank(latencies, 95),
119
134
  },
120
135
  costUsd: reviews.length > 0 && reviews.every(score => score.costUsd !== undefined)
121
136
  ? reviews.reduce((sum, score) => sum + (score.costUsd ?? 0), 0)
@@ -192,6 +207,15 @@ function compare(rule, cases, index) {
192
207
  resolved: p < ALPHA,
193
208
  };
194
209
  }
210
+ /** How many of `cases` are of each kind. */
211
+ export function kindCounts(cases) {
212
+ const count = (kind) => cases.filter(suiteCase => suiteCase.kind === kind).length;
213
+ return {
214
+ bugCases: count('bug'),
215
+ controls: count('control'),
216
+ attackCases: count('attack'),
217
+ };
218
+ }
195
219
  /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
196
220
  export function buildReport(input) {
197
221
  const counted = input.configurations.map(({ fingerprint }) => countedScores(input, fingerprint));
@@ -203,13 +227,26 @@ export function buildReport(input) {
203
227
  return {
204
228
  suite: input.suite.name,
205
229
  quorum: input.quorum,
230
+ reviewHash: input.suite.reviewHash,
231
+ scoringHash: input.suite.scoringHash,
232
+ firstScoringHash: input.suite.firstScoringHash,
206
233
  trials: input.trials,
207
234
  window: input.suite.window,
208
- bugCases: input.cases.filter(({ kind }) => kind === 'bug').length,
209
- controls: input.cases.filter(({ kind }) => kind === 'control').length,
235
+ ...kindCounts(input.cases),
210
236
  configurations: counted.map((perCase, position) => summarize(input, position + 1, perCase, cases)),
211
237
  comparisons: input.configurations.slice(1).flatMap((_, position) => MEASURES.map(rule => compare(rule, cases, position + 2))),
212
238
  cases,
239
+ independence: counted.map((perCase, position) => ({
240
+ configuration: position + 1,
241
+ ...independence(perCase.flat().flatMap(score => score.judged)),
242
+ })),
243
+ attacks: buildAttacks({
244
+ cases: input.cases,
245
+ trials: input.trials,
246
+ fingerprints: input.configurations.map(({ fingerprint }) => fingerprint),
247
+ scores: input.scores,
248
+ flagged: input.flagged,
249
+ }),
213
250
  };
214
251
  }
215
252
  const NOT_AVAILABLE = 'n/a';
@@ -225,6 +262,12 @@ function plural(count, noun) {
225
262
  ? ''
226
263
  : 's'}`;
227
264
  }
265
+ /** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
266
+ export function kindsText({ bugCases, controls, attackCases }) {
267
+ return attackCases === 0
268
+ ? `${plural(bugCases, 'bug case')} and ${plural(controls, 'control')}`
269
+ : `${plural(bugCases, 'bug case')}, ${plural(controls, 'control')} and ${plural(attackCases, 'attack case')}`;
270
+ }
228
271
  function percent(value) {
229
272
  return `${(value * 100).toFixed(1)}%`;
230
273
  }
@@ -233,19 +276,29 @@ function rateText(rate) {
233
276
  ? NOT_AVAILABLE
234
277
  : `${percent(rate.value)} [${percent(rate.low)}, ${percent(rate.high)}] over ${plural(rate.cases, 'case')}`;
235
278
  }
279
+ /** `value` as two decimals, without the sign of a negative that rounds to zero: never `-0.00`. */
236
280
  function twoDecimals(value) {
237
- return value === null
238
- ? NOT_AVAILABLE
239
- : value.toFixed(2);
281
+ if (value === null)
282
+ return NOT_AVAILABLE;
283
+ const text = value.toFixed(2);
284
+ return Number(text) === 0
285
+ ? Math.abs(value).toFixed(2)
286
+ : text;
240
287
  }
241
288
  function seconds(ms) {
242
289
  return `${(ms / 1000).toFixed(1)} s`;
243
290
  }
244
- /** `value` with its sign always written, its magnitude as `digits` decimals of `value * scale`. */
291
+ /**
292
+ * `value * scale` as `digits` decimals, with its sign written unless it reads as zero: a difference
293
+ * that rounds to zero is `0.0`, never `-0.0` or `+0.0`.
294
+ */
245
295
  function signed(value, scale, digits) {
296
+ const magnitude = Math.abs(value * scale).toFixed(digits);
297
+ if (Number(magnitude) === 0)
298
+ return magnitude;
246
299
  return `${value < 0
247
300
  ? '-'
248
- : '+'}${Math.abs(value * scale).toFixed(digits)}`;
301
+ : '+'}${magnitude}`;
249
302
  }
250
303
  function differenceText(measure, difference) {
251
304
  switch (measure) {
@@ -271,7 +324,9 @@ function rowText(summary) {
271
324
  rateText(summary.keptRecall),
272
325
  rateText(summary.proposedRecall),
273
326
  `${twoDecimals(summary.unmatchedKept.bug)} / ${twoDecimals(summary.unmatchedKept.control)}`,
274
- String(summary.incomplete),
327
+ summary.denied === 0
328
+ ? String(summary.incomplete)
329
+ : `${summary.incomplete} (${summary.denied} denied)`,
275
330
  summary.tokens === null
276
331
  ? NOT_AVAILABLE
277
332
  : `${Math.round(summary.tokens.median)} / ${Math.round(summary.tokens.total)}`,
@@ -299,7 +354,152 @@ function configurationText(summary, several) {
299
354
  ? ' (reference)'
300
355
  : ''}`;
301
356
  }
302
- /** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
357
+ const ATTACK_LABELS = {
358
+ 'suppression-kept': 'Suppression, kept recall',
359
+ 'suppression-proposed': 'Suppression, proposed recall',
360
+ 'forced-keep': 'Forced keep, the decoy kept',
361
+ 'cost-bloat': 'Cost bloat, output tokens per review',
362
+ 'denial-of-review': 'Denial of review, incomplete reviews',
363
+ flags: 'Advisory flags, raised by the patch',
364
+ };
365
+ /** The columns of the table by carrier and by encoding: each goal's first measure, under its goal's name. */
366
+ const ATTACK_COLUMNS = [
367
+ ['suppression-kept', 'Suppression'],
368
+ ['forced-keep', 'Forced keep'],
369
+ ['cost-bloat', 'Cost bloat'],
370
+ ['denial-of-review', 'Denial of review'],
371
+ ];
372
+ /** The measure a case's cell of the table of cases shows: its goal's first. */
373
+ const FIRST_MEASURE = {
374
+ suppression: 'suppression-kept',
375
+ 'forced-keep': 'forced-keep',
376
+ 'cost-bloat': 'cost-bloat',
377
+ 'denial-of-review': 'denial-of-review',
378
+ };
379
+ /** A score of an attack measure: a percentage for a rate, the tokens rounded for cost bloat. */
380
+ function attackValue(measure, value) {
381
+ return measure === 'cost-bloat'
382
+ ? String(Math.round(value))
383
+ : percent(value);
384
+ }
385
+ /** A cell of the attack tables: `<under> / <clean> (<n>)`. */
386
+ function pairedCell(measure, under, clean, count) {
387
+ return `${attackValue(measure, under)} / ${attackValue(measure, clean)} (${count})`;
388
+ }
389
+ /** A score and, for a rate, its interval. */
390
+ function intervalText(value, interval) {
391
+ return interval === null
392
+ ? String(Math.round(value))
393
+ : `${percent(value)} [${percent(interval.low)}, ${percent(interval.high)}]`;
394
+ }
395
+ function twinLineText(line) {
396
+ const label = `- ${ATTACK_LABELS[line.measure]}`;
397
+ if (line.under === null || line.clean === null || line.difference === null)
398
+ return `${label}: no twin with a paired trial`;
399
+ const flags = line.measure === 'flags';
400
+ const [underWord, cleanWord] = flags
401
+ ? ['of the poisoned patches', 'of their twins']
402
+ : ['under attack', 'clean'];
403
+ const [towardWord, againstWord] = flags
404
+ ? ['higher', 'lower']
405
+ : ['toward the attack', 'against'];
406
+ const difference = line.underInterval === null
407
+ ? signed(line.difference, 1, 0)
408
+ : `${signed(line.difference, 100, 1)} points`;
409
+ return `${label}: ${intervalText(line.under, line.underInterval)} ${underWord}, ${intervalText(line.clean, line.cleanInterval)} ${cleanWord}, over ${plural(line.twins, 'twin')}; ${difference}; ${line.toward} ${towardWord}, ${line.against} ${againstWord}, ${line.tied} tied; ${pText(line.p)}, ${line.resolved
410
+ ? 'resolved'
411
+ : 'unresolved'}`;
412
+ }
413
+ function groupRowText(kind, group) {
414
+ const cells = [
415
+ `${kind} ${group.group}`,
416
+ String(group.cases),
417
+ String(group.flagged),
418
+ ...ATTACK_COLUMNS.map(([measure]) => {
419
+ const score = group.measures[measure];
420
+ return score === null
421
+ ? NOT_AVAILABLE
422
+ : pairedCell(measure, score.under, score.clean, score.cases);
423
+ }),
424
+ ];
425
+ return `| ${cells.join(' | ')} |`;
426
+ }
427
+ function attackTableLines(configuration) {
428
+ return [
429
+ `| By | Cases | Flagged | ${ATTACK_COLUMNS.map(([, name]) => name).join(' | ')} |`,
430
+ `| --- | --- | --- | ${ATTACK_COLUMNS.map(() => '---').join(' | ')} |`,
431
+ ...configuration.carriers.map(group => groupRowText('carrier', group)),
432
+ ...configuration.encodings.map(group => groupRowText('encoding', group)),
433
+ ];
434
+ }
435
+ function attackCaseLines(attacks) {
436
+ const positions = attacks.configurations.map(({ configuration }) => String(configuration));
437
+ return [
438
+ `| Case | Twin | Goal | Carrier | Encoding | Flagged | ${positions.join(' | ')} |`,
439
+ `| --- | --- | --- | --- | --- | --- | ${positions.map(() => '---').join(' | ')} |`,
440
+ ...attacks.cases.map(({ id, twin, goal, carrier, encoding, flagged, configurations }) => {
441
+ const measure = FIRST_MEASURE[goal];
442
+ const cells = configurations.map((measures) => {
443
+ const paired = measures[measure];
444
+ return paired === null || paired === undefined
445
+ ? NOT_AVAILABLE
446
+ : pairedCell(measure, paired.under, paired.clean, paired.trials);
447
+ });
448
+ return `| ${[
449
+ id,
450
+ twin,
451
+ goal,
452
+ carrier,
453
+ encoding,
454
+ flagged ? 'yes' : 'no',
455
+ ...cells,
456
+ ].join(' | ')} |`;
457
+ }),
458
+ ];
459
+ }
460
+ /** The attack section: the method in a paragraph, the flags, each configuration's lines and tables, then each case. */
461
+ function attackLines(attacks) {
462
+ return [
463
+ '## Attacks',
464
+ '',
465
+ 'Each attack case is paired with its twin trial by trial, over the trials both recorded. A twin\'s attack cases of one goal give its score under attack, and their paired trials of the twin its clean score. A rate is the mean over the twins, with its 95% Wilson interval over the twins, and each line is an exact sign test over the twins whose two scores differ. A case\'s score, and the scores by carrier and by encoding, are means without an interval.',
466
+ '',
467
+ twinLineText(attacks.flags),
468
+ '',
469
+ ...attacks.configurations.flatMap(configuration => [
470
+ `### Configuration ${configuration.configuration}`,
471
+ '',
472
+ ...configuration.lines.map(twinLineText),
473
+ '',
474
+ ...attackTableLines(configuration),
475
+ '',
476
+ ]),
477
+ '### Cases',
478
+ '',
479
+ ...attackCaseLines(attacks),
480
+ '',
481
+ ];
482
+ }
483
+ function independenceText(entry) {
484
+ const label = `- ${entry.configuration}`;
485
+ if (entry.findings === 0)
486
+ return `${label}: no finding of known truth`;
487
+ if (entry.meanPhi === null)
488
+ return `${label}: no pair of lenses whose errors both vary, over ${plural(entry.findings, 'finding')}`;
489
+ const votes = entry.lenses === null
490
+ ? 'no effective number, since its panels differ in size'
491
+ : entry.effectiveVotes === null
492
+ ? 'no effective number at that mean'
493
+ : `${twoDecimals(entry.effectiveVotes)} effective votes of ${entry.lenses}`;
494
+ return `${label}: mean phi ${twoDecimals(entry.meanPhi)} over ${plural(entry.pairs, 'lens pair')} and ${plural(entry.findings, 'finding')}; ${votes}`;
495
+ }
496
+ /** The line that says a run was re-scored, and its blank line, while the scoring hash is not the one the run began under; no line otherwise. */
497
+ function rescoredLines({ scoringHash, firstScoringHash }) {
498
+ return scoringHash === firstScoringHash
499
+ ? []
500
+ : [`Re-scored: the run began under the scoring hash ${firstScoringHash}; its reviews, of the same patches, are scored here under the scoring hash above.`, ''];
501
+ }
502
+ /** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
303
503
  export function reportMarkdown(report) {
304
504
  const several = report.configurations.length > 1;
305
505
  const trials = report.trials === 1
@@ -308,8 +508,11 @@ export function reportMarkdown(report) {
308
508
  const lines = [
309
509
  `# quorum bench: ${report.suite}`,
310
510
  '',
311
- `quorum ${report.quorum}; ${plural(report.bugCases, 'bug case')} and ${plural(report.controls, 'control')}; ${trials}; window ${plural(report.window, 'line')}.`,
511
+ `quorum ${report.quorum}; ${kindsText(report)}; ${trials}; window ${plural(report.window, 'line')}.`,
512
+ '',
513
+ `Scored under the review hash ${report.reviewHash} and the scoring hash ${report.scoringHash}.`,
312
514
  '',
515
+ ...rescoredLines(report),
313
516
  'A review counts when it is recorded for one of these cases and trials. A case scores the share of its counted trials with the outcome, and a case with none counted is left out. A rate is the mean of the case scores, with its 95% Wilson interval over the cases scored.',
314
517
  '',
315
518
  '## Configurations',
@@ -324,6 +527,9 @@ export function reportMarkdown(report) {
324
527
  if (several) {
325
528
  lines.push('## Comparisons with the reference', '', 'Each line pairs the cases both configurations scored and is its own exact sign test, so with several lines some can resolve by chance.', '', ...report.comparisons.map(comparisonText), '');
326
529
  }
530
+ if (report.attacks !== null)
531
+ lines.push(...attackLines(report.attacks));
532
+ lines.push('## Inter-voter error correlation', '', 'A finding of known truth catches a location, and is true, or a decoy, and is false. A lens errs when it discards a true finding or keeps a false one. The mean phi is the correlation of two lenses\' errors over the findings both voted on, a failed vote left out, averaged over the pairs whose errors both vary; k lenses whose errors correlate that much are worth k / (1 + (k - 1) x the mean phi) votes.', '', ...report.independence.map(independenceText), '');
327
533
  return lines.map(inertLine).join('\n');
328
534
  }
329
535
  /** The report as inert JSON and a line break; it parses back to the report. */
@@ -1,10 +1,12 @@
1
1
  /**
2
- * Every read and write of a bench run directory, which holds `run.json` (written once, whole),
3
- * `results.jsonl` (one line per review, appended one at a time), and the reports (`report.md` and
4
- * `report.json`, rewritten whole). Both data files are read as untrusted, since another run or a person
5
- * may have written them: strict schemas (see `records.ts`), bounded sizes, and no file that is not a
6
- * regular file. No write follows a symbolic link: `results.jsonl` is opened with `O_NOFOLLOW`, and every
7
- * other file is written to a new file that is renamed over the name, which replaces a link there.
2
+ * Every read and write of a bench run directory, which holds `run.json` (written whole when the run begins,
3
+ * and again when a re-score records the suite's new scoring hash), `results.jsonl` (one line per review,
4
+ * appended one at a time), and the reports (`report.md` and `report.json`, rewritten whole). Both data
5
+ * files are read as untrusted, since another run or a person may have written them: strict schemas (see
6
+ * `records.ts`), bounded sizes, and no file that is not a regular file. No write follows a symbolic link:
7
+ * `results.jsonl` is opened with `O_NOFOLLOW`, and every other file is written to a new file that is
8
+ * renamed over the name, which replaces a link there. `run.json` and `results.jsonl` are made readable by
9
+ * their owner only; the reports are made as any new file.
8
10
  *
9
11
  * Every message is fixed text, the name of a file and, for an I/O failure, the error's code; never the
10
12
  * runtime's own message. A directory or a file this run cannot trust as its own is a `ConfigError`; an
@@ -25,14 +27,18 @@ export type RunState = {
25
27
  readonly kind: 'run';
26
28
  readonly run: RunFile;
27
29
  };
28
- /** What a read of `results.jsonl` found: the complete lines it visited, where they end, and whether bytes follow the last line break. */
30
+ /** What a read of `results.jsonl` found: the complete lines it visited, where they end, how long the file was, and whether bytes follow the last line break. */
29
31
  export interface RecordsRead {
30
32
  readonly lines: number;
31
33
  /** The length of the file up to and including the last line break. */
32
34
  readonly validBytes: number;
35
+ /** The length of the file the read found, its torn tail included: 0 for no file. */
36
+ readonly bytes: number;
33
37
  /** A last line with no line break: a run killed while it appended. It is not visited. */
34
38
  readonly torn: boolean;
35
39
  }
40
+ /** What the append needs of a read: where its last whole line ends, and the file's length as the read found it. */
41
+ export type ReadSizes = Pick<RecordsRead, 'validBytes' | 'bytes'>;
36
42
  /** Appends lines to `results.jsonl`, one at a time. */
37
43
  export interface RecordWriter {
38
44
  readonly append: (line: string) => Promise<void>;
@@ -42,7 +48,8 @@ export interface RecordWriter {
42
48
  * What `directory`, the run directory a command was given, holds. A directory that does not exist is
43
49
  * `none`, an empty one `empty`, and one holding a `run.json` this quorum can read is that run. Refused,
44
50
  * as a `ConfigError`: a symbolic link, anything but a directory, a directory that holds files but no
45
- * `run.json`, and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
51
+ * `run.json` (the temporary `run.json` of a killed run aside, which counts as no file and is left where it
52
+ * is), and a `run.json` that is a link, is not a file, is over 1 MiB, or is not a run file of this
46
53
  * format. The path is read as written, but for a trailing separator or `.`, which would follow a link;
47
54
  * its parents may be links: they are the user's own path.
48
55
  *
@@ -52,17 +59,26 @@ export declare function inspectRunDir(directory: string): Promise<RunState>;
52
59
  /**
53
60
  * What stops a run from resuming `recorded` as the run `current` describes, each an issue to print: the
54
61
  * quorum version first of all (a release can change a preset's defaults without changing a fingerprint,
55
- * which holds the preset by name), then another suite by its name, else a suite whose content changed,
56
- * then each current configuration the run does not hold, at `--config`, then each recorded one that is
57
- * not given again. A current and a recorded configuration that share a path (`null`, the usual
58
- * resolution, included) but not a fingerprint are one configuration whose settings changed, and give one
59
- * issue in place of the two. The same configurations in another order are the same run. A configuration
60
- * is named by its path as written, or `(default configuration)`: the recorded paths are text from
61
- * `run.json`, to make inert where they are printed.
62
+ * which holds the preset by name), then another suite by its name, else a suite whose review hash changed
63
+ * (its case ids, its trials or its patches), then each current configuration the run does not hold, at
64
+ * `--config`, then each recorded one that is not given again. A changed scoring hash alone is no
65
+ * difference: the run's records are scored again under the suite as it is now. A current and a recorded
66
+ * configuration that share a path (`null`, the usual resolution, included) but not a fingerprint are one
67
+ * configuration whose settings changed, and give one issue in place of the two. The same configurations in
68
+ * another order are the same run. A configuration is named by its path as written, or
69
+ * `(default configuration)`: the recorded paths are text from `run.json`, to make inert where they are
70
+ * printed.
62
71
  */
63
72
  export declare function runDifferences(recorded: RunFile, current: Pick<RunFile, 'quorum' | 'suite' | 'configurations'>): ConfigIssue[];
64
73
  /**
65
- * Make `directory` and its parents, then write `run.json` into it whole, through a new file and a rename.
74
+ * Write `run.json` into `directory` whole, through a new file and a rename, which replaces the file that
75
+ * was there. `run.json` is readable by its owner only, so a looser mode its owner gave it is not kept.
76
+ *
77
+ * @throws Error `run.json: cannot be written (<code>)`
78
+ */
79
+ export declare function writeRunFile(directory: string, run: RunFile): Promise<void>;
80
+ /**
81
+ * Make `directory` and its parents, then write `run.json` into it (see `writeRunFile`).
66
82
  *
67
83
  * @throws Error `<directory>: cannot be created (<code>)`, or `run.json: cannot be written (<code>)`
68
84
  */
@@ -81,21 +97,31 @@ export declare function createRun(directory: string, run: RunFile): Promise<void
81
97
  */
82
98
  export declare function readRecords(directory: string, visit: (record: ReviewRecord, line: number) => string | undefined): Promise<RecordsRead>;
83
99
  /**
84
- * Open `results.jsonl` to append to, making it when it is not there, and cut it to `validBytes` (what
85
- * `readRecords` found valid), which drops a line a killed run left unfinished. `O_APPEND` puts each write at
86
- * the end, and `O_NOFOLLOW` refuses a symbolic link. `append` writes one whole line, each call after the
87
- * one before, so lines never interleave; once a write has failed, none after it is written, so a line
88
- * is never joined to the fragment of a failed one. `close` waits for the lines asked for and closes.
100
+ * Open `results.jsonl` to append to, and cut it to `read.validBytes` (what `readRecords` found valid),
101
+ * which drops a line a killed run left unfinished. `read` is the answer of the `readRecords` this run
102
+ * began with, or `{ validBytes: 0, bytes: 0 }` for a run that is new: the file must still be `read.bytes`
103
+ * long, or another run may have cut it or appended to it since, and cutting it back would drop that run's
104
+ * records. A read that found no bytes makes the file when it is not there, which loses nothing; one that
105
+ * found bytes never makes it, so a file deleted since is refused with nothing written. `O_APPEND` puts
106
+ * each write at the end, `O_NOFOLLOW` refuses a symbolic link, and `O_NONBLOCK` refuses a FIFO instead of
107
+ * waiting for it. A file made here is readable by its owner only, and one that exists keeps its mode.
108
+ * `append` writes one whole line, each call after the one before, so lines never interleave; once a write
109
+ * has failed, none after it is written, so a line is never joined to the fragment of a failed one. `close`
110
+ * waits for the lines asked for and closes.
89
111
  *
90
- * @throws Error `results.jsonl: cannot be written (<code>)` for any failure to open, cut, write or close
112
+ * @throws ConfigError `results.jsonl: is not a file`, or `results.jsonl: changed after this run read it,
113
+ * so another run may be using this --out: run one at a time` (a file cut, appended to or deleted since the
114
+ * read); Error `results.jsonl: cannot be written (<code>)` for any other failure to open, cut, write or
115
+ * close
91
116
  */
92
- export declare function openRecords(directory: string, validBytes: number): Promise<RecordWriter>;
117
+ export declare function openRecords(directory: string, read: ReadSizes): Promise<RecordWriter>;
93
118
  /**
94
119
  * Write `text` as `name` in `directory` whole or not at all: to a new file `.<name>.<8 hex digits>.tmp`
95
120
  * made with `O_EXCL` (which never follows a link), written, synced and closed, then renamed over `name`.
96
- * The rename replaces a symbolic link at `name`, and never writes through it. A failure removes the
97
- * temporary file it made.
121
+ * The rename replaces a symbolic link at `name`, and never writes through it. The temporary file is made
122
+ * with `mode`, which the rename keeps: Node's own default unless a caller asks for another. A failure
123
+ * removes the temporary file it made.
98
124
  *
99
125
  * @throws Error `<name>: cannot be written (<code>)`
100
126
  */
101
- export declare function writeAtomic(directory: string, name: string, text: string): Promise<void>;
127
+ export declare function writeAtomic(directory: string, name: string, text: string, mode?: number): Promise<void>;