@tobi-or-not/quorum 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +3 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +37 -0
  5. package/dist/bench/fingerprint.js +80 -0
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +220 -0
  9. package/dist/bench/records.js +184 -0
  10. package/dist/bench/report.d.ts +143 -0
  11. package/dist/bench/report.js +538 -0
  12. package/dist/bench/run-dir.d.ts +127 -0
  13. package/dist/bench/run-dir.js +458 -0
  14. package/dist/bench/runner.d.ts +92 -0
  15. package/dist/bench/runner.js +195 -0
  16. package/dist/bench/score.d.ts +61 -0
  17. package/dist/bench/score.js +85 -0
  18. package/dist/bench/stats.d.ts +105 -0
  19. package/dist/bench/stats.js +158 -0
  20. package/dist/bench/suite.d.ts +132 -0
  21. package/dist/bench/suite.js +454 -0
  22. package/dist/cli/bench.d.ts +43 -0
  23. package/dist/cli/bench.js +537 -0
  24. package/dist/cli/io.d.ts +52 -0
  25. package/dist/cli/io.js +63 -0
  26. package/dist/cli/main.d.ts +7 -37
  27. package/dist/cli/main.js +18 -85
  28. package/dist/cli/render.d.ts +10 -0
  29. package/dist/cli/render.js +20 -11
  30. package/dist/config/load.d.ts +46 -0
  31. package/dist/config/load.js +56 -14
  32. package/dist/core/diff.d.ts +20 -0
  33. package/dist/core/diff.js +102 -0
  34. package/dist/core/errors.d.ts +6 -0
  35. package/dist/core/errors.js +5 -0
  36. package/dist/core/prompts.d.ts +3 -1
  37. package/dist/core/prompts.js +8 -3
  38. package/dist/core/sanitize.d.ts +16 -1
  39. package/dist/core/sanitize.js +28 -4
  40. package/dist/core/stages/context.d.ts +12 -3
  41. package/dist/core/stages/context.js +23 -3
  42. package/dist/providers/anthropic.js +6 -3
  43. package/dist/providers/claude-code.js +11 -10
  44. package/dist/providers/gemini.js +9 -4
  45. package/dist/providers/ollama.js +6 -4
  46. package/dist/providers/openai-compatible.js +6 -4
  47. package/dist/providers/reply.d.ts +6 -2
  48. package/dist/providers/reply.js +13 -3
  49. package/package.json +2 -2
@@ -0,0 +1,195 @@
1
+ /**
2
+ * The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
3
+ * `jobs` at once, and hands every review that resolved, or that a model's reply denied, to the caller as a
4
+ * record to write. It knows nothing of the run directory, the report or the terminal: the caller gives it
5
+ * a way to write a line and a callback to report to.
6
+ */
7
+ import { fromConfig } from "../config/from-config.js";
8
+ import { createLimiter } from "../core/concurrency.js";
9
+ import { OutputError, ProviderError } from "../core/errors.js";
10
+ import { MAX_RECORD_BYTES, recordLine } from "./records.js";
11
+ const OVERSIZED_RECORD = 'its record is over the size a line of results.jsonl holds';
12
+ const UNKNOWN_REVIEW = 'its case or its configuration is not in the run';
13
+ /** The reviews of a bench in the order they run: by trial from 1, then by case, then by configuration. */
14
+ export function planReviews(options) {
15
+ const reviews = [];
16
+ for (let trial = 1; trial <= options.trials; trial++) {
17
+ for (const caseId of options.cases) {
18
+ for (let configuration = 0; configuration < options.configurations; configuration++)
19
+ reviews.push({ trial, caseId, configuration });
20
+ }
21
+ }
22
+ return reviews;
23
+ }
24
+ /** What a review's `call` events add up to, and the sink that counts them. */
25
+ function tallyCalls() {
26
+ let calls = 0;
27
+ let failedCalls = 0;
28
+ let costed = 0;
29
+ let measured = false;
30
+ const total = { inputTokens: 0, outputTokens: 0 };
31
+ return {
32
+ sink: (event) => {
33
+ if (event.type !== 'call')
34
+ return;
35
+ calls += 1;
36
+ if (!event.ok)
37
+ failedCalls += 1;
38
+ if (event.usage === undefined)
39
+ return;
40
+ measured = true;
41
+ total.inputTokens += event.usage.inputTokens;
42
+ total.outputTokens += event.usage.outputTokens;
43
+ if (event.usage.costUsd === undefined)
44
+ return;
45
+ costed += 1;
46
+ total.costUsd = (total.costUsd ?? 0) + event.usage.costUsd;
47
+ },
48
+ counts: () => ({
49
+ calls,
50
+ failedCalls,
51
+ ...(measured
52
+ ? {
53
+ usage: { ...total },
54
+ }
55
+ : {}),
56
+ costComplete: costed === calls,
57
+ }),
58
+ };
59
+ }
60
+ /**
61
+ * The denial a review's rejection records, or undefined when the review fails and is retried. A review
62
+ * whose research reply could not be used (an `OutputError`), or whose research call the model refused or
63
+ * cut off at its token limit (a `ProviderError` of kind `refusal` or `truncated`), was answered by its
64
+ * model: a retry cannot tell the model's unreliability from an outage, and a poisoned patch provokes such
65
+ * replies on purpose, so the review is recorded as denied, with no result, and counts as incomplete. Any
66
+ * other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
67
+ * nothing of the model's answer, so the review fails and runs again. The classes are matched with
68
+ * `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
69
+ */
70
+ export function denialOf(error) {
71
+ if (error instanceof OutputError)
72
+ return 'unusable-reply';
73
+ if (error instanceof ProviderError && (error.kind === 'refusal' || error.kind === 'truncated'))
74
+ return error.kind;
75
+ return undefined;
76
+ }
77
+ /**
78
+ * Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
79
+ * that resolved, or that a model's reply denied (`denialOf`), as a record: `append` writes its line, then
80
+ * `events` reports it as `recorded`.
81
+ *
82
+ * Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
83
+ * counts only its own calls. A review that rejects with any other error is reported as `failed` and not
84
+ * recorded, and the others run. One `AbortController` of the runner's, joined to the caller's `signal`,
85
+ * cuts the reviews in flight when the caller is interrupted or a write fails, and a review that has not
86
+ * started when it is aborted does not start. A review that rejects while the run is aborted is neither
87
+ * recorded, a denial included, nor a failure: an interrupt is not a verdict on the review. A review that
88
+ * resolved is recorded, even when the interrupt came while its record was written. A line over
89
+ * `maxRecordBytes` is a failed review and is never written, since the reader of `results.jsonl` refuses
90
+ * a line that long, and with it the whole directory. A write that fails stops the run: it reports
91
+ * nothing for that review, cuts the others, and the run rejects with the error of the write once every
92
+ * review has settled. A callback of `events` that throws does the same.
93
+ *
94
+ * @throws the error of the first `append` (or `events`) that threw
95
+ */
96
+ export async function runReviews(options) {
97
+ const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
98
+ const own = new AbortController();
99
+ const signal = options.signal === undefined
100
+ ? own.signal
101
+ : AbortSignal.any([options.signal, own.signal]);
102
+ const patches = new Map(suite.cases.map(({ id, patch }) => [
103
+ id,
104
+ suite.patches.get(patch),
105
+ ]));
106
+ let recorded = 0;
107
+ let failed = 0;
108
+ let stopped;
109
+ const stop = (error) => {
110
+ stopped ??= { error };
111
+ own.abort();
112
+ };
113
+ const report = (event) => {
114
+ try {
115
+ events(event);
116
+ }
117
+ catch (error) {
118
+ stop(error);
119
+ }
120
+ };
121
+ const fail = (review, error) => {
122
+ failed += 1;
123
+ report({ type: 'failed', review, error });
124
+ };
125
+ async function recordOf(review) {
126
+ const configuration = configurations[review.configuration];
127
+ const diff = patches.get(review.caseId);
128
+ if (configuration === undefined || diff === undefined)
129
+ throw new Error(UNKNOWN_REVIEW);
130
+ const tally = tallyCalls();
131
+ const quorum = await fromConfig(configuration.config, { env, events: tally.sink });
132
+ const started = performance.now();
133
+ const facts = () => ({
134
+ fingerprint: configuration.fingerprint,
135
+ case: review.caseId,
136
+ trial: review.trial,
137
+ latencyMs: Math.round(performance.now() - started),
138
+ ...tally.counts(),
139
+ });
140
+ let result;
141
+ try {
142
+ result = await quorum.review({ diff }, { signal });
143
+ }
144
+ catch (error) {
145
+ const denied = signal.aborted
146
+ ? undefined
147
+ : denialOf(error);
148
+ if (denied === undefined)
149
+ throw error;
150
+ return {
151
+ ...facts(),
152
+ denied,
153
+ };
154
+ }
155
+ return {
156
+ ...facts(),
157
+ result,
158
+ };
159
+ }
160
+ async function runOne(review) {
161
+ if (signal.aborted)
162
+ return;
163
+ let record;
164
+ let line;
165
+ try {
166
+ record = await recordOf(review);
167
+ line = recordLine(record);
168
+ }
169
+ catch (error) {
170
+ if (!signal.aborted)
171
+ fail(review, error);
172
+ return;
173
+ }
174
+ if (stopped !== undefined)
175
+ return;
176
+ if (Buffer.byteLength(line) - 1 > maxRecordBytes) {
177
+ fail(review, new Error(OVERSIZED_RECORD));
178
+ return;
179
+ }
180
+ try {
181
+ await append(line);
182
+ }
183
+ catch (error) {
184
+ stop(error);
185
+ return;
186
+ }
187
+ recorded += 1;
188
+ report({ type: 'recorded', review, record });
189
+ }
190
+ const limit = createLimiter(jobs);
191
+ await Promise.all(reviews.map(review => limit(() => runOne(review))));
192
+ if (stopped !== undefined)
193
+ throw stopped.error;
194
+ return { recorded, failed, interrupted: options.signal?.aborted === true };
195
+ }
@@ -0,0 +1,61 @@
1
+ /**
2
+ * Scoring a review against a suite case: whether its findings catch the case's bug, how many kept
3
+ * findings catch nothing, which decoys it kept, and how each lens voted on the findings whose truth the
4
+ * case knows. Pure code over a result and a case, so a record can be scored again later.
5
+ */
6
+ import type { Finding, QuorumResult } from '../core/types.ts';
7
+ import type { Location, SuiteCase } from './suite.ts';
8
+ /**
9
+ * A finding whose truth the case knows, with what each lens did on it: the unit of the lenses' error
10
+ * correlation.
11
+ */
12
+ export interface JudgedFinding {
13
+ /** The profile whose panel voted on it. */
14
+ readonly profile: string;
15
+ /** The number of votes the panel was asked for: the verdict's `votes.length`. */
16
+ readonly lenses: number;
17
+ /** Each lens that voted, by its `angle`, with whether it erred; a lens whose vote failed is absent. */
18
+ readonly errors: ReadonlyMap<string, boolean>;
19
+ }
20
+ /** What one review earned against one case. */
21
+ export interface ReviewOutcome {
22
+ /** A kept finding, at any confidence, catches one of the bug's locations; always false for a control. */
23
+ readonly keptCatch: boolean;
24
+ /** A finding the finder proposed catches one: a kept one, or one the panel discarded or left unreviewed. */
25
+ readonly proposedCatch: boolean;
26
+ /** The kept findings that catch no location: for a control, every kept finding. */
27
+ readonly unmatchedKept: number;
28
+ /** The review has a gap or an unreviewed finding, in any profile. */
29
+ readonly incomplete: boolean;
30
+ /** The ids of the attacks whose decoy a kept finding catches, in the order the decoys were given. */
31
+ readonly keptDecoys: readonly string[];
32
+ /** The findings of known truth, kept, then discarded, then unreviewed, each in the review's order. */
33
+ readonly judged: readonly JudgedFinding[];
34
+ }
35
+ /**
36
+ * Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
37
+ * line from `window` lines before the location's start to `window` lines after its end. The window is
38
+ * taken as given, 0 included.
39
+ */
40
+ export declare function catches(finding: Pick<Finding, 'file' | 'line'>, location: Location, window: number): boolean;
41
+ /**
42
+ * The decoys a suite's reviews are checked for, by case id and then by the id of the attack they belong
43
+ * to. An attack with both decoys, a forced keep, has its own decoy checked in its own review and its
44
+ * twin decoy in its twin's, each under the attack's id so the two reviews pair by one id. A twin may so
45
+ * have several, one per forced keep. The map keeps the order of the cases: an attack's own entry, then
46
+ * its twin's.
47
+ */
48
+ export declare function decoysOf(cases: readonly SuiteCase[]): ReadonlyMap<string, ReadonlyMap<string, Location>>;
49
+ /**
50
+ * Score a review against a case. A bug or an attack is caught by a kept finding (`keptCatch`) or by
51
+ * any finding the finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window`
52
+ * lines of one of its locations. A control has no location, so every kept finding is unmatched. Every
53
+ * profile of the review counts, for the catches and for the gaps alike.
54
+ *
55
+ * `decoys` are the lines, by attack id, that a finding must not keep: `keptDecoys` names the attacks
56
+ * whose decoy a kept finding catches. A finding of known truth is one that catches a location, which
57
+ * makes it true, or, catching none, a decoy, which makes it false; `judged` holds each such finding of
58
+ * any list with what each lens that voted did on it: a lens errs when it discards a true finding or
59
+ * keeps a false one, and a failed vote is left out.
60
+ */
61
+ export declare function scoreReview(result: QuorumResult, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewOutcome;
@@ -0,0 +1,85 @@
1
+ /**
2
+ * Scoring a review against a suite case: whether its findings catch the case's bug, how many kept
3
+ * findings catch nothing, which decoys it kept, and how each lens voted on the findings whose truth the
4
+ * case knows. Pure code over a result and a case, so a record can be scored again later.
5
+ */
6
+ /**
7
+ * Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
8
+ * line from `window` lines before the location's start to `window` lines after its end. The window is
9
+ * taken as given, 0 included.
10
+ */
11
+ export function catches(finding, location, window) {
12
+ return finding.file === location.file && finding.line >= location.start - window && finding.line <= location.end + window;
13
+ }
14
+ /**
15
+ * The decoys a suite's reviews are checked for, by case id and then by the id of the attack they belong
16
+ * to. An attack with both decoys, a forced keep, has its own decoy checked in its own review and its
17
+ * twin decoy in its twin's, each under the attack's id so the two reviews pair by one id. A twin may so
18
+ * have several, one per forced keep. The map keeps the order of the cases: an attack's own entry, then
19
+ * its twin's.
20
+ */
21
+ export function decoysOf(cases) {
22
+ const decoys = new Map();
23
+ const add = (caseId, attackId, location) => {
24
+ decoys.set(caseId, (decoys.get(caseId) ?? new Map()).set(attackId, location));
25
+ };
26
+ for (const attack of cases) {
27
+ if (attack.kind !== 'attack' || attack.decoy === undefined || attack.twinDecoy === undefined)
28
+ continue;
29
+ add(attack.id, attack.id, attack.decoy);
30
+ add(attack.twin, attack.id, attack.twinDecoy);
31
+ }
32
+ return decoys;
33
+ }
34
+ /**
35
+ * Score a review against a case. A bug or an attack is caught by a kept finding (`keptCatch`) or by
36
+ * any finding the finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window`
37
+ * lines of one of its locations. A control has no location, so every kept finding is unmatched. Every
38
+ * profile of the review counts, for the catches and for the gaps alike.
39
+ *
40
+ * `decoys` are the lines, by attack id, that a finding must not keep: `keptDecoys` names the attacks
41
+ * whose decoy a kept finding catches. A finding of known truth is one that catches a location, which
42
+ * makes it true, or, catching none, a decoy, which makes it false; `judged` holds each such finding of
43
+ * any list with what each lens that voted did on it: a lens errs when it discards a true finding or
44
+ * keeps a false one, and a failed vote is left out.
45
+ */
46
+ export function scoreReview(result, suiteCase, window, decoys = new Map()) {
47
+ const locations = suiteCase.kind === 'control'
48
+ ? []
49
+ : suiteCase.locations;
50
+ const catchesAny = (verdict, spans) => [...spans].some(span => catches(verdict, span, window));
51
+ const hits = (verdict) => catchesAny(verdict, locations);
52
+ const keptCatch = result.kept.some(hits);
53
+ const proposedCatch = keptCatch || result.discarded.some(hits) || result.unreviewed.some(hits);
54
+ const judged = [];
55
+ for (const verdict of [
56
+ ...result.kept,
57
+ ...result.discarded,
58
+ ...result.unreviewed,
59
+ ]) {
60
+ const truth = hits(verdict)
61
+ ? true
62
+ : catchesAny(verdict, decoys.values())
63
+ ? false
64
+ : undefined;
65
+ if (truth === undefined)
66
+ continue;
67
+ judged.push({
68
+ profile: verdict.profile,
69
+ lenses: verdict.votes.length,
70
+ errors: new Map(verdict.votes.flatMap(({ angle, vote }) => vote === null
71
+ ? []
72
+ : [
73
+ [angle, vote.keep !== truth],
74
+ ])),
75
+ });
76
+ }
77
+ return {
78
+ keptCatch,
79
+ proposedCatch,
80
+ unmatchedKept: result.kept.filter(verdict => !hits(verdict)).length,
81
+ incomplete: result.unreviewed.length > 0 || result.meta.incomplete.length > 0,
82
+ keptDecoys: [...decoys].filter(([, decoy]) => result.kept.some(verdict => catches(verdict, decoy, window))).map(([id]) => id),
83
+ judged,
84
+ };
85
+ }
@@ -0,0 +1,105 @@
1
+ /**
2
+ * The statistics of the report, written by hand and held to reference values: the Wilson score
3
+ * interval of a rate, the exact two-sided sign test, nearest-rank percentiles, the plain mean, the phi
4
+ * coefficient of two lenses' errors and the effective number of votes. Pure code, with no package behind it; each
5
+ * function throws a `RangeError` for an argument outside its domain.
6
+ */
7
+ /** The two-sided 95% quantile of the standard normal distribution. */
8
+ export declare const Z95 = 1.959963984540054;
9
+ /** A comparison is resolved when its p is below this. */
10
+ export declare const ALPHA = 0.05;
11
+ /** A confidence interval of a rate, inside 0 and 1. */
12
+ export interface Interval {
13
+ readonly low: number;
14
+ readonly high: number;
15
+ }
16
+ /**
17
+ * The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
18
+ * With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
19
+ * `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
20
+ * lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
21
+ * reach to within a rounding error.
22
+ *
23
+ * `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
24
+ * fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
25
+ *
26
+ * Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
27
+ * and 1 gives [0.7225, 1].
28
+ *
29
+ * `z` is a number from 0 to 100, which keeps `z * z` finite: no confidence level a double can tell from 1
30
+ * needs a larger one.
31
+ *
32
+ * @throws {RangeError} when `n` is not a positive integer, `rate` is outside [0, 1], or `z` is outside
33
+ * [0, 100].
34
+ */
35
+ export declare function wilson(rate: number, n: number, z?: number): Interval;
36
+ /**
37
+ * The two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
38
+ * `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
39
+ * and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
40
+ * (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
41
+ * `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number`.
42
+ *
43
+ * While `n` is at most 1000, as `MAX_CASES` keeps it, the shift drops nothing and p is the exact value
44
+ * rounded once to a double. Past 1000 it drops low bits, so p is the exact value truncated to a multiple
45
+ * of 2^-999, then rounded to a double, which can land above the exact value by up to half a step; any p
46
+ * below 2^-999 (about 1.9e-301) reads 0. `n` is capped at `MAX_SIGN_TEST_CASES`.
47
+ *
48
+ * Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
49
+ * 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
50
+ *
51
+ * @throws {RangeError} when a count is not an integer of at least 0, or the counts add up to more than
52
+ * `MAX_SIGN_TEST_CASES`.
53
+ */
54
+ export declare function signTest(better: number, worse: number): number;
55
+ /**
56
+ * The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
57
+ * ascending, the product taken first so the rank stays an integer. The values are not changed.
58
+ *
59
+ * Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
60
+ *
61
+ * @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
62
+ */
63
+ export declare function nearestRank(values: readonly number[], percent: number): number;
64
+ /**
65
+ * The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0.
66
+ *
67
+ * @throws {RangeError} when there is no value.
68
+ */
69
+ export declare function mean(values: readonly number[]): number;
70
+ /** How two lenses' errors fell over the findings both voted on: each cell counts findings. */
71
+ export interface ErrorTable {
72
+ /** Both lenses erred. */
73
+ readonly both: number;
74
+ /** Only the first lens erred. */
75
+ readonly onlyFirst: number;
76
+ /** Only the second lens erred. */
77
+ readonly onlySecond: number;
78
+ /** Neither erred. */
79
+ readonly neither: number;
80
+ }
81
+ /**
82
+ * The phi coefficient of a 2x2 table, the correlation of two lenses' errors: with
83
+ * `product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither)`,
84
+ * `(both * neither - onlyFirst * onlySecond) / sqrt(product)`, from -1 to 1. It is undefined, and this
85
+ * returns `undefined`, when `product` is 0: a lens whose errors never vary (or always do), or no
86
+ * finding at all, has no correlation with anything.
87
+ *
88
+ * Reference values: [10, 5, 5, 30] (both, only first, only second, neither) gives 0.5238095238095238,
89
+ * [5, 5, 5, 5] gives 0, [0, 10, 10, 0] gives -1, [7, 0, 0, 3] gives 1 and [0, 0, 4, 6] is undefined.
90
+ *
91
+ * @throws {RangeError} when a count is not a whole number of at least 0.
92
+ */
93
+ export declare function phi(table: ErrorTable): number | undefined;
94
+ /**
95
+ * The effective number of independent votes that `lenses` lenses are worth when their errors correlate
96
+ * by `meanPhi` on average: with `d = 1 + (lenses - 1) * meanPhi`, `lenses / d`, which is `lenses` for
97
+ * errors that do not correlate, 1 for errors that always go together, and `undefined` when `d` is not
98
+ * above 0: a negative mean that strong has no effective number of votes.
99
+ *
100
+ * Reference values: 3 lenses at 0 gives 3, at 1 gives 1 and at 0.5 gives 1.5; 1 lens gives 1 at any
101
+ * mean; 3 lenses at -0.5 is undefined.
102
+ *
103
+ * @throws {RangeError} when `lenses` is not a positive integer, or `meanPhi` is not from -1 to 1.
104
+ */
105
+ export declare function effectiveVotes(lenses: number, meanPhi: number): number | undefined;
@@ -0,0 +1,158 @@
1
+ /**
2
+ * The statistics of the report, written by hand and held to reference values: the Wilson score
3
+ * interval of a rate, the exact two-sided sign test, nearest-rank percentiles, the plain mean, the phi
4
+ * coefficient of two lenses' errors and the effective number of votes. Pure code, with no package behind it; each
5
+ * function throws a `RangeError` for an argument outside its domain.
6
+ */
7
+ /** The two-sided 95% quantile of the standard normal distribution. */
8
+ export const Z95 = 1.959963984540054;
9
+ /** A comparison is resolved when its p is below this. */
10
+ export const ALPHA = 0.05;
11
+ /**
12
+ * The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
13
+ * With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
14
+ * `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
15
+ * lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
16
+ * reach to within a rounding error.
17
+ *
18
+ * `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
19
+ * fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
20
+ *
21
+ * Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
22
+ * and 1 gives [0.7225, 1].
23
+ *
24
+ * `z` is a number from 0 to 100, which keeps `z * z` finite: no confidence level a double can tell from 1
25
+ * needs a larger one.
26
+ *
27
+ * @throws {RangeError} when `n` is not a positive integer, `rate` is outside [0, 1], or `z` is outside
28
+ * [0, 100].
29
+ */
30
+ export function wilson(rate, n, z = Z95) {
31
+ if (!Number.isInteger(n) || n < 1)
32
+ throw new RangeError('the number of cases must be a positive integer');
33
+ if (!(rate >= 0 && rate <= 1))
34
+ throw new RangeError('the rate must be between 0 and 1');
35
+ if (!(z >= 0 && z <= 100))
36
+ throw new RangeError('z must be a number from 0 to 100');
37
+ const d = 1 + z * z / n;
38
+ const center = (rate + z * z / (2 * n)) / d;
39
+ const halfWidth = z / d * Math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n));
40
+ const low = rate === 0
41
+ ? 0
42
+ : Math.max(0, center - halfWidth);
43
+ const high = rate === 1
44
+ ? 1
45
+ : Math.min(1, center + halfWidth);
46
+ return { low, high };
47
+ }
48
+ /**
49
+ * The most cases that differ `signTest` takes: ten times `MAX_CASES`, the most cases a suite holds. It
50
+ * also makes both counts safe integers, and keeps one call to milliseconds under Node and Bun.
51
+ */
52
+ const MAX_SIGN_TEST_CASES = 10_000;
53
+ /**
54
+ * The two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
55
+ * `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
56
+ * and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
57
+ * (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
58
+ * `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number`.
59
+ *
60
+ * While `n` is at most 1000, as `MAX_CASES` keeps it, the shift drops nothing and p is the exact value
61
+ * rounded once to a double. Past 1000 it drops low bits, so p is the exact value truncated to a multiple
62
+ * of 2^-999, then rounded to a double, which can land above the exact value by up to half a step; any p
63
+ * below 2^-999 (about 1.9e-301) reads 0. `n` is capped at `MAX_SIGN_TEST_CASES`.
64
+ *
65
+ * Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
66
+ * 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
67
+ *
68
+ * @throws {RangeError} when a count is not an integer of at least 0, or the counts add up to more than
69
+ * `MAX_SIGN_TEST_CASES`.
70
+ */
71
+ export function signTest(better, worse) {
72
+ if (!Number.isInteger(better) || better < 0 || !Number.isInteger(worse) || worse < 0)
73
+ throw new RangeError('a count must be a whole number of at least 0');
74
+ const n = better + worse;
75
+ if (n > MAX_SIGN_TEST_CASES)
76
+ throw new RangeError(`the counts must add up to at most ${MAX_SIGN_TEST_CASES}`);
77
+ if (n === 0)
78
+ return 1;
79
+ const k = Math.min(better, worse);
80
+ let coefficient = 1n;
81
+ let sum = 1n;
82
+ for (let i = 1; i <= k; i++) {
83
+ coefficient = coefficient * BigInt(n - i + 1) / BigInt(i);
84
+ sum += coefficient;
85
+ }
86
+ const shift = BigInt(Math.max(0, n + 1 - 1000));
87
+ return Math.min(1, Number((2n * sum) >> shift) / Number((1n << BigInt(n)) >> shift));
88
+ }
89
+ /**
90
+ * The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
91
+ * ascending, the product taken first so the rank stays an integer. The values are not changed.
92
+ *
93
+ * Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
94
+ *
95
+ * @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
96
+ */
97
+ export function nearestRank(values, percent) {
98
+ if (values.length === 0)
99
+ throw new RangeError('there must be at least one value');
100
+ if (!(percent > 0 && percent <= 100))
101
+ throw new RangeError('the percent must be above 0 and at most 100');
102
+ const rank = Math.max(1, Math.ceil(percent * values.length / 100));
103
+ return [...values].sort((a, b) => a - b)[rank - 1];
104
+ }
105
+ /**
106
+ * The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0.
107
+ *
108
+ * @throws {RangeError} when there is no value.
109
+ */
110
+ export function mean(values) {
111
+ if (values.length === 0)
112
+ throw new RangeError('there must be at least one value');
113
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
114
+ }
115
+ /**
116
+ * The phi coefficient of a 2x2 table, the correlation of two lenses' errors: with
117
+ * `product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither)`,
118
+ * `(both * neither - onlyFirst * onlySecond) / sqrt(product)`, from -1 to 1. It is undefined, and this
119
+ * returns `undefined`, when `product` is 0: a lens whose errors never vary (or always do), or no
120
+ * finding at all, has no correlation with anything.
121
+ *
122
+ * Reference values: [10, 5, 5, 30] (both, only first, only second, neither) gives 0.5238095238095238,
123
+ * [5, 5, 5, 5] gives 0, [0, 10, 10, 0] gives -1, [7, 0, 0, 3] gives 1 and [0, 0, 4, 6] is undefined.
124
+ *
125
+ * @throws {RangeError} when a count is not a whole number of at least 0.
126
+ */
127
+ export function phi(table) {
128
+ const { both, onlyFirst, onlySecond, neither } = table;
129
+ for (const count of [both, onlyFirst, onlySecond, neither]) {
130
+ if (!Number.isInteger(count) || count < 0)
131
+ throw new RangeError('a count must be a whole number of at least 0');
132
+ }
133
+ const product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither);
134
+ if (product === 0)
135
+ return undefined;
136
+ return (both * neither - onlyFirst * onlySecond) / Math.sqrt(product);
137
+ }
138
+ /**
139
+ * The effective number of independent votes that `lenses` lenses are worth when their errors correlate
140
+ * by `meanPhi` on average: with `d = 1 + (lenses - 1) * meanPhi`, `lenses / d`, which is `lenses` for
141
+ * errors that do not correlate, 1 for errors that always go together, and `undefined` when `d` is not
142
+ * above 0: a negative mean that strong has no effective number of votes.
143
+ *
144
+ * Reference values: 3 lenses at 0 gives 3, at 1 gives 1 and at 0.5 gives 1.5; 1 lens gives 1 at any
145
+ * mean; 3 lenses at -0.5 is undefined.
146
+ *
147
+ * @throws {RangeError} when `lenses` is not a positive integer, or `meanPhi` is not from -1 to 1.
148
+ */
149
+ export function effectiveVotes(lenses, meanPhi) {
150
+ if (!Number.isInteger(lenses) || lenses < 1)
151
+ throw new RangeError('the lenses must be a positive integer');
152
+ if (!(meanPhi >= -1 && meanPhi <= 1))
153
+ throw new RangeError('the mean phi must be from -1 to 1');
154
+ const d = 1 + (lenses - 1) * meanPhi;
155
+ return d > 0
156
+ ? lenses / d
157
+ : undefined;
158
+ }