@tobi-or-not/quorum 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +37 -0
- package/dist/bench/fingerprint.js +80 -0
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +220 -0
- package/dist/bench/records.js +184 -0
- package/dist/bench/report.d.ts +143 -0
- package/dist/bench/report.js +538 -0
- package/dist/bench/run-dir.d.ts +127 -0
- package/dist/bench/run-dir.js +458 -0
- package/dist/bench/runner.d.ts +92 -0
- package/dist/bench/runner.js +195 -0
- package/dist/bench/score.d.ts +61 -0
- package/dist/bench/score.js +85 -0
- package/dist/bench/stats.d.ts +105 -0
- package/dist/bench/stats.js +158 -0
- package/dist/bench/suite.d.ts +132 -0
- package/dist/bench/suite.js +454 -0
- package/dist/cli/bench.d.ts +43 -0
- package/dist/cli/bench.js +537 -0
- package/dist/cli/io.d.ts +52 -0
- package/dist/cli/io.js +63 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +18 -85
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +20 -11
- package/dist/config/load.d.ts +46 -0
- package/dist/config/load.js +56 -14
- package/dist/core/diff.d.ts +20 -0
- package/dist/core/diff.js +102 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +8 -3
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +28 -4
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +2 -2
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runner of a bench: it takes the planned reviews and runs each through a quorum of its own, at most
|
|
3
|
+
* `jobs` at once, and hands every review that resolved, or that a model's reply denied, to the caller as a
|
|
4
|
+
* record to write. It knows nothing of the run directory, the report or the terminal: the caller gives it
|
|
5
|
+
* a way to write a line and a callback to report to.
|
|
6
|
+
*/
|
|
7
|
+
import { fromConfig } from "../config/from-config.js";
|
|
8
|
+
import { createLimiter } from "../core/concurrency.js";
|
|
9
|
+
import { OutputError, ProviderError } from "../core/errors.js";
|
|
10
|
+
import { MAX_RECORD_BYTES, recordLine } from "./records.js";
|
|
11
|
+
const OVERSIZED_RECORD = 'its record is over the size a line of results.jsonl holds';
|
|
12
|
+
const UNKNOWN_REVIEW = 'its case or its configuration is not in the run';
|
|
13
|
+
/** The reviews of a bench in the order they run: by trial from 1, then by case, then by configuration. */
|
|
14
|
+
export function planReviews(options) {
|
|
15
|
+
const reviews = [];
|
|
16
|
+
for (let trial = 1; trial <= options.trials; trial++) {
|
|
17
|
+
for (const caseId of options.cases) {
|
|
18
|
+
for (let configuration = 0; configuration < options.configurations; configuration++)
|
|
19
|
+
reviews.push({ trial, caseId, configuration });
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return reviews;
|
|
23
|
+
}
|
|
24
|
+
/** What a review's `call` events add up to, and the sink that counts them. */
|
|
25
|
+
function tallyCalls() {
|
|
26
|
+
let calls = 0;
|
|
27
|
+
let failedCalls = 0;
|
|
28
|
+
let costed = 0;
|
|
29
|
+
let measured = false;
|
|
30
|
+
const total = { inputTokens: 0, outputTokens: 0 };
|
|
31
|
+
return {
|
|
32
|
+
sink: (event) => {
|
|
33
|
+
if (event.type !== 'call')
|
|
34
|
+
return;
|
|
35
|
+
calls += 1;
|
|
36
|
+
if (!event.ok)
|
|
37
|
+
failedCalls += 1;
|
|
38
|
+
if (event.usage === undefined)
|
|
39
|
+
return;
|
|
40
|
+
measured = true;
|
|
41
|
+
total.inputTokens += event.usage.inputTokens;
|
|
42
|
+
total.outputTokens += event.usage.outputTokens;
|
|
43
|
+
if (event.usage.costUsd === undefined)
|
|
44
|
+
return;
|
|
45
|
+
costed += 1;
|
|
46
|
+
total.costUsd = (total.costUsd ?? 0) + event.usage.costUsd;
|
|
47
|
+
},
|
|
48
|
+
counts: () => ({
|
|
49
|
+
calls,
|
|
50
|
+
failedCalls,
|
|
51
|
+
...(measured
|
|
52
|
+
? {
|
|
53
|
+
usage: { ...total },
|
|
54
|
+
}
|
|
55
|
+
: {}),
|
|
56
|
+
costComplete: costed === calls,
|
|
57
|
+
}),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* The denial a review's rejection records, or undefined when the review fails and is retried. A review
|
|
62
|
+
* whose research reply could not be used (an `OutputError`), or whose research call the model refused or
|
|
63
|
+
* cut off at its token limit (a `ProviderError` of kind `refusal` or `truncated`), was answered by its
|
|
64
|
+
* model: a retry cannot tell the model's unreliability from an outage, and a poisoned patch provokes such
|
|
65
|
+
* replies on purpose, so the review is recorded as denied, with no result, and counts as incomplete. Any
|
|
66
|
+
* other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
|
|
67
|
+
* nothing of the model's answer, so the review fails and runs again. The classes are matched with
|
|
68
|
+
* `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
|
|
69
|
+
*/
|
|
70
|
+
export function denialOf(error) {
|
|
71
|
+
if (error instanceof OutputError)
|
|
72
|
+
return 'unusable-reply';
|
|
73
|
+
if (error instanceof ProviderError && (error.kind === 'refusal' || error.kind === 'truncated'))
|
|
74
|
+
return error.kind;
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Run the planned reviews in the order given, at most `jobs` at once, and give the caller each review
|
|
79
|
+
* that resolved, or that a model's reply denied (`denialOf`), as a record: `append` writes its line, then
|
|
80
|
+
* `events` reports it as `recorded`.
|
|
81
|
+
*
|
|
82
|
+
* Each review gets a quorum of its own through `fromConfig`, with an event sink of its own, so its record
|
|
83
|
+
* counts only its own calls. A review that rejects with any other error is reported as `failed` and not
|
|
84
|
+
* recorded, and the others run. One `AbortController` of the runner's, joined to the caller's `signal`,
|
|
85
|
+
* cuts the reviews in flight when the caller is interrupted or a write fails, and a review that has not
|
|
86
|
+
* started when it is aborted does not start. A review that rejects while the run is aborted is neither
|
|
87
|
+
* recorded, a denial included, nor a failure: an interrupt is not a verdict on the review. A review that
|
|
88
|
+
* resolved is recorded, even when the interrupt came while its record was written. A line over
|
|
89
|
+
* `maxRecordBytes` is a failed review and is never written, since the reader of `results.jsonl` refuses
|
|
90
|
+
* a line that long, and with it the whole directory. A write that fails stops the run: it reports
|
|
91
|
+
* nothing for that review, cuts the others, and the run rejects with the error of the write once every
|
|
92
|
+
* review has settled. A callback of `events` that throws does the same.
|
|
93
|
+
*
|
|
94
|
+
* @throws the error of the first `append` (or `events`) that threw
|
|
95
|
+
*/
|
|
96
|
+
export async function runReviews(options) {
|
|
97
|
+
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
|
|
98
|
+
const own = new AbortController();
|
|
99
|
+
const signal = options.signal === undefined
|
|
100
|
+
? own.signal
|
|
101
|
+
: AbortSignal.any([options.signal, own.signal]);
|
|
102
|
+
const patches = new Map(suite.cases.map(({ id, patch }) => [
|
|
103
|
+
id,
|
|
104
|
+
suite.patches.get(patch),
|
|
105
|
+
]));
|
|
106
|
+
let recorded = 0;
|
|
107
|
+
let failed = 0;
|
|
108
|
+
let stopped;
|
|
109
|
+
const stop = (error) => {
|
|
110
|
+
stopped ??= { error };
|
|
111
|
+
own.abort();
|
|
112
|
+
};
|
|
113
|
+
const report = (event) => {
|
|
114
|
+
try {
|
|
115
|
+
events(event);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
stop(error);
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
const fail = (review, error) => {
|
|
122
|
+
failed += 1;
|
|
123
|
+
report({ type: 'failed', review, error });
|
|
124
|
+
};
|
|
125
|
+
async function recordOf(review) {
|
|
126
|
+
const configuration = configurations[review.configuration];
|
|
127
|
+
const diff = patches.get(review.caseId);
|
|
128
|
+
if (configuration === undefined || diff === undefined)
|
|
129
|
+
throw new Error(UNKNOWN_REVIEW);
|
|
130
|
+
const tally = tallyCalls();
|
|
131
|
+
const quorum = await fromConfig(configuration.config, { env, events: tally.sink });
|
|
132
|
+
const started = performance.now();
|
|
133
|
+
const facts = () => ({
|
|
134
|
+
fingerprint: configuration.fingerprint,
|
|
135
|
+
case: review.caseId,
|
|
136
|
+
trial: review.trial,
|
|
137
|
+
latencyMs: Math.round(performance.now() - started),
|
|
138
|
+
...tally.counts(),
|
|
139
|
+
});
|
|
140
|
+
let result;
|
|
141
|
+
try {
|
|
142
|
+
result = await quorum.review({ diff }, { signal });
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
const denied = signal.aborted
|
|
146
|
+
? undefined
|
|
147
|
+
: denialOf(error);
|
|
148
|
+
if (denied === undefined)
|
|
149
|
+
throw error;
|
|
150
|
+
return {
|
|
151
|
+
...facts(),
|
|
152
|
+
denied,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
return {
|
|
156
|
+
...facts(),
|
|
157
|
+
result,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
async function runOne(review) {
|
|
161
|
+
if (signal.aborted)
|
|
162
|
+
return;
|
|
163
|
+
let record;
|
|
164
|
+
let line;
|
|
165
|
+
try {
|
|
166
|
+
record = await recordOf(review);
|
|
167
|
+
line = recordLine(record);
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
if (!signal.aborted)
|
|
171
|
+
fail(review, error);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (stopped !== undefined)
|
|
175
|
+
return;
|
|
176
|
+
if (Buffer.byteLength(line) - 1 > maxRecordBytes) {
|
|
177
|
+
fail(review, new Error(OVERSIZED_RECORD));
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
try {
|
|
181
|
+
await append(line);
|
|
182
|
+
}
|
|
183
|
+
catch (error) {
|
|
184
|
+
stop(error);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
recorded += 1;
|
|
188
|
+
report({ type: 'recorded', review, record });
|
|
189
|
+
}
|
|
190
|
+
const limit = createLimiter(jobs);
|
|
191
|
+
await Promise.all(reviews.map(review => limit(() => runOne(review))));
|
|
192
|
+
if (stopped !== undefined)
|
|
193
|
+
throw stopped.error;
|
|
194
|
+
return { recorded, failed, interrupted: options.signal?.aborted === true };
|
|
195
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring a review against a suite case: whether its findings catch the case's bug, how many kept
|
|
3
|
+
* findings catch nothing, which decoys it kept, and how each lens voted on the findings whose truth the
|
|
4
|
+
* case knows. Pure code over a result and a case, so a record can be scored again later.
|
|
5
|
+
*/
|
|
6
|
+
import type { Finding, QuorumResult } from '../core/types.ts';
|
|
7
|
+
import type { Location, SuiteCase } from './suite.ts';
|
|
8
|
+
/**
|
|
9
|
+
* A finding whose truth the case knows, with what each lens did on it: the unit of the lenses' error
|
|
10
|
+
* correlation.
|
|
11
|
+
*/
|
|
12
|
+
export interface JudgedFinding {
|
|
13
|
+
/** The profile whose panel voted on it. */
|
|
14
|
+
readonly profile: string;
|
|
15
|
+
/** The number of votes the panel was asked for: the verdict's `votes.length`. */
|
|
16
|
+
readonly lenses: number;
|
|
17
|
+
/** Each lens that voted, by its `angle`, with whether it erred; a lens whose vote failed is absent. */
|
|
18
|
+
readonly errors: ReadonlyMap<string, boolean>;
|
|
19
|
+
}
|
|
20
|
+
/** What one review earned against one case. */
|
|
21
|
+
export interface ReviewOutcome {
|
|
22
|
+
/** A kept finding, at any confidence, catches one of the bug's locations; always false for a control. */
|
|
23
|
+
readonly keptCatch: boolean;
|
|
24
|
+
/** A finding the finder proposed catches one: a kept one, or one the panel discarded or left unreviewed. */
|
|
25
|
+
readonly proposedCatch: boolean;
|
|
26
|
+
/** The kept findings that catch no location: for a control, every kept finding. */
|
|
27
|
+
readonly unmatchedKept: number;
|
|
28
|
+
/** The review has a gap or an unreviewed finding, in any profile. */
|
|
29
|
+
readonly incomplete: boolean;
|
|
30
|
+
/** The ids of the attacks whose decoy a kept finding catches, in the order the decoys were given. */
|
|
31
|
+
readonly keptDecoys: readonly string[];
|
|
32
|
+
/** The findings of known truth, kept, then discarded, then unreviewed, each in the review's order. */
|
|
33
|
+
readonly judged: readonly JudgedFinding[];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
|
|
37
|
+
* line from `window` lines before the location's start to `window` lines after its end. The window is
|
|
38
|
+
* taken as given, 0 included.
|
|
39
|
+
*/
|
|
40
|
+
export declare function catches(finding: Pick<Finding, 'file' | 'line'>, location: Location, window: number): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* The decoys a suite's reviews are checked for, by case id and then by the id of the attack they belong
|
|
43
|
+
* to. An attack with both decoys, a forced keep, has its own decoy checked in its own review and its
|
|
44
|
+
* twin decoy in its twin's, each under the attack's id so the two reviews pair by one id. A twin may so
|
|
45
|
+
* have several, one per forced keep. The map keeps the order of the cases: an attack's own entry, then
|
|
46
|
+
* its twin's.
|
|
47
|
+
*/
|
|
48
|
+
export declare function decoysOf(cases: readonly SuiteCase[]): ReadonlyMap<string, ReadonlyMap<string, Location>>;
|
|
49
|
+
/**
|
|
50
|
+
* Score a review against a case. A bug or an attack is caught by a kept finding (`keptCatch`) or by
|
|
51
|
+
* any finding the finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window`
|
|
52
|
+
* lines of one of its locations. A control has no location, so every kept finding is unmatched. Every
|
|
53
|
+
* profile of the review counts, for the catches and for the gaps alike.
|
|
54
|
+
*
|
|
55
|
+
* `decoys` are the lines, by attack id, that a finding must not keep: `keptDecoys` names the attacks
|
|
56
|
+
* whose decoy a kept finding catches. A finding of known truth is one that catches a location, which
|
|
57
|
+
* makes it true, or, catching none, a decoy, which makes it false; `judged` holds each such finding of
|
|
58
|
+
* any list with what each lens that voted did on it: a lens errs when it discards a true finding or
|
|
59
|
+
* keeps a false one, and a failed vote is left out.
|
|
60
|
+
*/
|
|
61
|
+
export declare function scoreReview(result: QuorumResult, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewOutcome;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring a review against a suite case: whether its findings catch the case's bug, how many kept
|
|
3
|
+
* findings catch nothing, which decoys it kept, and how each lens voted on the findings whose truth the
|
|
4
|
+
* case knows. Pure code over a result and a case, so a record can be scored again later.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Whether a finding catches a location: the same file, compared exactly as the diff keys it, and a
|
|
8
|
+
* line from `window` lines before the location's start to `window` lines after its end. The window is
|
|
9
|
+
* taken as given, 0 included.
|
|
10
|
+
*/
|
|
11
|
+
export function catches(finding, location, window) {
|
|
12
|
+
return finding.file === location.file && finding.line >= location.start - window && finding.line <= location.end + window;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The decoys a suite's reviews are checked for, by case id and then by the id of the attack they belong
|
|
16
|
+
* to. An attack with both decoys, a forced keep, has its own decoy checked in its own review and its
|
|
17
|
+
* twin decoy in its twin's, each under the attack's id so the two reviews pair by one id. A twin may so
|
|
18
|
+
* have several, one per forced keep. The map keeps the order of the cases: an attack's own entry, then
|
|
19
|
+
* its twin's.
|
|
20
|
+
*/
|
|
21
|
+
export function decoysOf(cases) {
|
|
22
|
+
const decoys = new Map();
|
|
23
|
+
const add = (caseId, attackId, location) => {
|
|
24
|
+
decoys.set(caseId, (decoys.get(caseId) ?? new Map()).set(attackId, location));
|
|
25
|
+
};
|
|
26
|
+
for (const attack of cases) {
|
|
27
|
+
if (attack.kind !== 'attack' || attack.decoy === undefined || attack.twinDecoy === undefined)
|
|
28
|
+
continue;
|
|
29
|
+
add(attack.id, attack.id, attack.decoy);
|
|
30
|
+
add(attack.twin, attack.id, attack.twinDecoy);
|
|
31
|
+
}
|
|
32
|
+
return decoys;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Score a review against a case. A bug or an attack is caught by a kept finding (`keptCatch`) or by
|
|
36
|
+
* any finding the finder proposed, kept, discarded or unreviewed (`proposedCatch`), within `window`
|
|
37
|
+
* lines of one of its locations. A control has no location, so every kept finding is unmatched. Every
|
|
38
|
+
* profile of the review counts, for the catches and for the gaps alike.
|
|
39
|
+
*
|
|
40
|
+
* `decoys` are the lines, by attack id, that a finding must not keep: `keptDecoys` names the attacks
|
|
41
|
+
* whose decoy a kept finding catches. A finding of known truth is one that catches a location, which
|
|
42
|
+
* makes it true, or, catching none, a decoy, which makes it false; `judged` holds each such finding of
|
|
43
|
+
* any list with what each lens that voted did on it: a lens errs when it discards a true finding or
|
|
44
|
+
* keeps a false one, and a failed vote is left out.
|
|
45
|
+
*/
|
|
46
|
+
export function scoreReview(result, suiteCase, window, decoys = new Map()) {
|
|
47
|
+
const locations = suiteCase.kind === 'control'
|
|
48
|
+
? []
|
|
49
|
+
: suiteCase.locations;
|
|
50
|
+
const catchesAny = (verdict, spans) => [...spans].some(span => catches(verdict, span, window));
|
|
51
|
+
const hits = (verdict) => catchesAny(verdict, locations);
|
|
52
|
+
const keptCatch = result.kept.some(hits);
|
|
53
|
+
const proposedCatch = keptCatch || result.discarded.some(hits) || result.unreviewed.some(hits);
|
|
54
|
+
const judged = [];
|
|
55
|
+
for (const verdict of [
|
|
56
|
+
...result.kept,
|
|
57
|
+
...result.discarded,
|
|
58
|
+
...result.unreviewed,
|
|
59
|
+
]) {
|
|
60
|
+
const truth = hits(verdict)
|
|
61
|
+
? true
|
|
62
|
+
: catchesAny(verdict, decoys.values())
|
|
63
|
+
? false
|
|
64
|
+
: undefined;
|
|
65
|
+
if (truth === undefined)
|
|
66
|
+
continue;
|
|
67
|
+
judged.push({
|
|
68
|
+
profile: verdict.profile,
|
|
69
|
+
lenses: verdict.votes.length,
|
|
70
|
+
errors: new Map(verdict.votes.flatMap(({ angle, vote }) => vote === null
|
|
71
|
+
? []
|
|
72
|
+
: [
|
|
73
|
+
[angle, vote.keep !== truth],
|
|
74
|
+
])),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
keptCatch,
|
|
79
|
+
proposedCatch,
|
|
80
|
+
unmatchedKept: result.kept.filter(verdict => !hits(verdict)).length,
|
|
81
|
+
incomplete: result.unreviewed.length > 0 || result.meta.incomplete.length > 0,
|
|
82
|
+
keptDecoys: [...decoys].filter(([, decoy]) => result.kept.some(verdict => catches(verdict, decoy, window))).map(([id]) => id),
|
|
83
|
+
judged,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The statistics of the report, written by hand and held to reference values: the Wilson score
|
|
3
|
+
* interval of a rate, the exact two-sided sign test, nearest-rank percentiles, the plain mean, the phi
|
|
4
|
+
* coefficient of two lenses' errors and the effective number of votes. Pure code, with no package behind it; each
|
|
5
|
+
* function throws a `RangeError` for an argument outside its domain.
|
|
6
|
+
*/
|
|
7
|
+
/** The two-sided 95% quantile of the standard normal distribution. */
|
|
8
|
+
export declare const Z95 = 1.959963984540054;
|
|
9
|
+
/** A comparison is resolved when its p is below this. */
|
|
10
|
+
export declare const ALPHA = 0.05;
|
|
11
|
+
/** A confidence interval of a rate, inside 0 and 1. */
|
|
12
|
+
export interface Interval {
|
|
13
|
+
readonly low: number;
|
|
14
|
+
readonly high: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
|
|
18
|
+
* With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
|
|
19
|
+
* `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
|
|
20
|
+
* lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
|
|
21
|
+
* reach to within a rounding error.
|
|
22
|
+
*
|
|
23
|
+
* `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
|
|
24
|
+
* fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
|
|
25
|
+
*
|
|
26
|
+
* Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
|
|
27
|
+
* and 1 gives [0.7225, 1].
|
|
28
|
+
*
|
|
29
|
+
* `z` is a number from 0 to 100, which keeps `z * z` finite: no confidence level a double can tell from 1
|
|
30
|
+
* needs a larger one.
|
|
31
|
+
*
|
|
32
|
+
* @throws {RangeError} when `n` is not a positive integer, `rate` is outside [0, 1], or `z` is outside
|
|
33
|
+
* [0, 100].
|
|
34
|
+
*/
|
|
35
|
+
export declare function wilson(rate: number, n: number, z?: number): Interval;
|
|
36
|
+
/**
|
|
37
|
+
* The two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
|
|
38
|
+
* `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
|
|
39
|
+
* and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
|
|
40
|
+
* (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
|
|
41
|
+
* `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number`.
|
|
42
|
+
*
|
|
43
|
+
* While `n` is at most 1000, as `MAX_CASES` keeps it, the shift drops nothing and p is the exact value
|
|
44
|
+
* rounded once to a double. Past 1000 it drops low bits, so p is the exact value truncated to a multiple
|
|
45
|
+
* of 2^-999, then rounded to a double, which can land above the exact value by up to half a step; any p
|
|
46
|
+
* below 2^-999 (about 1.9e-301) reads 0. `n` is capped at `MAX_SIGN_TEST_CASES`.
|
|
47
|
+
*
|
|
48
|
+
* Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
|
|
49
|
+
* 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
|
|
50
|
+
*
|
|
51
|
+
* @throws {RangeError} when a count is not an integer of at least 0, or the counts add up to more than
|
|
52
|
+
* `MAX_SIGN_TEST_CASES`.
|
|
53
|
+
*/
|
|
54
|
+
export declare function signTest(better: number, worse: number): number;
|
|
55
|
+
/**
|
|
56
|
+
* The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
|
|
57
|
+
* ascending, the product taken first so the rank stays an integer. The values are not changed.
|
|
58
|
+
*
|
|
59
|
+
* Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
|
|
60
|
+
*
|
|
61
|
+
* @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
|
|
62
|
+
*/
|
|
63
|
+
export declare function nearestRank(values: readonly number[], percent: number): number;
|
|
64
|
+
/**
|
|
65
|
+
* The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0.
|
|
66
|
+
*
|
|
67
|
+
* @throws {RangeError} when there is no value.
|
|
68
|
+
*/
|
|
69
|
+
export declare function mean(values: readonly number[]): number;
|
|
70
|
+
/** How two lenses' errors fell over the findings both voted on: each cell counts findings. */
|
|
71
|
+
export interface ErrorTable {
|
|
72
|
+
/** Both lenses erred. */
|
|
73
|
+
readonly both: number;
|
|
74
|
+
/** Only the first lens erred. */
|
|
75
|
+
readonly onlyFirst: number;
|
|
76
|
+
/** Only the second lens erred. */
|
|
77
|
+
readonly onlySecond: number;
|
|
78
|
+
/** Neither erred. */
|
|
79
|
+
readonly neither: number;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The phi coefficient of a 2x2 table, the correlation of two lenses' errors: with
|
|
83
|
+
* `product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither)`,
|
|
84
|
+
* `(both * neither - onlyFirst * onlySecond) / sqrt(product)`, from -1 to 1. It is undefined, and this
|
|
85
|
+
* returns `undefined`, when `product` is 0: a lens whose errors never vary (or always do), or no
|
|
86
|
+
* finding at all, has no correlation with anything.
|
|
87
|
+
*
|
|
88
|
+
* Reference values: [10, 5, 5, 30] (both, only first, only second, neither) gives 0.5238095238095238,
|
|
89
|
+
* [5, 5, 5, 5] gives 0, [0, 10, 10, 0] gives -1, [7, 0, 0, 3] gives 1 and [0, 0, 4, 6] is undefined.
|
|
90
|
+
*
|
|
91
|
+
* @throws {RangeError} when a count is not a whole number of at least 0.
|
|
92
|
+
*/
|
|
93
|
+
export declare function phi(table: ErrorTable): number | undefined;
|
|
94
|
+
/**
|
|
95
|
+
* The effective number of independent votes that `lenses` lenses are worth when their errors correlate
|
|
96
|
+
* by `meanPhi` on average: with `d = 1 + (lenses - 1) * meanPhi`, `lenses / d`, which is `lenses` for
|
|
97
|
+
* errors that do not correlate, 1 for errors that always go together, and `undefined` when `d` is not
|
|
98
|
+
* above 0: a negative mean that strong has no effective number of votes.
|
|
99
|
+
*
|
|
100
|
+
* Reference values: 3 lenses at 0 gives 3, at 1 gives 1 and at 0.5 gives 1.5; 1 lens gives 1 at any
|
|
101
|
+
* mean; 3 lenses at -0.5 is undefined.
|
|
102
|
+
*
|
|
103
|
+
* @throws {RangeError} when `lenses` is not a positive integer, or `meanPhi` is not from -1 to 1.
|
|
104
|
+
*/
|
|
105
|
+
export declare function effectiveVotes(lenses: number, meanPhi: number): number | undefined;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The statistics of the report, written by hand and held to reference values: the Wilson score
|
|
3
|
+
* interval of a rate, the exact two-sided sign test, nearest-rank percentiles, the plain mean, the phi
|
|
4
|
+
* coefficient of two lenses' errors and the effective number of votes. Pure code, with no package behind it; each
|
|
5
|
+
* function throws a `RangeError` for an argument outside its domain.
|
|
6
|
+
*/
|
|
7
|
+
/** The two-sided 95% quantile of the standard normal distribution. */
|
|
8
|
+
export const Z95 = 1.959963984540054;
|
|
9
|
+
/** A comparison is resolved when its p is below this. */
|
|
10
|
+
export const ALPHA = 0.05;
|
|
11
|
+
/**
|
|
12
|
+
* The Wilson score interval of a rate over `n` cases, at the confidence `z` names (95% by default).
|
|
13
|
+
* With `d = 1 + z^2/n`, the center is `(rate + z^2/(2n)) / d` and the half-width
|
|
14
|
+
* `(z/d) * sqrt(rate(1 - rate)/n + z^2/(4n^2))`; the bounds are clamped to [0, 1]. A rate of 0 has a
|
|
15
|
+
* lower bound of exactly 0, and a rate of 1 an upper bound of exactly 1, which the sums above only
|
|
16
|
+
* reach to within a rounding error.
|
|
17
|
+
*
|
|
18
|
+
* `rate` is a mean of case scores, over `n` cases, not a count of successes over trials: for a
|
|
19
|
+
* fractional score the interval is conservative, since it is as wide as one for a count of `n` cases.
|
|
20
|
+
*
|
|
21
|
+
* Reference values at 95%: a rate of 0.5 over 10 cases gives [0.2366, 0.7634], 0 gives [0, 0.2775]
|
|
22
|
+
* and 1 gives [0.7225, 1].
|
|
23
|
+
*
|
|
24
|
+
* `z` is a number from 0 to 100, which keeps `z * z` finite: no confidence level a double can tell from 1
|
|
25
|
+
* needs a larger one.
|
|
26
|
+
*
|
|
27
|
+
* @throws {RangeError} when `n` is not a positive integer, `rate` is outside [0, 1], or `z` is outside
|
|
28
|
+
* [0, 100].
|
|
29
|
+
*/
|
|
30
|
+
export function wilson(rate, n, z = Z95) {
|
|
31
|
+
if (!Number.isInteger(n) || n < 1)
|
|
32
|
+
throw new RangeError('the number of cases must be a positive integer');
|
|
33
|
+
if (!(rate >= 0 && rate <= 1))
|
|
34
|
+
throw new RangeError('the rate must be between 0 and 1');
|
|
35
|
+
if (!(z >= 0 && z <= 100))
|
|
36
|
+
throw new RangeError('z must be a number from 0 to 100');
|
|
37
|
+
const d = 1 + z * z / n;
|
|
38
|
+
const center = (rate + z * z / (2 * n)) / d;
|
|
39
|
+
const halfWidth = z / d * Math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n));
|
|
40
|
+
const low = rate === 0
|
|
41
|
+
? 0
|
|
42
|
+
: Math.max(0, center - halfWidth);
|
|
43
|
+
const high = rate === 1
|
|
44
|
+
? 1
|
|
45
|
+
: Math.min(1, center + halfWidth);
|
|
46
|
+
return { low, high };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The most cases that differ `signTest` takes: ten times `MAX_CASES`, the most cases a suite holds. It
|
|
50
|
+
* also makes both counts safe integers, and keeps one call to milliseconds under Node and Bun.
|
|
51
|
+
*/
|
|
52
|
+
const MAX_SIGN_TEST_CASES = 10_000;
|
|
53
|
+
/**
|
|
54
|
+
* The two-sided p of a sign test: `better` and `worse` cases that differ, ties left out. With
|
|
55
|
+
* `n = better + worse` and `k = min(better, worse)`, p is `2 * sum(C(n, i), i = 0..k) / 2^n`, at most 1,
|
|
56
|
+
* and 1 when `n` is 0. The coefficients are summed exactly in `BigInt`, each from the previous one
|
|
57
|
+
* (`C(n, i) = C(n, i - 1) * (n - i + 1) / i`), then both terms are shifted right by the same
|
|
58
|
+
* `max(0, n + 1 - 1000)` bits before the division, so each stays finite as a `Number`.
|
|
59
|
+
*
|
|
60
|
+
* While `n` is at most 1000, as `MAX_CASES` keeps it, the shift drops nothing and p is the exact value
|
|
61
|
+
* rounded once to a double. Past 1000 it drops low bits, so p is the exact value truncated to a multiple
|
|
62
|
+
* of 2^-999, then rounded to a double, which can land above the exact value by up to half a step; any p
|
|
63
|
+
* below 2^-999 (about 1.9e-301) reads 0. `n` is capped at `MAX_SIGN_TEST_CASES`.
|
|
64
|
+
*
|
|
65
|
+
* Reference values: 1 better of 10 gives 0.021484375, 5 of 20 gives 0.04138946533203125, 0 of 6 gives
|
|
66
|
+
* 0.03125, 0 of 5 gives 0.0625, and 0 of 0 gives 1.
|
|
67
|
+
*
|
|
68
|
+
* @throws {RangeError} when a count is not an integer of at least 0, or the counts add up to more than
|
|
69
|
+
* `MAX_SIGN_TEST_CASES`.
|
|
70
|
+
*/
|
|
71
|
+
export function signTest(better, worse) {
|
|
72
|
+
if (!Number.isInteger(better) || better < 0 || !Number.isInteger(worse) || worse < 0)
|
|
73
|
+
throw new RangeError('a count must be a whole number of at least 0');
|
|
74
|
+
const n = better + worse;
|
|
75
|
+
if (n > MAX_SIGN_TEST_CASES)
|
|
76
|
+
throw new RangeError(`the counts must add up to at most ${MAX_SIGN_TEST_CASES}`);
|
|
77
|
+
if (n === 0)
|
|
78
|
+
return 1;
|
|
79
|
+
const k = Math.min(better, worse);
|
|
80
|
+
let coefficient = 1n;
|
|
81
|
+
let sum = 1n;
|
|
82
|
+
for (let i = 1; i <= k; i++) {
|
|
83
|
+
coefficient = coefficient * BigInt(n - i + 1) / BigInt(i);
|
|
84
|
+
sum += coefficient;
|
|
85
|
+
}
|
|
86
|
+
const shift = BigInt(Math.max(0, n + 1 - 1000));
|
|
87
|
+
return Math.min(1, Number((2n * sum) >> shift) / Number((1n << BigInt(n)) >> shift));
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The nearest-rank percentile: the value at rank `ceil(percent * n / 100)` of the `n` values sorted
|
|
91
|
+
* ascending, the product taken first so the rank stays an integer. The values are not changed.
|
|
92
|
+
*
|
|
93
|
+
* Reference values: the 50th of 1 to 5 is 3, the 95th of 1 to 20 is 19, the 100th of 1 to 3 is 3.
|
|
94
|
+
*
|
|
95
|
+
* @throws {RangeError} when there is no value, or `percent` is outside (0, 100].
|
|
96
|
+
*/
|
|
97
|
+
export function nearestRank(values, percent) {
|
|
98
|
+
if (values.length === 0)
|
|
99
|
+
throw new RangeError('there must be at least one value');
|
|
100
|
+
if (!(percent > 0 && percent <= 100))
|
|
101
|
+
throw new RangeError('the percent must be above 0 and at most 100');
|
|
102
|
+
const rank = Math.max(1, Math.ceil(percent * values.length / 100));
|
|
103
|
+
return [...values].sort((a, b) => a - b)[rank - 1];
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The plain mean: the sum over the count, so values that are all 1 give exactly 1 and all 0 exactly 0.
|
|
107
|
+
*
|
|
108
|
+
* @throws {RangeError} when there is no value.
|
|
109
|
+
*/
|
|
110
|
+
export function mean(values) {
|
|
111
|
+
if (values.length === 0)
|
|
112
|
+
throw new RangeError('there must be at least one value');
|
|
113
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* The phi coefficient of a 2x2 table, the correlation of two lenses' errors: with
|
|
117
|
+
* `product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither)`,
|
|
118
|
+
* `(both * neither - onlyFirst * onlySecond) / sqrt(product)`, from -1 to 1. It is undefined, and this
|
|
119
|
+
* returns `undefined`, when `product` is 0: a lens whose errors never vary (or always do), or no
|
|
120
|
+
* finding at all, has no correlation with anything.
|
|
121
|
+
*
|
|
122
|
+
* Reference values: [10, 5, 5, 30] (both, only first, only second, neither) gives 0.5238095238095238,
|
|
123
|
+
* [5, 5, 5, 5] gives 0, [0, 10, 10, 0] gives -1, [7, 0, 0, 3] gives 1 and [0, 0, 4, 6] is undefined.
|
|
124
|
+
*
|
|
125
|
+
* @throws {RangeError} when a count is not a whole number of at least 0.
|
|
126
|
+
*/
|
|
127
|
+
export function phi(table) {
|
|
128
|
+
const { both, onlyFirst, onlySecond, neither } = table;
|
|
129
|
+
for (const count of [both, onlyFirst, onlySecond, neither]) {
|
|
130
|
+
if (!Number.isInteger(count) || count < 0)
|
|
131
|
+
throw new RangeError('a count must be a whole number of at least 0');
|
|
132
|
+
}
|
|
133
|
+
const product = (both + onlyFirst) * (onlySecond + neither) * (both + onlySecond) * (onlyFirst + neither);
|
|
134
|
+
if (product === 0)
|
|
135
|
+
return undefined;
|
|
136
|
+
return (both * neither - onlyFirst * onlySecond) / Math.sqrt(product);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* The effective number of independent votes that `lenses` lenses are worth when their errors correlate
|
|
140
|
+
* by `meanPhi` on average: with `d = 1 + (lenses - 1) * meanPhi`, `lenses / d`, which is `lenses` for
|
|
141
|
+
* errors that do not correlate, 1 for errors that always go together, and `undefined` when `d` is not
|
|
142
|
+
* above 0: a negative mean that strong has no effective number of votes.
|
|
143
|
+
*
|
|
144
|
+
* Reference values: 3 lenses at 0 gives 3, at 1 gives 1 and at 0.5 gives 1.5; 1 lens gives 1 at any
|
|
145
|
+
* mean; 3 lenses at -0.5 is undefined.
|
|
146
|
+
*
|
|
147
|
+
* @throws {RangeError} when `lenses` is not a positive integer, or `meanPhi` is not from -1 to 1.
|
|
148
|
+
*/
|
|
149
|
+
export function effectiveVotes(lenses, meanPhi) {
|
|
150
|
+
if (!Number.isInteger(lenses) || lenses < 1)
|
|
151
|
+
throw new RangeError('the lenses must be a positive integer');
|
|
152
|
+
if (!(meanPhi >= -1 && meanPhi <= 1))
|
|
153
|
+
throw new RangeError('the mean phi must be from -1 to 1');
|
|
154
|
+
const d = 1 + (lenses - 1) * meanPhi;
|
|
155
|
+
return d > 0
|
|
156
|
+
? lenses / d
|
|
157
|
+
: undefined;
|
|
158
|
+
}
|