accuracy-claim-kit 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/assess.js ADDED
@@ -0,0 +1,962 @@
1
+ import { countDecimals, matchesValue } from './census.js';
2
+ import { dayNumber } from './dates.js';
3
+ import { parseDisplayFigure, textFiguresDiffering } from './display.js';
4
+ import { beyondNoise, snapAbsolute, VALUE_NOISE } from './exact.js';
5
+ import { confidenceLabel, decimalCut, maxDisplayDecimals, pctBound, pctCeil, pctFloor, pctNearest, pctNumericBound, pctRate } from './format.js';
6
+ import { clip, isRecord, MAX_COUNT, readCount, readFiniteInRange, readLength, assertKnownKeys, assertPlainObject, readOpenUnit, requireRecord, } from './validate.js';
7
+ import { assertPlainText, foldText, MAX_LABEL_LENGTH, MAX_TEXT_LENGTH, mentionsName, readName, shownText } from './text.js';
8
+ import { DEFAULT_CONFIDENCE, wilsonWithZ, zForConfidence, zForTail } from './wilson.js';
9
+ /** Lowest confidence level `assessAccuracyClaim` accepts. `wilsonInterval` takes any level strictly between 0 and 1. */
10
+ const MIN_ASSESS_CONFIDENCE = 0.5;
11
+ /** Smallest sample that gets a verdict unless options.minN says otherwise. */
12
+ export const DEFAULT_MIN_N = 30;
13
+ /**
14
+ * Absolute tolerance, as a proportion, when a claim is compared with an
15
+ * interval limit. A claim within this distance of a limit counts as being at
16
+ * the limit, on the claimant's side. It exists only to absorb binary
17
+ * floating-point noise (a claim of exactly 0.98 against a limit that
18
+ * computes to 0.98000000000000001). It is far below the width of any
19
+ * interval met in practice; it can only matter for samples above about 4e12
20
+ * items with a rate within about 1e-12 of 0 or 1, where the interval itself
21
+ * is narrower than the tolerance. A claim of exactly 0 or 1 is absolute and
22
+ * never uses it.
23
+ */
24
+ export const COMPARISON_TOLERANCE = 1e-12;
25
+ const SOURCES = ['human', 'automated', 'by-construction', 'model', 'mixed', 'unknown'];
26
+ const SAMPLINGS = ['random', 'stratified', 'ranked', 'convenience', 'unknown', 'complete', 'held-out-split', 'cross-validation'];
27
+ const METRICS = [
28
+ 'accuracy',
29
+ 'error-rate',
30
+ 'precision',
31
+ 'recall',
32
+ 'false-positive-rate',
33
+ 'false-negative-rate',
34
+ 'specificity',
35
+ 'npv',
36
+ 'sensitivity',
37
+ 'ppa',
38
+ 'ppv',
39
+ 'npa',
40
+ 'opa',
41
+ ];
42
+ /** Metrics the kit knows it cannot judge: they get a refusal, not a throw. */
43
+ const UNSUPPORTED_METRICS = [
44
+ 'f1', 'auc', 'roc-auc', 'wer', 'cer', 'bleu', 'rouge', 'map', 'composite', 'other',
45
+ ];
46
+ const HIGHER_IS_BETTER = ['accuracy', 'precision', 'recall', 'specificity', 'npv', 'sensitivity', 'ppa', 'ppv', 'npa', 'opa'];
47
+ const METRIC_PHRASE = {
48
+ accuracy: 'accurate',
49
+ 'error-rate': 'error rate',
50
+ precision: 'precision',
51
+ recall: 'recall',
52
+ 'false-positive-rate': 'false-positive rate',
53
+ 'false-negative-rate': 'false-negative rate',
54
+ specificity: 'specificity',
55
+ npv: 'negative predictive value',
56
+ sensitivity: 'sensitivity',
57
+ ppa: 'positive percent agreement',
58
+ ppv: 'positive predictive value',
59
+ npa: 'negative percent agreement',
60
+ opa: 'overall percent agreement',
61
+ };
62
+ /** What a bare value (no display) is read at; nothing was printed, so the text must not say so. */
63
+ const VALUE_PRECISION = 'the precision of the value (a whole percent at the coarsest)';
64
+ const SUPPORTED_LIST = 'accuracy, error-rate, precision, recall, false-positive-rate, false-negative-rate, specificity, npv and the diagnostic names sensitivity, ppa, ppv, npa and opa';
65
+ // The keys each caller-supplied object may carry. Anything else is a typo that
66
+ // would silently switch a check off, so it throws.
67
+ const OPTION_KEYS = ['confidence', 'minN', 'minAgreement', 'maxAgeDays', 'now', 'strictPrecision', 'conservative', 'requireScope'];
68
+ const INPUT_KEYS = ['claim', 'sample', 'provenance', 'sampling', 'strata', 'retractions', 'benchmark', 'population', 'evaluatedAt'];
69
+ const CLAIM_KEYS = ['metric', 'value', 'display', 'text'];
70
+ const SAMPLE_KEYS = ['n', 'correct', 'errors'];
71
+ const STRATUM_KEYS = ['name', 'n', 'correct', 'errors'];
72
+ const PROVENANCE_KEYS = ['source', 'raters', 'agreement'];
73
+ const AGREEMENT_KEYS = ['statistic', 'status', 'value', 'n', 'agreements', 'categories', 'reason'];
74
+ const RETRACTION_KEYS = ['count', 'includedInSample'];
75
+ const BENCHMARK_KEYS = ['name'];
76
+ const POPULATION_KEYS = ['claimed', 'evaluated'];
77
+ const STATISTICS = ['percent-agreement', 'cohen-kappa', 'gwet-ac1'];
78
+ const UNDEFINED_REASONS = ['no-pairs', 'expected-agreement-is-1', 'single-category'];
79
+ const STATISTIC_LABEL = {
80
+ 'percent-agreement': 'percent agreement',
81
+ 'cohen-kappa': 'Cohen kappa',
82
+ 'gwet-ac1': 'Gwet AC1',
83
+ };
84
+ function readEnum(value, allowed, name) {
85
+ if (typeof value !== 'string')
86
+ throw new TypeError(`${name} must be a string`);
87
+ const found = allowed.find((candidate) => candidate === value);
88
+ if (found === undefined) {
89
+ throw new RangeError(`${name} must be one of ${allowed.join(', ')}; got "${clip(value)}"`);
90
+ }
91
+ return found;
92
+ }
93
+ /**
94
+ * The metric: judged (exact, lower-case name), known-unsupported (refused
95
+ * later; case does not matter, since such a name can only be refused), or
96
+ * unknown (thrown). A judged name in the wrong case is a typo that throws, with
97
+ * the name the caller probably meant.
98
+ */
99
+ function readMetric(raw) {
100
+ if (typeof raw !== 'string')
101
+ throw new TypeError('claim.metric must be a string');
102
+ const supported = METRICS.find((m) => m === raw);
103
+ if (supported !== undefined)
104
+ return { metric: supported, judged: true };
105
+ const lower = raw.toLowerCase();
106
+ const unsupported = UNSUPPORTED_METRICS.find((m) => m === lower);
107
+ if (unsupported !== undefined)
108
+ return { metric: unsupported, judged: false };
109
+ const meant = METRICS.find((m) => m === lower);
110
+ const hint = meant === undefined ? '' : `; did you mean "${meant}"?`;
111
+ throw new RangeError(`claim.metric must be one of ${METRICS.join(', ')}, or one of ${UNSUPPORTED_METRICS.join(', ')} (which the kit knows and refuses, in any case); got "${clip(raw)}"${hint}`);
112
+ }
113
+ /** Reads {n, correct?, errors?} for the sample; null when no counts were disclosed at all. */
114
+ function readSampleCounts(raw) {
115
+ if (raw === undefined)
116
+ return null;
117
+ const rec = requireRecord(raw, 'sample');
118
+ assertKnownKeys(rec, SAMPLE_KEYS, 'sample');
119
+ const nRaw = rec.n;
120
+ const correctRaw = rec.correct;
121
+ const errorsRaw = rec.errors;
122
+ if (nRaw === undefined && correctRaw === undefined && errorsRaw === undefined)
123
+ return null;
124
+ return readCounts({ n: nRaw, correct: correctRaw, errors: errorsRaw }, 'sample');
125
+ }
126
+ /** Reads {n, correct?, errors?}; each property once. */
127
+ function readCounts(raw, name) {
128
+ const rec = requireRecord(raw, name);
129
+ const nRaw = rec.n;
130
+ const correctRaw = rec.correct;
131
+ const errorsRaw = rec.errors;
132
+ const n = readCount(nRaw, `${name}.n`, 1);
133
+ if (correctRaw === undefined && errorsRaw === undefined) {
134
+ throw new TypeError(`${name} needs correct or errors`);
135
+ }
136
+ const correct = correctRaw === undefined ? undefined : readCount(correctRaw, `${name}.correct`, 0);
137
+ const errors = errorsRaw === undefined ? undefined : readCount(errorsRaw, `${name}.errors`, 0);
138
+ if (correct !== undefined && correct > n) {
139
+ throw new RangeError(`${name}.correct (${String(correct)}) is larger than n (${String(n)})`);
140
+ }
141
+ if (errors !== undefined && errors > n) {
142
+ throw new RangeError(`${name}.errors (${String(errors)}) is larger than n (${String(n)})`);
143
+ }
144
+ if (correct !== undefined && errors !== undefined && correct + errors !== n) {
145
+ throw new RangeError(`${name}.correct + ${name}.errors must equal n`);
146
+ }
147
+ return { n, correct: correct !== undefined ? correct : n - errors };
148
+ }
149
+ function readAgreement(raw) {
150
+ const rec = requireRecord(raw, 'provenance.agreement');
151
+ assertKnownKeys(rec, AGREEMENT_KEYS, 'provenance.agreement');
152
+ const statisticRaw = rec.statistic;
153
+ const statusRaw = rec.status;
154
+ const valueRaw = rec.value;
155
+ const nRaw = rec.n;
156
+ const agreementsRaw = rec.agreements;
157
+ const categoriesRaw = rec.categories;
158
+ const reasonRaw = rec.reason;
159
+ const statistic = readEnum(statisticRaw, STATISTICS, 'provenance.agreement.statistic');
160
+ if (statusRaw !== undefined && statusRaw !== 'defined' && statusRaw !== 'undefined') {
161
+ throw new RangeError('provenance.agreement.status must be "defined" or "undefined"');
162
+ }
163
+ const n = nRaw === undefined ? null : readCount(nRaw, 'provenance.agreement.n', 0);
164
+ // The keys a kit result carries are allowed so it can be passed straight in; they
165
+ // are not used, but nonsense in them means the object is not what it claims to be.
166
+ const agreements = agreementsRaw === undefined ? null : readCount(agreementsRaw, 'provenance.agreement.agreements', 0);
167
+ if (agreements !== null && n !== null && agreements > n) {
168
+ throw new RangeError(`provenance.agreement.agreements (${String(agreements)}) is larger than provenance.agreement.n (${String(n)})`);
169
+ }
170
+ if (categoriesRaw !== undefined)
171
+ readCount(categoriesRaw, 'provenance.agreement.categories', 0);
172
+ if (reasonRaw !== undefined)
173
+ readEnum(reasonRaw, UNDEFINED_REASONS, 'provenance.agreement.reason');
174
+ if (statusRaw === 'undefined') {
175
+ // A statistic that has no value must say so with null; NaN, 0 or text in
176
+ // its place is malformed input, not "cannot tell".
177
+ if (valueRaw !== null) {
178
+ const message = 'provenance.agreement.value must be null when status is "undefined"';
179
+ throw typeof valueRaw === 'number' ? new RangeError(message) : new TypeError(message);
180
+ }
181
+ return { statistic, value: null, n };
182
+ }
183
+ return { statistic, value: readFiniteInRange(valueRaw, 'provenance.agreement.value', -1, 1), n };
184
+ }
185
+ function readStrata(raw) {
186
+ if (!Array.isArray(raw))
187
+ throw new TypeError('strata must be an array');
188
+ const length = readLength(raw.length, 'strata');
189
+ if (length === 0)
190
+ throw new RangeError('strata must not be empty; leave it out if there are none');
191
+ const out = [];
192
+ const names = new Set();
193
+ for (let i = 0; i < length; i++) {
194
+ const entry = raw[i];
195
+ const rec = requireRecord(entry, `strata[${i}]`);
196
+ assertKnownKeys(rec, STRATUM_KEYS, `strata[${i}]`);
197
+ const nameRaw = rec.name;
198
+ if (typeof nameRaw !== 'string')
199
+ throw new TypeError(`strata[${i}].name must be a string`);
200
+ if (nameRaw === '')
201
+ throw new RangeError(`strata[${i}].name must not be empty`);
202
+ if (nameRaw.length > MAX_LABEL_LENGTH) {
203
+ throw new RangeError(`strata[${i}].name must be at most ${String(MAX_LABEL_LENGTH)} characters`);
204
+ }
205
+ const name = shownText(nameRaw, `strata[${i}].name`);
206
+ if (name === '')
207
+ throw new RangeError(`strata[${i}].name must not be blank`);
208
+ if (names.has(name))
209
+ throw new RangeError(`strata[${i}].name "${clip(name)}" is a duplicate`);
210
+ names.add(name);
211
+ const counts = readCounts(rec, `strata[${i}]`);
212
+ out.push({ name, ...counts });
213
+ }
214
+ return out;
215
+ }
216
+ const REFUSAL_PHRASE = {
217
+ 'metric-not-supported': 'this kit cannot judge that kind of metric',
218
+ 'n-undisclosed': 'the number of items checked was not disclosed',
219
+ 'no-human-labels': 'the labels were not checked by people',
220
+ 'unknown-label-source': 'nobody has said where the labels came from',
221
+ 'mixed-label-source': 'only some labels came from people and the counts are not split',
222
+ 'ranked-sample': 'the sample was picked by rank, not at random',
223
+ 'convenience-sample': 'the sample was whatever was easy to collect',
224
+ 'unknown-sampling': 'nobody has said how the sample was picked',
225
+ 'sample-too-small': 'the sample is too small',
226
+ 'merged-strata-differ': 'groups with clearly different rates were merged into one number',
227
+ 'retractions-not-included': 'known retractions are not in the counts',
228
+ 'agreement-missing': 'no agreement statistic was supplied for the required agreement check',
229
+ 'agreement-undefined': 'the agreement statistic could not be computed',
230
+ 'agreement-below-minimum': 'the raters agreed less than the required minimum',
231
+ 'agreement-sample-too-small': 'the agreement statistic was computed on too few items',
232
+ 'stratified-without-strata': 'the sample is stratified but the strata were not supplied, so the pooled counts cannot be read as the population rate',
233
+ 'benchmark-unnamed': 'a complete-set or held-out-split claim needs the name of the fixed set',
234
+ 'claim-not-scoped-to-benchmark': 'the claim wording does not name the benchmark, so a benchmark score is standing in for a general claim',
235
+ 'evaluation-date-missing': 'a maximum age was set but the evaluation date was not supplied',
236
+ 'evaluation-stale': 'the evaluation is older than the allowed age',
237
+ 'scope-mismatch': 'the claim covers a different population than the one evaluated',
238
+ 'scope-undeclared': 'the claimed and evaluated populations were not declared',
239
+ };
240
+ const LABEL_WORD = {
241
+ human: 'human labels',
242
+ automated: 'automated labels',
243
+ 'by-construction': 'labels fixed by construction',
244
+ };
245
+ /** What the interval in a statement does and does not cover, said in the statement itself. */
246
+ const SAMPLING_ERROR_ONLY = 'the interval covers sampling error only';
247
+ const SAMPLING_ERROR_ONLY_CV = 'the interval covers sampling error only and is optimistic because cross-validation folds share data';
248
+ const SAMPLE_WORD = {
249
+ stratified: 'stratified sample, unweighted pooled',
250
+ 'cross-validation': 'cross-validation estimate',
251
+ };
252
+ /** "1 decimal place", "2 decimal places"; whole percents have none. */
253
+ function placesWord(decimals) {
254
+ return decimals === 1 ? '1 decimal place' : `${String(decimals)} decimal places`;
255
+ }
256
+ /** What a precision limit allows: "whole percents only", "at most 1 decimal place". */
257
+ function supportsWord(decimals) {
258
+ return decimals === 0 ? 'whole percents only' : `at most ${placesWord(decimals)}`;
259
+ }
260
+ /**
261
+ * The observed rate as a figure to print that the same assessment would pass as
262
+ * `matches-observed-rate`, or null when none exists. With `maxDecimals` null
263
+ * (the default, where excess precision is only noted) the figure is printed to
264
+ * three decimals, or to as many more as it takes to keep it off 0% and 100%
265
+ * (those are absolute claims a sample cannot show); with a limit (strictPrecision)
266
+ * it must fit that many decimals, and an exact tie may be reached from either
267
+ * neighbor. A candidate is judged by the display parser itself, so it cannot
268
+ * drift from what the assessment accepts.
269
+ */
270
+ function observedAdvice(point, successes, n, maxDecimals, higherIsBetter) {
271
+ if (point === 0 || point === 1)
272
+ return null;
273
+ const first = maxDecimals === null ? 3 : maxDecimals;
274
+ const last = maxDecimals === null ? 12 : maxDecimals;
275
+ for (let d = first; d <= last; d++) {
276
+ for (const printed of [pctNearest(point, d), pctFloor(point, d), pctCeil(point, d)]) {
277
+ const figure = parseDisplayFigure(printed, higherIsBetter);
278
+ if (figure.info.figure !== 0 && figure.info.figure !== 1 && figure.matchesObserved(successes, n))
279
+ return printed;
280
+ }
281
+ }
282
+ return null;
283
+ }
284
+ /** The statement's advice on how to print the figure. */
285
+ function printAdvice(decimals) {
286
+ return decimals === 0 ? '; print it as a whole percent' : `; print it to at most ${placesWord(decimals)}`;
287
+ }
288
+ /** Trim fixed decimal padding while preserving the exponent of a tiny point rate. */
289
+ function mismatchPercent(proportion) {
290
+ return pctRate(proportion, 3).replace(/(\.\d*?[1-9])0+%$|\.0+%$/, '$1%');
291
+ }
292
+ /**
293
+ * The informational note for a figure in `claim.text` that is not the figure
294
+ * judged (`text-figure-differs`), or null. The text is only scanned: a verdict
295
+ * is never changed by it.
296
+ */
297
+ function textFigureNote(text, claimed, assessed, higherIsBetter) {
298
+ if (text === undefined)
299
+ return null;
300
+ const differing = textFiguresDiffering(text, claimed, assessed, higherIsBetter);
301
+ if (differing.length === 0)
302
+ return null;
303
+ const shown = differing.slice(0, 3).map((figure) => `"${clip(figure, 40)}"`).join(', ');
304
+ const more = differing.length > 3 ? ` and ${String(differing.length - 3)} more` : '';
305
+ return `claim.text says ${shown}${more}, which is not the figure judged (${pctRate(assessed, 3)}). The verdict judges value and display, never the wording, so make the text say what was judged.`;
306
+ }
307
+ /**
308
+ * Does this accuracy or error-rate claim have the evidence to carry it?
309
+ *
310
+ * Pure statistics on the counts you pass in. It never judges whether the
311
+ * labels are right and cannot see how the sample was really drawn; it takes
312
+ * your `provenance` and `sampling` declarations at their word and refuses
313
+ * when they do not support a figure.
314
+ *
315
+ * Verdicts:
316
+ * - refused: a precondition failed. This means "cannot tell", not "false".
317
+ * - contradicted: the claim is more favorable than the interval's best end.
318
+ * - overstated: inside the interval but more favorable than the observed
319
+ * rate (or, with options.conservative, than the conservative end), or
320
+ * printed with finer precision than the interval allows.
321
+ * - supported: no more favorable than the conservative end, or an honest
322
+ * report of the observed rate to its printed precision.
323
+ *
324
+ * Malformed input throws (TypeError for a wrong type, RangeError for a wrong
325
+ * value); it never returns a verdict. Every input property is read once.
326
+ */
327
+ export function assessAccuracyClaim(input, options) {
328
+ // ---- read and validate everything before judging anything -------------
329
+ if (options !== undefined && !isRecord(options)) {
330
+ throw new TypeError('options must be an object');
331
+ }
332
+ if (options !== undefined) {
333
+ assertPlainObject(options, 'options');
334
+ assertKnownKeys(options, OPTION_KEYS, 'options');
335
+ }
336
+ const confidenceRaw = options === undefined ? undefined : options.confidence;
337
+ const minNRaw = options === undefined ? undefined : options.minN;
338
+ const minAgreementRaw = options === undefined ? undefined : options.minAgreement;
339
+ const confidence = confidenceRaw === undefined ? DEFAULT_CONFIDENCE : readOpenUnit(confidenceRaw, 'options.confidence');
340
+ // A level below one half is a typo (0.05 for 0.95): its "interval" is narrower
341
+ // than the data's own noise and an "at least" statement built on it means nothing.
342
+ if (confidence < MIN_ASSESS_CONFIDENCE) {
343
+ throw new RangeError(`options.confidence must be at least ${String(MIN_ASSESS_CONFIDENCE)} for a verdict (0.95 is the default); got ${String(confidence)}`);
344
+ }
345
+ const z = zForConfidence(confidence);
346
+ const minN = minNRaw === undefined ? DEFAULT_MIN_N : readCount(minNRaw, 'options.minN', 1);
347
+ const minAgreement = minAgreementRaw === undefined ? undefined : readFiniteInRange(minAgreementRaw, 'options.minAgreement', -1, 1);
348
+ const maxAgeRaw = options === undefined ? undefined : options.maxAgeDays;
349
+ const nowRaw = options === undefined ? undefined : options.now;
350
+ const requireScopeRaw = options === undefined ? undefined : options.requireScope;
351
+ const conservativeRaw = options === undefined ? undefined : options.conservative;
352
+ const strictPrecisionRaw = options === undefined ? undefined : options.strictPrecision;
353
+ const maxAgeDays = maxAgeRaw === undefined ? undefined : readCount(maxAgeRaw, 'options.maxAgeDays', 0);
354
+ if (maxAgeDays !== undefined && nowRaw === undefined) {
355
+ throw new TypeError('options.now is required with options.maxAgeDays: the kit never reads a clock');
356
+ }
357
+ const nowDay = nowRaw === undefined ? undefined : dayNumber(nowRaw, 'options.now');
358
+ if (requireScopeRaw !== undefined && typeof requireScopeRaw !== 'boolean') {
359
+ throw new TypeError('options.requireScope must be a boolean');
360
+ }
361
+ if (conservativeRaw !== undefined && typeof conservativeRaw !== 'boolean') {
362
+ throw new TypeError('options.conservative must be a boolean');
363
+ }
364
+ if (strictPrecisionRaw !== undefined && typeof strictPrecisionRaw !== 'boolean') {
365
+ throw new TypeError('options.strictPrecision must be a boolean');
366
+ }
367
+ const requireScope = requireScopeRaw === true;
368
+ const conservative = conservativeRaw === true;
369
+ // Excess printed precision is informational unless asked for, or in conservative mode.
370
+ const strictPrecision = strictPrecisionRaw === true || conservative;
371
+ const root = requireRecord(input, 'input');
372
+ assertKnownKeys(root, INPUT_KEYS, 'input');
373
+ const claimRaw = root.claim;
374
+ const sampleRaw = root.sample;
375
+ const provenanceRaw = root.provenance;
376
+ const samplingRaw = root.sampling;
377
+ const strataRaw = root.strata;
378
+ const retractionsRaw = root.retractions;
379
+ const benchmarkRaw = root.benchmark;
380
+ const populationRaw = root.population;
381
+ const evaluatedAtRaw = root.evaluatedAt;
382
+ const claimRec = requireRecord(claimRaw, 'claim');
383
+ assertKnownKeys(claimRec, CLAIM_KEYS, 'claim');
384
+ const metricRaw = claimRec.metric;
385
+ const valueRaw = claimRec.value;
386
+ const displayRaw = claimRec.display;
387
+ const textRaw = claimRec.text;
388
+ // The metric: judged, or known-unsupported (refused below), or a typo (thrown).
389
+ const { metric, judged } = readMetric(metricRaw);
390
+ const higherIsBetter = judged && HIGHER_IS_BETTER.includes(metric);
391
+ // A value is a proportion from 0 to 1 for a judged metric. An unsupported
392
+ // metric (a word error rate can pass 1) only needs to be a number.
393
+ const claimed = readFiniteInRange(valueRaw, 'claim.value', -Infinity, Infinity);
394
+ // A value within the floating-point noise of 0 or 1 is the absolute claim of 0 or
395
+ // 1 (ten 0.1s added up is 100%, not "99.99999999999999% so matches 99.5%"), and
396
+ // that includes a computed value that overshoots by the noise (1.0000000000000002).
397
+ // Further out it is not a proportion. `claimed` stays as supplied for the result;
398
+ // every judgment uses `judgedValue`.
399
+ if (judged && (claimed < -VALUE_NOISE || claimed > 1 + VALUE_NOISE)) {
400
+ throw new RangeError(`claim.value must be a finite number between 0 and 1, got ${String(claimed)}`);
401
+ }
402
+ const judgedValue = judged ? snapAbsolute(claimed) : claimed;
403
+ if (displayRaw !== undefined && typeof displayRaw !== 'string')
404
+ throw new TypeError('claim.display must be a string');
405
+ let figure;
406
+ if (displayRaw !== undefined && judged) {
407
+ figure = parseDisplayFigure(displayRaw, higherIsBetter);
408
+ // Exact: the value must agree with the printed figure to its rounding plus
409
+ // one ulp of the value. No float tolerance is involved.
410
+ if (!figure.consistentWith(judgedValue)) {
411
+ throw new RangeError(`claim.display "${clip(displayRaw, 40)}" (${mismatchPercent(figure.info.figure)}) does not match claim.value ${String(claimed)} (${mismatchPercent(claimed)}): ${figure.mismatchHint}`);
412
+ }
413
+ }
414
+ const displayIsPoint = figure !== undefined && figure.bound === null;
415
+ const displayDecimals = figure !== undefined && figure.info.form !== 'one-in-n' && figure.bound === null ? figure.info.decimals : null;
416
+ let claimText;
417
+ if (textRaw !== undefined) {
418
+ if (typeof textRaw !== 'string')
419
+ throw new TypeError('claim.text must be a string');
420
+ if (textRaw.length > MAX_TEXT_LENGTH)
421
+ throw new RangeError(`claim.text must be at most ${String(MAX_TEXT_LENGTH)} characters`);
422
+ assertPlainText(textRaw, 'claim.text', 'put the claim on one line (join the lines with spaces) before passing it');
423
+ claimText = textRaw;
424
+ }
425
+ const counts = readSampleCounts(sampleRaw);
426
+ const provRec = requireRecord(provenanceRaw, 'provenance');
427
+ assertKnownKeys(provRec, PROVENANCE_KEYS, 'provenance');
428
+ const sourceRaw = provRec.source;
429
+ const ratersRaw = provRec.raters;
430
+ const agreementRaw = provRec.agreement;
431
+ const source = readEnum(sourceRaw, SOURCES, 'provenance.source');
432
+ const raters = ratersRaw === undefined ? undefined : readCount(ratersRaw, 'provenance.raters', 1);
433
+ const agreement = agreementRaw === undefined ? undefined : readAgreement(agreementRaw);
434
+ if (agreement !== undefined && raters !== undefined && raters < 2) {
435
+ // Agreement is between raters; one rater cannot have produced it.
436
+ throw new RangeError(`provenance.raters is ${String(raters)} but provenance.agreement was supplied; agreement needs at least 2 raters`);
437
+ }
438
+ const sampling = readEnum(samplingRaw, SAMPLINGS, 'sampling');
439
+ const isFixedSet = sampling === 'complete' || sampling === 'held-out-split';
440
+ let benchmarkName;
441
+ if (benchmarkRaw !== undefined) {
442
+ const rec = requireRecord(benchmarkRaw, 'benchmark');
443
+ assertKnownKeys(rec, BENCHMARK_KEYS, 'benchmark');
444
+ const nameRaw = rec.name;
445
+ benchmarkName = readName(nameRaw, 'benchmark.name');
446
+ if (!isFixedSet) {
447
+ throw new RangeError("benchmark is only for sampling: 'complete' or 'held-out-split'");
448
+ }
449
+ }
450
+ let population;
451
+ if (populationRaw !== undefined) {
452
+ const rec = requireRecord(populationRaw, 'population');
453
+ assertKnownKeys(rec, POPULATION_KEYS, 'population');
454
+ const claimedRaw = rec.claimed;
455
+ const evaluatedRaw = rec.evaluated;
456
+ population = {
457
+ claimed: readName(claimedRaw, 'population.claimed'),
458
+ evaluated: readName(evaluatedRaw, 'population.evaluated'),
459
+ };
460
+ }
461
+ const evaluatedDay = evaluatedAtRaw === undefined ? undefined : dayNumber(evaluatedAtRaw, 'evaluatedAt');
462
+ if (evaluatedDay !== undefined && nowDay !== undefined && evaluatedDay > nowDay) {
463
+ throw new RangeError('evaluatedAt is after options.now');
464
+ }
465
+ const strata = strataRaw === undefined ? undefined : readStrata(strataRaw);
466
+ // The strata's own totals, as text for a refusal that asks for the pooled counts (null when none or too big to say)
467
+ let strataTotals = null;
468
+ if (strata !== undefined) {
469
+ // BigInt: a plain sum can round past 2^53 and falsely equal the total.
470
+ let sumN = 0n;
471
+ let sumCorrect = 0n;
472
+ for (const s of strata) {
473
+ sumN += BigInt(s.n);
474
+ sumCorrect += BigInt(s.correct);
475
+ }
476
+ if (counts !== null) {
477
+ if (sumN !== BigInt(counts.n) || sumCorrect !== BigInt(counts.correct)) {
478
+ throw new RangeError('strata counts must add up to the sample counts');
479
+ }
480
+ }
481
+ else if (sumN <= BigInt(MAX_COUNT)) {
482
+ strataTotals = `{ n: ${String(sumN)}, correct: ${String(sumCorrect)} }`;
483
+ }
484
+ }
485
+ let retractionCount = 0;
486
+ let retractionsIncluded = true;
487
+ if (retractionsRaw !== undefined) {
488
+ const rec = requireRecord(retractionsRaw, 'retractions');
489
+ assertKnownKeys(rec, RETRACTION_KEYS, 'retractions');
490
+ const countRaw = rec.count;
491
+ const includedRaw = rec.includedInSample;
492
+ retractionCount = readCount(countRaw, 'retractions.count', 0);
493
+ if (typeof includedRaw !== 'boolean') {
494
+ throw new TypeError('retractions.includedInSample must be a boolean');
495
+ }
496
+ retractionsIncluded = includedRaw;
497
+ if (retractionsIncluded && counts !== null && retractionCount > counts.n) {
498
+ throw new RangeError('retractions.count is larger than the sample, so it cannot all be included');
499
+ }
500
+ }
501
+ // Readers see the printed figure, so the more favorable of the value and the
502
+ // parsed figure is what gets judged. A display that disagrees with the value
503
+ // by more than its own rounding already threw above.
504
+ let assessed = judgedValue;
505
+ let fromDisplay = false;
506
+ if (figure !== undefined) {
507
+ const shown = figure.info.figure;
508
+ // A little noise apart is the same figure (82.7 / 100 against 0.827, 1 - 0.8 against
509
+ // 0.2), not a more favorable one.
510
+ const displayWins = beyondNoise(shown, judgedValue, higherIsBetter);
511
+ if (displayWins) {
512
+ assessed = shown;
513
+ fromDisplay = true;
514
+ }
515
+ }
516
+ const parsedDisplay = figure === undefined ? null : figure.info;
517
+ // Which optional checks this call actually ran. A gate that was asked for
518
+ // ('applied') is different from one that was left out ('not-supplied').
519
+ const checks = {
520
+ retractions: retractionsRaw !== undefined ? 'applied' : 'not-supplied',
521
+ scope: population !== undefined ? 'applied' : 'not-supplied',
522
+ age: maxAgeDays !== undefined ? 'applied' : 'not-supplied',
523
+ // applied = a minimum was required, or a statistic with a value and an n had its size checked;
524
+ // a statistic that has no value is never examined unless a minimum asks for it
525
+ agreement: minAgreement !== undefined || (agreement !== undefined && agreement.n !== null && agreement.value !== null) ? 'applied' : 'not-supplied',
526
+ // applied = two or more strata could be compared; one stratum has nothing to differ from
527
+ strata: strata !== undefined && strata.length > 1 && !isFixedSet ? 'applied' : 'not-supplied',
528
+ };
529
+ // ---- preconditions -----------------------------------------------------
530
+ const refusals = [];
531
+ const notes = [];
532
+ const metricWord = String(metric);
533
+ if (!judged) {
534
+ refusals.push('metric-not-supported');
535
+ notes.push(`"${clip(String(metric))}" is not a proportion of independently checked items. This kit judges ${SUPPORTED_LIST}; for anything else, report the counts and method instead.`);
536
+ }
537
+ if (source === 'model') {
538
+ refusals.push('no-human-labels');
539
+ notes.push('The labels came from a model, not from people, so there is no human ground truth to count against.');
540
+ }
541
+ else if (source === 'unknown') {
542
+ refusals.push('unknown-label-source');
543
+ notes.push('The source of the labels is unknown, so it is not possible to tell whether people checked them.');
544
+ }
545
+ else if (source === 'mixed') {
546
+ refusals.push('mixed-label-source');
547
+ notes.push('Some labels came from a model. Count only the human-labeled items and assess those as a human sample.');
548
+ }
549
+ if (sampling === 'ranked') {
550
+ refusals.push('ranked-sample');
551
+ notes.push('The sample was chosen by rank (for example the top-scored items), which does not represent the rest.');
552
+ }
553
+ else if (sampling === 'convenience') {
554
+ refusals.push('convenience-sample');
555
+ notes.push('The sample is a convenience sample, so the interval does not describe any larger group.');
556
+ }
557
+ else if (sampling === 'unknown') {
558
+ refusals.push('unknown-sampling');
559
+ notes.push('How the sample was chosen is unknown, so the interval cannot be read as covering a larger group.');
560
+ }
561
+ if (sampling === 'stratified' && strata === undefined) {
562
+ refusals.push('stratified-without-strata');
563
+ notes.push('The sample is stratified but no strata were supplied. The pooled counts are not a simple random sample, so they cannot be read as the population rate; supply the strata.');
564
+ }
565
+ if (isFixedSet) {
566
+ if (benchmarkName === undefined) {
567
+ refusals.push('benchmark-unnamed');
568
+ notes.push("A complete-set or held-out-split claim must name the fixed set it was measured on (the 'benchmark' field).");
569
+ }
570
+ else if (claimText === undefined || !mentionsName(claimText, benchmarkName)) {
571
+ refusals.push('claim-not-scoped-to-benchmark');
572
+ notes.push(`The claim wording does not mention "${clip(benchmarkName)}". A score on a fixed set describes that set; word the claim for it, or supply a random sample for a general claim.`);
573
+ }
574
+ }
575
+ if (counts === null) {
576
+ refusals.push('n-undisclosed');
577
+ notes.push(strataTotals === null
578
+ ? 'The sample counts (n and correct or errors) were not supplied, so there is nothing to weigh the figure against.'
579
+ : `The pooled sample counts (n and correct or errors) were not supplied. The strata hold them: pass sample: ${strataTotals}, their sums, with the strata.`);
580
+ }
581
+ else if (!isFixedSet && counts.n < minN) {
582
+ refusals.push('sample-too-small');
583
+ notes.push(`The sample has ${String(counts.n)} items, below the minimum of ${String(minN)} needed for a verdict.`);
584
+ }
585
+ if (!isFixedSet && strata !== undefined && strata.length > 1) {
586
+ // Bonferroni: split the error budget across the strata so that many
587
+ // strata do not make a refusal likelier by chance alone.
588
+ const zStrata = zForTail((1 - confidence) / (2 * strata.length));
589
+ let highestLow = -Infinity;
590
+ let lowestHigh = Infinity;
591
+ let highestName = '';
592
+ let lowestName = '';
593
+ for (const s of strata) {
594
+ const w = wilsonWithZ(s.correct, s.n, zStrata);
595
+ if (w.low > highestLow) {
596
+ highestLow = w.low;
597
+ highestName = s.name;
598
+ }
599
+ if (w.high < lowestHigh) {
600
+ lowestHigh = w.high;
601
+ lowestName = s.name;
602
+ }
603
+ }
604
+ if (highestLow - lowestHigh > COMPARISON_TOLERANCE) {
605
+ refusals.push('merged-strata-differ');
606
+ notes.push(`The strata do not agree: "${clip(highestName)}" and "${clip(lowestName)}" have intervals that do not overlap, so one combined figure hides a real difference. Report each stratum.`);
607
+ }
608
+ }
609
+ if (retractionCount > 0 && !retractionsIncluded) {
610
+ refusals.push('retractions-not-included');
611
+ notes.push(`${String(retractionCount)} retracted item(s) are known but are not counted in the sample.`);
612
+ }
613
+ if (maxAgeDays !== undefined && nowDay !== undefined) {
614
+ if (evaluatedDay === undefined) {
615
+ refusals.push('evaluation-date-missing');
616
+ notes.push('A maximum age was set but no evaluation date (evaluatedAt) was supplied.');
617
+ }
618
+ else if (nowDay - evaluatedDay > maxAgeDays) {
619
+ refusals.push('evaluation-stale');
620
+ notes.push(`The evaluation is ${String(nowDay - evaluatedDay)} days old, over the allowed ${String(maxAgeDays)} days.`);
621
+ }
622
+ }
623
+ if (population !== undefined) {
624
+ if (foldText(population.claimed) !== foldText(population.evaluated)) {
625
+ refusals.push('scope-mismatch');
626
+ notes.push(`The claim covers "${clip(population.claimed)}" but the evaluation covered "${clip(population.evaluated)}".`);
627
+ }
628
+ }
629
+ else if (requireScope) {
630
+ refusals.push('scope-undeclared');
631
+ notes.push('The claimed and evaluated populations were not declared (population).');
632
+ }
633
+ if (minAgreement !== undefined) {
634
+ if (agreement === undefined) {
635
+ refusals.push('agreement-missing');
636
+ notes.push('A minimum agreement was required but no agreement statistic was supplied.');
637
+ }
638
+ else if (agreement.value === null) {
639
+ refusals.push('agreement-undefined');
640
+ notes.push(`${STATISTIC_LABEL[agreement.statistic]} is undefined for these labels, so the agreement requirement cannot be checked.`);
641
+ }
642
+ else if (agreement.value + COMPARISON_TOLERANCE < minAgreement) {
643
+ refusals.push('agreement-below-minimum');
644
+ notes.push(`${STATISTIC_LABEL[agreement.statistic]} is ${decimalCut(agreement.value, 3, 'down')}, below the required ${decimalCut(minAgreement, 3, 'up')}.`);
645
+ }
646
+ }
647
+ // An agreement statistic from a handful of items is noise (kappa of 1.0 from 6
648
+ // pairs), whether or not a minimum is required. A hand-written statistic with
649
+ // no n cannot be checked and is trusted.
650
+ if (agreement !== undefined && agreement.value !== null && agreement.n !== null && agreement.n < minN) {
651
+ refusals.push('agreement-sample-too-small');
652
+ notes.push(`${STATISTIC_LABEL[agreement.statistic]} was computed on ${String(agreement.n)} items, below the minimum of ${String(minN)}, so it says too little about the raters to count.`);
653
+ }
654
+ if (refusals.length > 0 || counts === null) {
655
+ const phrases = refusals.map((code) => REFUSAL_PHRASE[code]);
656
+ return {
657
+ verdict: 'refused',
658
+ reasons: refusals,
659
+ explanation: notes,
660
+ metric,
661
+ claimed,
662
+ assessedValue: assessed,
663
+ n: counts === null ? null : counts.n,
664
+ basis: 'none',
665
+ parsedDisplay,
666
+ checks,
667
+ interval: null,
668
+ conservativeBound: null,
669
+ suggestedStatement: `No ${metricWord} figure can be supported from this evidence: ${phrases.join('; ')}.`,
670
+ };
671
+ }
672
+ const { n, correct } = counts;
673
+ const supportedMetric = metric;
674
+ const successes = higherIsBetter ? correct : n - correct;
675
+ const labelWord = LABEL_WORD[source];
676
+ const datePart = evaluatedAtRaw === undefined ? '' : `, evaluated ${evaluatedAtRaw}`;
677
+ const metricName = METRIC_PHRASE[supportedMetric];
678
+ // ---- complete set or held-out split: compare with the exact count ---------
679
+ if (isFixedSet) {
680
+ const k = successes;
681
+ const censusReasons = [];
682
+ const censusNotes = [];
683
+ let censusVerdict;
684
+ const shown = Math.min(countDecimals(n), 2);
685
+ const exactText = pctRate(k / n, shown);
686
+ // Every figure the reader can see is judged according to its form: a point
687
+ // or approximate figure must match the count to its digits; a bound is an
688
+ // inequality (strict ones exclude equality); a range and "1 in N" are judged
689
+ // at their favorable end / precision band. The value is judged as a point
690
+ // too whenever it is more favorable than the printed figure, or when the
691
+ // printed figure is itself a point figure. 0 and 1 stay exact.
692
+ const form = figure?.info.form;
693
+ const pointDisplay = form === 'percent' || form === 'approx';
694
+ const valueMoreFavorable = figure !== undefined && beyondNoise(judgedValue, figure.info.figure, higherIsBetter);
695
+ const displayOk = figure === undefined || figure.satisfiedBy(k, n);
696
+ const valueOk = figure === undefined || valueMoreFavorable || pointDisplay ? matchesValue(judgedValue, k, n) : true;
697
+ const printedOnly = form === 'floor' || form === 'ceiling' || form === 'range' || form === 'one-in-n';
698
+ const ok = displayOk && valueOk;
699
+ if (ok) {
700
+ censusVerdict = 'supported';
701
+ censusReasons.push('census-matches-count');
702
+ censusNotes.push(`The figure ${printedOnly && form !== 'one-in-n' && form !== 'range' ? 'is consistent with' : 'matches'} the exact count (${String(k)} of ${String(n)}, ${exactText}) to ${figure === undefined ? VALUE_PRECISION : 'its printed precision'}.`);
703
+ }
704
+ else {
705
+ censusVerdict = 'contradicted';
706
+ censusReasons.push('claim-differs-from-count');
707
+ censusNotes.push(`The figure does not match the exact count (${String(k)} of ${String(n)}, ${exactText}). It may be stale or mistyped.`);
708
+ }
709
+ if (figure !== undefined && figure.bandNote !== null) {
710
+ censusNotes[0] = `${censusNotes[0]} ${figure.bandNote}`;
711
+ }
712
+ let censusPrecisionNote = '';
713
+ if (displayDecimals !== null && displayDecimals > countDecimals(n)) {
714
+ if (strictPrecision && censusVerdict === 'supported') {
715
+ censusReasons.length = 0;
716
+ censusNotes.length = 0;
717
+ censusVerdict = 'overstated';
718
+ }
719
+ censusReasons.push('precision-too-fine');
720
+ censusNotes.push(`The figure is printed to ${placesWord(displayDecimals)}; ${String(n)} items resolve ${supportsWord(countDecimals(n))}.`);
721
+ censusPrecisionNote = printAdvice(countDecimals(n));
722
+ }
723
+ const censusTextNote = textFigureNote(claimText, judgedValue, assessed, higherIsBetter);
724
+ if (censusTextNote !== null) {
725
+ censusReasons.push('text-figure-differs');
726
+ censusNotes.push(censusTextNote);
727
+ }
728
+ const name = benchmarkName;
729
+ const counted = higherIsBetter ? `${String(k)} of ${String(n)} correct` : `${String(k)} of ${String(n)} wrong`;
730
+ const setWord = sampling === 'held-out-split' ? 'held-out split' : 'complete set';
731
+ return {
732
+ verdict: censusVerdict,
733
+ reasons: censusReasons,
734
+ explanation: censusNotes,
735
+ metric,
736
+ claimed,
737
+ assessedValue: assessed,
738
+ n,
739
+ basis: 'census',
740
+ parsedDisplay,
741
+ checks,
742
+ interval: null,
743
+ conservativeBound: null,
744
+ suggestedStatement: `${counted} on ${name} (${setWord}, ${labelWord}${datePart}): ${exactText} ${metricName === 'accurate' ? 'accuracy' : metricName}. This describes that set only; it is not an estimate for other data${censusPrecisionNote}`,
745
+ };
746
+ }
747
+ // ---- the interval and the verdict ------------------------------------
748
+ const w = wilsonWithZ(successes, n, z);
749
+ const interval = { ...w, confidence, z, successes, n };
750
+ const conservativeBound = higherIsBetter ? w.low : w.high;
751
+ const cl = confidenceLabel(confidence);
752
+ // The tolerance absorbs binary noise at an interval limit. An exact 0 or 1 is
753
+ // an absolute claim with no noise in it, and must never be excused: at about
754
+ // 4e12 items the limit next to 0 or 1 is closer than the tolerance.
755
+ const tolFor = (x) => (x === 0 || x === 1 ? 0 : COMPARISON_TOLERANCE);
756
+ let verdict;
757
+ const reasons = [];
758
+ const explanation = [];
759
+ const claimedText = pctRate(assessed, 3);
760
+ const subject = fromDisplay
761
+ ? `The printed figure ${claimedText} (more favorable than the claim value ${pctRate(claimed, 3)})`
762
+ : `The claimed ${claimedText}`;
763
+ const qualifier = higherIsBetter ? '' : ` ${metricName}`;
764
+ // An honest report of the observed rate: a point figure (not a bound, not 0 or
765
+ // 100%) equal to successes / n to its printed precision. Never for a bound,
766
+ // and not at all with options.conservative.
767
+ const absolute = assessed === 0 || assessed === 1;
768
+ let matchesObserved = false;
769
+ if (!conservative && !absolute && (figure === undefined || displayIsPoint)) {
770
+ // The printed figure must match the rate to its own precision; a value that
771
+ // is more favorable than the printed figure must match as well, so a coarse
772
+ // value cannot stand in for a precise print.
773
+ const valueMoreFavorable = figure !== undefined && beyondNoise(judgedValue, figure.info.figure, higherIsBetter);
774
+ matchesObserved =
775
+ figure === undefined
776
+ ? matchesValue(judgedValue, successes, n)
777
+ : figure.matchesObserved(successes, n) && (!valueMoreFavorable || matchesValue(judgedValue, successes, n));
778
+ }
779
+ const top = pctBound(w.high, 3, 'ceil');
780
+ const bottom = pctBound(w.low, 3, 'floor');
781
+ const observedText = `${String(successes)} of ${String(n)}`;
782
+ const strictBound = figure !== undefined && figure.strict;
783
+ // A bound claim is supported when it sits on the cautious side of the
784
+ // interval's conservative end; a strict one ("over 99%") needs to clear it.
785
+ // The value and the printed figure are each judged on their own terms: only
786
+ // the printed bound carries the strictness.
787
+ const clears = (x, strict) => {
788
+ const tol = tolFor(x);
789
+ return higherIsBetter ? (strict ? x < w.low - tol : x <= w.low + tol) : strict ? x > w.high + tol : x >= w.high - tol;
790
+ };
791
+ const withinBound = clears(judgedValue, false) && (figure === undefined || clears(figure.info.figure, strictBound));
792
+ const matchesText = `matches the observed rate (${observedText}) to ${figure === undefined ? VALUE_PRECISION : 'its printed precision'}. The ${cl} Wilson interval is ${bottom} to ${top}.`;
793
+ if (withinBound) {
794
+ verdict = 'supported';
795
+ reasons.push('claim-within-bound');
796
+ explanation.push(higherIsBetter
797
+ ? `${subject} is ${strictBound ? 'below' : 'no higher than'} the lower end of the ${cl} Wilson interval (${bottom}).`
798
+ : `${subject}${qualifier} is ${strictBound ? 'above' : 'no lower than'} the upper end of the ${cl} Wilson interval (${top}).`);
799
+ }
800
+ else if (matchesObserved) {
801
+ // An honest report of the observed rate comes first: a rounded figure can
802
+ // land just outside the interval it was rounded from.
803
+ verdict = 'supported';
804
+ reasons.push('matches-observed-rate');
805
+ explanation.push(`${subject}${qualifier} ${matchesText}`);
806
+ }
807
+ else if (higherIsBetter ? assessed > w.high + tolFor(assessed) : assessed < w.low - tolFor(assessed)) {
808
+ verdict = 'contradicted';
809
+ reasons.push('claim-outside-interval');
810
+ // For an absolute claim near the interval limit, spell out why the counts rule it out.
811
+ const nearCertain = higherIsBetter
812
+ ? absolute && successes < n && pctCeil(w.high, 3) === pctCeil(1, 3)
813
+ ? ', which is just under 100% because not every item was counted right'
814
+ : null
815
+ : absolute && successes > 0 && pctFloor(w.low, 3) === pctFloor(0, 3)
816
+ ? ', which is just above 0% because at least one error was counted'
817
+ : null;
818
+ explanation.push(higherIsBetter
819
+ ? `${subject} is above the top of the ${cl} Wilson interval${nearCertain ?? ` (${top})`}.`
820
+ : `${subject}${qualifier} is below the bottom of the ${cl} Wilson interval${nearCertain ?? ` (${bottom})`}.`);
821
+ }
822
+ else {
823
+ verdict = 'overstated';
824
+ reasons.push('claim-beyond-conservative-bound');
825
+ if (absolute) {
826
+ // 100% (or 0%) is certainty: the Wilson interval itself ends at 1 (or 0) when every item agreed,
827
+ // but the conservative limit never establishes it.
828
+ explanation.push(higherIsBetter
829
+ ? `${subject} cannot be stated from a sample: even when every item was right (${observedText}), the conservative lower limit cannot establish 100%. The most the evidence supports is ${bottom}, the lower end of the ${cl} Wilson interval.`
830
+ : `${subject}${qualifier} cannot be stated from a sample: even when no item was wrong (${observedText}), the conservative upper limit cannot establish 0%. The least the evidence supports is ${top}, the upper end of the ${cl} Wilson interval.`);
831
+ }
832
+ else {
833
+ let sentence = higherIsBetter
834
+ ? `${subject} is inside the ${cl} Wilson interval but above its lower end (${bottom}), which is the most the evidence supports.`
835
+ : `${subject}${qualifier} is inside the ${cl} Wilson interval but below its upper end (${top}), which is the least the evidence supports.`;
836
+ // A bound printed rounded to the nearest digit ("99.1%+" from a limit of 99.046%) can land
837
+ // just past the limit; the figure to print is the limit rounded in the cautious direction.
838
+ const allowed = maxDisplayDecimals(w.high - w.low);
839
+ const adviceMaxDecimals = strictPrecision ? allowed : undefined;
840
+ const boundMode = higherIsBetter ? 'floor' : 'ceil';
841
+ const printedDecimals = figure !== undefined && figure.bound !== null ? figure.info.decimals : null;
842
+ if (printedDecimals !== null && figure !== undefined) {
843
+ const oneUnit = 10 ** -(printedDecimals + 2);
844
+ const shortfall = higherIsBetter ? assessed - w.low : w.high - assessed;
845
+ if (figure.bound === (higherIsBetter ? 'floor' : 'ceiling') && shortfall < oneUnit) {
846
+ const rounded = pctNumericBound(conservativeBound, printedDecimals, boundMode, adviceMaxDecimals);
847
+ if (rounded !== null) {
848
+ sentence += ` The shortfall is within the rounding of the last printed digit, so print the bound rounded ${higherIsBetter ? 'down' : 'up'}: ${rounded}.`;
849
+ }
850
+ }
851
+ }
852
+ // A figure on the cautious side of the observed rate is the usual honest round-down or
853
+ // truncation (99.6 printed as 99): say so, and what to print instead. Only advise a figure the
854
+ // same assessment would pass (see observedAdvice): the observed rate is no way out when it is 0
855
+ // or 100% (a sample cannot show certainty), with options.conservative, or when no display
856
+ // within the precision strictPrecision allows can match it; the bound is printed at the
857
+ // precision the interval permits when strictPrecision or conservative is set.
858
+ if (beyondNoise(w.point, assessed, higherIsBetter)) {
859
+ const printDecimals = strictPrecision ? allowed : Math.min(allowed + 2, 5);
860
+ const observedFigure = conservative ? null : observedAdvice(w.point, successes, n, strictPrecision ? allowed : null, higherIsBetter);
861
+ const observedRate = pctRate(w.point, 3);
862
+ // Repeat the figure only when the advice differs or carries a strict precision limit.
863
+ const observedPart = observedFigure === null
864
+ ? ''
865
+ : !strictPrecision && observedFigure === observedRate
866
+ ? 'print the observed rate, or '
867
+ : `print the observed rate (${observedFigure}), or `;
868
+ const lead = higherIsBetter
869
+ ? ` The figure is also below the observed rate (${observedText}, ${observedRate}): `
870
+ : ` The figure is also above the observed rate (${observedText}, ${observedRate}): `;
871
+ const rounded = pctNumericBound(conservativeBound, printDecimals, boundMode, adviceMaxDecimals);
872
+ const boundPart = rounded === null
873
+ ? 'report the counts'
874
+ : `go ${higherIsBetter ? 'down to the lower' : 'up to the upper'} end (${rounded})`;
875
+ sentence += `${lead}${observedPart}${boundPart}.`;
876
+ }
877
+ explanation.push(sentence);
878
+ }
879
+ }
880
+ if (figure !== undefined && figure.bandNote !== null) {
881
+ explanation[0] = `${explanation[0]} ${figure.bandNote}`;
882
+ }
883
+ let precisionNote = '';
884
+ if (displayDecimals !== null) {
885
+ const allowed = maxDisplayDecimals(w.high - w.low);
886
+ if (displayDecimals > allowed) {
887
+ const text = `The figure is printed to ${placesWord(displayDecimals)}, but the interval is ${pctNearest(w.high - w.low, 2).replace('%', '')} percentage points wide and supports ${supportsWord(allowed)}.`;
888
+ if (strictPrecision && verdict === 'supported') {
889
+ // The value itself is fine; only the printed precision overreaches.
890
+ reasons.length = 0;
891
+ explanation.length = 0;
892
+ verdict = 'overstated';
893
+ }
894
+ reasons.push('precision-too-fine');
895
+ explanation.push(text);
896
+ precisionNote = printAdvice(allowed);
897
+ }
898
+ }
899
+ // A claim far less favorable than the counts (outside the 99.9% interval on
900
+ // the pessimistic side) looks like a stale or mistyped figure. Informational:
901
+ // the verdict is unchanged.
902
+ if (verdict !== 'contradicted') {
903
+ const wide = wilsonWithZ(successes, n, zForConfidence(0.999));
904
+ const worse = higherIsBetter ? assessed < wide.low - tolFor(assessed) : assessed > wide.high + tolFor(assessed);
905
+ if (worse) {
906
+ reasons.push('claim-worse-than-counts');
907
+ explanation.push(`${subject} is far less favorable than the counts support (the 99.9% Wilson interval is ${pctBound(wide.low, 3, 'floor')} to ${pctBound(wide.high, 3, 'ceil')}). If this is a published figure it may be stale or wrong.`);
908
+ }
909
+ }
910
+ // The interval pools the strata as one simple random sample. It is not
911
+ // reweighted by population share, and the kit has no population shares to
912
+ // reweight with, so a stratified result says so (informational).
913
+ if (sampling === 'stratified') {
914
+ reasons.push('strata-not-reweighted');
915
+ explanation.push('The strata are pooled as one sample and not reweighted by population share, so the interval describes the pooled counts, not the population, unless each stratum was sampled in proportion to its size. Report each stratum, or reweight them yourself.');
916
+ }
917
+ const textNote = textFigureNote(claimText, judgedValue, assessed, higherIsBetter);
918
+ if (textNote !== null) {
919
+ reasons.push('text-figure-differs');
920
+ explanation.push(textNote);
921
+ }
922
+ // ---- statement ---------------------------------------------------------
923
+ // Use the same printed bound in the headline and the quoted interval.
924
+ const decimals = Math.min(maxDisplayDecimals(w.high - w.low) + 2, 5);
925
+ const statementLow = pctBound(w.low, decimals, 'floor');
926
+ const statementHigh = pctBound(w.high, decimals, 'ceil');
927
+ // The lead names the scope first: the figure is about this sample (or the
928
+ // declared evaluated population), so a pasted headline cannot read as a
929
+ // property of the system.
930
+ const scope = population === undefined ? 'On this sample' : `On this sample of ${population.evaluated}`;
931
+ const headline = higherIsBetter
932
+ ? `${scope}, at least ${statementLow} ${metricName}`
933
+ : `${scope}, at most ${statementHigh} ${metricName}`;
934
+ const caveat = sampling === 'cross-validation' ? SAMPLING_ERROR_ONLY_CV : SAMPLING_ERROR_ONLY;
935
+ const parts = [
936
+ `${cl} Wilson interval ${statementLow} to ${statementHigh}`,
937
+ `n = ${String(n)}`,
938
+ labelWord,
939
+ SAMPLE_WORD[sampling] ?? 'random sample',
940
+ ];
941
+ if (evaluatedAtRaw !== undefined)
942
+ parts.push(`evaluated ${evaluatedAtRaw}`);
943
+ if (agreement !== undefined && agreement.value !== null) {
944
+ parts.push(`${STATISTIC_LABEL[agreement.statistic]} ${decimalCut(agreement.value, 2, 'down')}`);
945
+ }
946
+ return {
947
+ verdict,
948
+ reasons,
949
+ explanation,
950
+ metric,
951
+ claimed,
952
+ assessedValue: assessed,
953
+ n,
954
+ basis: 'wilson-interval',
955
+ parsedDisplay,
956
+ checks,
957
+ interval,
958
+ conservativeBound,
959
+ suggestedStatement: `${headline} (${parts.join(', ')}); ${caveat}${precisionNote}`,
960
+ };
961
+ }
962
+ //# sourceMappingURL=assess.js.map