@mark1russell7/rigor-bench 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/build/index.d.ts +223 -3
- package/build/index.d.ts.map +1 -1
- package/build/index.js +459 -2
- package/build/index.js.map +1 -1
- package/package.json +1 -1
package/build/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as checkPairedTime, r as firstArm, t as caseRatios } from "./paired.js";
|
|
2
|
-
import { bootstrapMedian, bootstrapTwo, compareRatios, median, medianInterval, quantile, quantileInterval, quantileSamples, wilson } from "@mark1russell7/rigor-statistics";
|
|
2
|
+
import { bootstrapMedian, bootstrapTwo, compareRatios, holm, median, medianInterval, quantile, quantileInterval, quantileSamples, signTest, signTestPower, wilson } from "@mark1russell7/rigor-statistics";
|
|
3
3
|
import { random } from "@mark1russell7/rigor-generators";
|
|
4
4
|
// src/budget.ts
|
|
5
5
|
/**
|
|
@@ -1089,6 +1089,463 @@ function armVerdicts(run, aaPair, options = {}) {
|
|
|
1089
1089
|
for (const [pair, records] of Object.entries(run.pairs)) if (pair !== aaPair) out[pair] = timeVerdict(balanceOrders(records).records, aa, options);
|
|
1090
1090
|
return out;
|
|
1091
1091
|
}
|
|
1092
|
+
// src/measures.ts
|
|
1093
|
+
/**
|
|
1094
|
+
* The check that a trick does not buy a measure. A control is a strategy made to win one measure and nothing else.
|
|
1095
|
+
* A real entry that wins that measure can do the same trick, so the win alone tells little. This check reads the controls of a
|
|
1096
|
+
* bench, and it tells which wins need a partner measure beside them. It also tells which measures separate nothing.
|
|
1097
|
+
*
|
|
1098
|
+
* A test compares paired values with the sign test of `rigor-statistics` and with the Holm correction. The keys of one stratum come
|
|
1099
|
+
* from one family of inputs, so they are not independent draws. The p of a test holds only under the sign test's own null: each
|
|
1100
|
+
* untied key is a fair coin. A result is about these keys, and this check claims nothing about other inputs.
|
|
1101
|
+
*/
|
|
1102
|
+
const DEFAULT_ALPHA = .05;
|
|
1103
|
+
const DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
1104
|
+
function checkedDate(name, date) {
|
|
1105
|
+
const time = Date.parse(`${date}T00:00:00Z`);
|
|
1106
|
+
if (!DATE.test(date) || !Number.isFinite(time) || new Date(time).toISOString().slice(0, 10) !== date) throw new RangeError(`checkMeasures: ${name} is a date as YYYY-MM-DD, not ${JSON.stringify(date)}.`);
|
|
1107
|
+
return date;
|
|
1108
|
+
}
|
|
1109
|
+
/** This function refuses the rules that contradict each other, because a wrong rule reads as a passed check. */
|
|
1110
|
+
function checkedRules(options) {
|
|
1111
|
+
const controls = new Set(options.controls);
|
|
1112
|
+
const rules = /* @__PURE__ */ new Map();
|
|
1113
|
+
for (const rule of options.rules) {
|
|
1114
|
+
if (rules.has(rule.measure)) throw new RangeError(`checkMeasures: the measure ${JSON.stringify(rule.measure)} has two rules.`);
|
|
1115
|
+
rules.set(rule.measure, rule);
|
|
1116
|
+
}
|
|
1117
|
+
for (const rule of options.rules) {
|
|
1118
|
+
const name = JSON.stringify(rule.measure);
|
|
1119
|
+
for (const control of rule.controls) if (!controls.has(control)) throw new RangeError(`checkMeasures: the control ${JSON.stringify(control)} of ${name} is not in the controls of the options.`);
|
|
1120
|
+
for (const partner of rule.partners) {
|
|
1121
|
+
if (partner === rule.measure) throw new RangeError(`checkMeasures: ${name} names itself as a partner.`);
|
|
1122
|
+
if (!rules.has(partner)) throw new RangeError(`checkMeasures: the partner ${JSON.stringify(partner)} of ${name} has no rule.`);
|
|
1123
|
+
}
|
|
1124
|
+
if (rule.resolution !== void 0 && !(Number.isFinite(rule.resolution) && rule.resolution >= 0)) throw new RangeError(`checkMeasures: the resolution of ${name} is finite and 0 or more, not ${rule.resolution}.`);
|
|
1125
|
+
if (rule.noControl !== void 0) {
|
|
1126
|
+
if (rule.controls.length > 0) throw new RangeError(`checkMeasures: ${name} has controls and also noControl.`);
|
|
1127
|
+
checkedDate(`the date of ${name}`, rule.noControl.until);
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
return rules;
|
|
1131
|
+
}
|
|
1132
|
+
/** This function groups the rows. It refuses a value that is not finite, a repeat, and a key in two strata. */
|
|
1133
|
+
function groupedRows(rows) {
|
|
1134
|
+
const strata = /* @__PURE__ */ new Map();
|
|
1135
|
+
const measures = /* @__PURE__ */ new Map();
|
|
1136
|
+
for (const row of rows) {
|
|
1137
|
+
const where = `${JSON.stringify(row.key)} of ${JSON.stringify(row.group)} on ${JSON.stringify(row.measure)}`;
|
|
1138
|
+
if (!Number.isFinite(row.value)) throw new RangeError(`checkMeasures: the value of ${where} is not finite: ${row.value}.`);
|
|
1139
|
+
if (strata.has(row.key) && strata.get(row.key) !== row.stratum) throw new RangeError(`checkMeasures: the key ${JSON.stringify(row.key)} has two strata.`);
|
|
1140
|
+
strata.set(row.key, row.stratum);
|
|
1141
|
+
const measure = measures.get(row.measure) ?? /* @__PURE__ */ new Map();
|
|
1142
|
+
measures.set(row.measure, measure);
|
|
1143
|
+
const groups = measure.get(row.stratum) ?? /* @__PURE__ */ new Map();
|
|
1144
|
+
measure.set(row.stratum, groups);
|
|
1145
|
+
const keys = groups.get(row.group) ?? /* @__PURE__ */ new Map();
|
|
1146
|
+
groups.set(row.group, keys);
|
|
1147
|
+
if (keys.has(row.key)) throw new RangeError(`checkMeasures: the row ${where} comes twice. State how the repeats reduce to one row.`);
|
|
1148
|
+
keys.set(row.key, row);
|
|
1149
|
+
}
|
|
1150
|
+
return measures;
|
|
1151
|
+
}
|
|
1152
|
+
/** This function gives the least count of untied keys with the power to pass at `level`. */
|
|
1153
|
+
function pairsNeeded(level, sides) {
|
|
1154
|
+
let n = 1;
|
|
1155
|
+
while (signTestPower(n, level, sides) === void 0) n += 1;
|
|
1156
|
+
return n;
|
|
1157
|
+
}
|
|
1158
|
+
/** This function compares two groups on the keys that both finished. A positive difference favours `better`. */
|
|
1159
|
+
function rawTest(better, other, stratum, groups, sense) {
|
|
1160
|
+
const a = groups.get(better);
|
|
1161
|
+
const b = groups.get(other);
|
|
1162
|
+
if (a === void 0 || b === void 0) return void 0;
|
|
1163
|
+
const differences = [];
|
|
1164
|
+
for (const [key, row] of a) {
|
|
1165
|
+
const pair = b.get(key);
|
|
1166
|
+
if (pair === void 0 || row.censored === true || pair.censored === true) continue;
|
|
1167
|
+
differences.push(sense === "higher" ? row.value - pair.value : pair.value - row.value);
|
|
1168
|
+
}
|
|
1169
|
+
if (differences.length === 0) return void 0;
|
|
1170
|
+
const wins = differences.filter((d) => d > 0).length;
|
|
1171
|
+
const losses = differences.filter((d) => d < 0).length;
|
|
1172
|
+
return {
|
|
1173
|
+
better,
|
|
1174
|
+
other,
|
|
1175
|
+
stratum,
|
|
1176
|
+
shared: differences.length,
|
|
1177
|
+
wins,
|
|
1178
|
+
losses,
|
|
1179
|
+
medianDifference: median(differences)
|
|
1180
|
+
};
|
|
1181
|
+
}
|
|
1182
|
+
/** This function corrects one family of tests, over all strata. A test with too few keys beats nothing, even when a step of Holm passes it. */
|
|
1183
|
+
function corrected(tests, sides, alpha, resolution) {
|
|
1184
|
+
if (tests.length === 0) return [];
|
|
1185
|
+
const family = tests.length;
|
|
1186
|
+
const needs = pairsNeeded(alpha / family, sides);
|
|
1187
|
+
const ps = tests.map((t) => signTest(t.wins, t.losses, sides));
|
|
1188
|
+
const steps = holm(ps, alpha);
|
|
1189
|
+
return tests.map((t, i) => {
|
|
1190
|
+
const n = t.wins + t.losses;
|
|
1191
|
+
const tooFew = n < needs;
|
|
1192
|
+
const size = sides === 1 ? t.medianDifference : Math.abs(t.medianDifference);
|
|
1193
|
+
const clears = resolution === void 0 || size > resolution;
|
|
1194
|
+
return {
|
|
1195
|
+
...t,
|
|
1196
|
+
n,
|
|
1197
|
+
ties: t.shared - n,
|
|
1198
|
+
p: ps[i],
|
|
1199
|
+
adjusted: steps[i]?.adjusted,
|
|
1200
|
+
family,
|
|
1201
|
+
needs,
|
|
1202
|
+
tooFew,
|
|
1203
|
+
tied: tooFew && t.shared >= needs,
|
|
1204
|
+
beats: !tooFew && steps[i]?.reject === true && clears
|
|
1205
|
+
};
|
|
1206
|
+
});
|
|
1207
|
+
}
|
|
1208
|
+
const stratumText = (stratum) => stratum === void 0 ? "" : ` in the stratum ${JSON.stringify(stratum)}`;
|
|
1209
|
+
/** This function tells that a test has too few keys to read, so a tie on enough keys is a read. */
|
|
1210
|
+
const unread = (c) => c.tooFew && !c.tied;
|
|
1211
|
+
/** The reads of one control on one measure: one for each stratum of the measure. */
|
|
1212
|
+
function controlReads(measure, control, data, rule, real, alpha) {
|
|
1213
|
+
if (data === void 0) return [];
|
|
1214
|
+
const comparisons = corrected([...data].flatMap(([stratum, groups]) => real.flatMap((entry) => rawTest(entry, control, stratum, groups, rule.sense) ?? [])), 1, alpha, rule.resolution);
|
|
1215
|
+
const attacks = corrected([...data].flatMap(([stratum, groups]) => real.flatMap((entry) => rawTest(control, entry, stratum, groups, rule.sense) ?? [])), 1, alpha, rule.resolution);
|
|
1216
|
+
return [...data.keys()].map((stratum) => {
|
|
1217
|
+
const here = comparisons.filter((c) => c.stratum === stratum);
|
|
1218
|
+
const against = attacks.filter((c) => c.stratum === stratum);
|
|
1219
|
+
return {
|
|
1220
|
+
measure,
|
|
1221
|
+
control,
|
|
1222
|
+
stratum,
|
|
1223
|
+
status: here.length === 0 ? "no-rows" : here.some((c) => c.beats) ? "beaten" : here.every(unread) ? "too-few" : "wins",
|
|
1224
|
+
aheadOf: here.filter((c) => c.losses > c.wins).length,
|
|
1225
|
+
comparisons: here,
|
|
1226
|
+
beatsEntries: against.filter((c) => c.beats).length,
|
|
1227
|
+
of: against.filter((c) => !unread(c)).length,
|
|
1228
|
+
against
|
|
1229
|
+
};
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1232
|
+
/** The read of the spread of one measure: one family over all pairs of real entries and all strata. */
|
|
1233
|
+
function spreadReads(data, rule, real, alpha) {
|
|
1234
|
+
const comparisons = corrected([...data].flatMap(([stratum, groups]) => real.flatMap((entry, i) => real.slice(i + 1).flatMap((other) => rawTest(entry, other, stratum, groups, rule.sense) ?? []))), 2, alpha, rule.resolution);
|
|
1235
|
+
return {
|
|
1236
|
+
reads: [...data.keys()].map((stratum) => {
|
|
1237
|
+
const here = comparisons.filter((c) => c.stratum === stratum);
|
|
1238
|
+
const widest = here.length === 0 ? void 0 : Math.max(...here.map((c) => Math.abs(c.medianDifference)));
|
|
1239
|
+
return {
|
|
1240
|
+
stratum,
|
|
1241
|
+
pairs: here.length,
|
|
1242
|
+
separated: here.filter((c) => c.beats).length,
|
|
1243
|
+
tooFew: here.filter(unread).length,
|
|
1244
|
+
widest
|
|
1245
|
+
};
|
|
1246
|
+
}),
|
|
1247
|
+
comparisons
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
function countsOf(data) {
|
|
1251
|
+
const counts = [];
|
|
1252
|
+
for (const [stratum, groups] of data ?? []) for (const [group, keys] of groups) {
|
|
1253
|
+
const censored = [...keys.values()].filter((row) => row.censored === true).length;
|
|
1254
|
+
counts.push({
|
|
1255
|
+
group,
|
|
1256
|
+
stratum,
|
|
1257
|
+
finished: keys.size - censored,
|
|
1258
|
+
censored
|
|
1259
|
+
});
|
|
1260
|
+
}
|
|
1261
|
+
return counts;
|
|
1262
|
+
}
|
|
1263
|
+
/** This function gives the methods of a measure. A row with no method counts as one more method beside a named one. */
|
|
1264
|
+
function methodsOf(data) {
|
|
1265
|
+
const methods = /* @__PURE__ */ new Set();
|
|
1266
|
+
for (const groups of data?.values() ?? []) for (const keys of groups.values()) for (const row of keys.values()) methods.add(row.method);
|
|
1267
|
+
return [...methods];
|
|
1268
|
+
}
|
|
1269
|
+
/** This function tells that a test clears a control on one of `partners` partners. One passed partner is enough, so the level divides by their count. */
|
|
1270
|
+
const clearsPartner = (c, partners, alpha) => c.beats && c.adjusted * partners <= alpha;
|
|
1271
|
+
/** This function reads a control on one of `partners` partners. A test with too few shared keys for the divided level has no power, so a miss there is not a read. */
|
|
1272
|
+
const partnerStatus = (read, partners, alpha) => {
|
|
1273
|
+
if (read === void 0 || read.status === "no-rows") return "no-rows";
|
|
1274
|
+
if (read.comparisons.some((c) => clearsPartner(c, partners, alpha))) return "lost";
|
|
1275
|
+
const needs = pairsNeeded(alpha / ((read.comparisons[0]?.family ?? 1) * partners), 1);
|
|
1276
|
+
return read.comparisons.every((c) => c.shared < needs) ? "too-few" : "not-lost";
|
|
1277
|
+
};
|
|
1278
|
+
function readsOf(reading, measure, control) {
|
|
1279
|
+
const memoKey = JSON.stringify([measure, control]);
|
|
1280
|
+
const known = reading.memo.get(memoKey);
|
|
1281
|
+
if (known !== void 0) return known;
|
|
1282
|
+
const rule = reading.rules.get(measure);
|
|
1283
|
+
const reads = reading.mixed.has(measure) ? [] : controlReads(measure, control, reading.data.get(measure), rule, reading.real, reading.alpha);
|
|
1284
|
+
reading.memo.set(memoKey, reads);
|
|
1285
|
+
return reads;
|
|
1286
|
+
}
|
|
1287
|
+
/** The findings of the rows and the controls of one rule that have nothing to read. */
|
|
1288
|
+
function missingFindings(rule, reading) {
|
|
1289
|
+
const findings = [];
|
|
1290
|
+
const hasRows = (measure, group) => [...reading.data.get(measure)?.values() ?? []].some((groups) => groups.has(group));
|
|
1291
|
+
const missing = (text, control) => {
|
|
1292
|
+
findings.push({
|
|
1293
|
+
kind: "no-rows",
|
|
1294
|
+
measure: rule.measure,
|
|
1295
|
+
...control === void 0 ? {} : { control },
|
|
1296
|
+
text,
|
|
1297
|
+
comparisons: []
|
|
1298
|
+
});
|
|
1299
|
+
};
|
|
1300
|
+
if (!reading.data.has(rule.measure)) missing(`The measure ${rule.measure} has a rule and no rows.`);
|
|
1301
|
+
for (const control of rule.controls) {
|
|
1302
|
+
if (reading.data.has(rule.measure) && !hasRows(rule.measure, control)) missing(`The control ${control} has no rows on ${rule.measure}, so it reads as nothing and not as "not beaten".`, control);
|
|
1303
|
+
for (const partner of rule.partners) if (!hasRows(partner, control)) missing(`The control ${control} has no rows on the partner ${partner} of ${rule.measure}, so it reads as nothing and not as "not beaten".`, control);
|
|
1304
|
+
}
|
|
1305
|
+
return findings;
|
|
1306
|
+
}
|
|
1307
|
+
/** The findings of the win of a control on one stratum: the partners and the censored keys. */
|
|
1308
|
+
function winFindings(rule, read, guard, counts, partnerReads) {
|
|
1309
|
+
const findings = [];
|
|
1310
|
+
const where = stratumText(read.stratum);
|
|
1311
|
+
const count = guard.partners.length;
|
|
1312
|
+
const thin = guard.partners.filter((p) => p.status === "too-few").map((p) => p.partner);
|
|
1313
|
+
const noRows = guard.partners.some((p) => p.status === "no-rows");
|
|
1314
|
+
const behind = partnerReads.flatMap((r) => r.comparisons);
|
|
1315
|
+
if (!guard.partners.some((p) => p.status === "lost")) {
|
|
1316
|
+
if (thin.length > 0) findings.push({
|
|
1317
|
+
kind: "too-few",
|
|
1318
|
+
measure: rule.measure,
|
|
1319
|
+
stratum: read.stratum,
|
|
1320
|
+
control: read.control,
|
|
1321
|
+
partners: count,
|
|
1322
|
+
text: `The control ${read.control} wins ${rule.measure}${where} and loses no partner. Of its ${count} partners, ${thin.join(", ")} have too few keys for a read.`,
|
|
1323
|
+
comparisons: behind
|
|
1324
|
+
});
|
|
1325
|
+
else if (!noRows) {
|
|
1326
|
+
const list = count === 0 ? "It has no partner." : `It loses none of its ${count} partners, ${guard.partners.map((p) => p.partner).join(", ")}.`;
|
|
1327
|
+
findings.push({
|
|
1328
|
+
kind: "uncaught",
|
|
1329
|
+
measure: rule.measure,
|
|
1330
|
+
stratum: read.stratum,
|
|
1331
|
+
control: read.control,
|
|
1332
|
+
partners: count,
|
|
1333
|
+
text: `The control ${read.control} wins ${rule.measure}${where}. ${list}`,
|
|
1334
|
+
comparisons: [...read.comparisons, ...behind]
|
|
1335
|
+
});
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
const own = counts.find((c) => c.group === read.control && c.stratum === read.stratum)?.censored ?? 0;
|
|
1339
|
+
const fewer = counts.filter((c) => c.group !== read.control && c.stratum === read.stratum && read.comparisons.some((t) => t.better === c.group) && c.censored < own);
|
|
1340
|
+
if (fewer.length > 0) {
|
|
1341
|
+
const list = fewer.map((c) => `${c.group} ${c.censored}`).join(", ");
|
|
1342
|
+
findings.push({
|
|
1343
|
+
kind: "censored-win",
|
|
1344
|
+
measure: rule.measure,
|
|
1345
|
+
stratum: read.stratum,
|
|
1346
|
+
control: read.control,
|
|
1347
|
+
text: `The control ${read.control} wins ${rule.measure}${where} with ${own} censored keys, more than ${list}. Its win can come from the easy keys.`,
|
|
1348
|
+
comparisons: read.comparisons.filter((t) => fewer.some((c) => c.group === t.better))
|
|
1349
|
+
});
|
|
1350
|
+
}
|
|
1351
|
+
return findings;
|
|
1352
|
+
}
|
|
1353
|
+
/** The read of one rule, with its findings. */
|
|
1354
|
+
function checkRule(rule, reading, asOf) {
|
|
1355
|
+
const data = reading.data.get(rule.measure);
|
|
1356
|
+
const methods = methodsOf(data);
|
|
1357
|
+
const mixed = reading.mixed.has(rule.measure);
|
|
1358
|
+
const counts = countsOf(data);
|
|
1359
|
+
const findings = missingFindings(rule, reading);
|
|
1360
|
+
const reads = [];
|
|
1361
|
+
const guards = [];
|
|
1362
|
+
if (mixed) findings.push({
|
|
1363
|
+
kind: "mixed-method",
|
|
1364
|
+
measure: rule.measure,
|
|
1365
|
+
text: `The measure ${rule.measure} mixes the methods ${methods.map((m) => m ?? "(none)").join(", ")}. The check has no read of it until the rows are split or marked.`,
|
|
1366
|
+
comparisons: []
|
|
1367
|
+
});
|
|
1368
|
+
if (rule.controls.length === 0) {
|
|
1369
|
+
if (rule.noControl === void 0) findings.push({
|
|
1370
|
+
kind: "no-control",
|
|
1371
|
+
measure: rule.measure,
|
|
1372
|
+
text: `The measure ${rule.measure} has no control and no noControl with a date.`,
|
|
1373
|
+
comparisons: []
|
|
1374
|
+
});
|
|
1375
|
+
else if (rule.noControl.until < asOf) findings.push({
|
|
1376
|
+
kind: "control-late",
|
|
1377
|
+
measure: rule.measure,
|
|
1378
|
+
text: `The control of ${rule.measure} was due on ${rule.noControl.until}, and the date of the check is ${asOf}.`,
|
|
1379
|
+
comparisons: []
|
|
1380
|
+
});
|
|
1381
|
+
}
|
|
1382
|
+
for (const control of rule.controls) for (const read of readsOf(reading, rule.measure, control)) {
|
|
1383
|
+
const partnerReads = rule.partners.map((partner) => readsOf(reading, partner, control).find((r) => r.stratum === read.stratum));
|
|
1384
|
+
const partners = rule.partners.map((partner, i) => ({
|
|
1385
|
+
partner,
|
|
1386
|
+
status: partnerStatus(partnerReads[i], rule.partners.length, reading.alpha)
|
|
1387
|
+
}));
|
|
1388
|
+
const guard = {
|
|
1389
|
+
control,
|
|
1390
|
+
stratum: read.stratum,
|
|
1391
|
+
status: read.status,
|
|
1392
|
+
partners
|
|
1393
|
+
};
|
|
1394
|
+
guards.push(guard);
|
|
1395
|
+
reads.push(read, ...partnerReads.filter((r) => r !== void 0));
|
|
1396
|
+
if (read.status === "wins") findings.push(...winFindings(rule, read, guard, counts, partnerReads.filter((r) => r !== void 0)));
|
|
1397
|
+
}
|
|
1398
|
+
const controlComparisons = guards.length === 0 ? [] : reads.filter((r) => r.measure === rule.measure).flatMap((r) => r.comparisons);
|
|
1399
|
+
if (rule.controls.length > 0 && guards.length > 0 && !mixed) {
|
|
1400
|
+
if (guards.every((g) => g.status === "too-few" || g.status === "no-rows")) {
|
|
1401
|
+
const needs = Math.max(0, ...controlComparisons.map((c) => c.needs));
|
|
1402
|
+
findings.push({
|
|
1403
|
+
kind: "too-few",
|
|
1404
|
+
measure: rule.measure,
|
|
1405
|
+
text: `No control of ${rule.measure} has enough pairs for a read. A test needs ${needs} untied pairs.`,
|
|
1406
|
+
comparisons: controlComparisons
|
|
1407
|
+
});
|
|
1408
|
+
} else for (const control of rule.controls) {
|
|
1409
|
+
const own = reads.filter((r) => r.measure === rule.measure && r.control === control);
|
|
1410
|
+
const of = own.reduce((sum, r) => sum + r.of, 0);
|
|
1411
|
+
if (of > 0 && own.every((r) => r.beatsEntries === 0)) findings.push({
|
|
1412
|
+
kind: "weak-control",
|
|
1413
|
+
measure: rule.measure,
|
|
1414
|
+
control,
|
|
1415
|
+
text: `The control ${control} beats 0 of ${of} real entries on ${rule.measure} by test, in all strata. The trick buys nothing.`,
|
|
1416
|
+
comparisons: own.flatMap((r) => r.against)
|
|
1417
|
+
});
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
let spread = [];
|
|
1421
|
+
if (data !== void 0 && !mixed) {
|
|
1422
|
+
const { reads: spreadRows, comparisons } = spreadReads(data, rule, reading.real, reading.alpha);
|
|
1423
|
+
spread = spreadRows;
|
|
1424
|
+
const testable = comparisons.filter((c) => !unread(c));
|
|
1425
|
+
if (comparisons.length > 0 && testable.length === 0) findings.push({
|
|
1426
|
+
kind: "too-few",
|
|
1427
|
+
measure: rule.measure,
|
|
1428
|
+
text: `No pair of real entries has enough keys for a read of the spread of ${rule.measure}. A test needs ${comparisons[0]?.needs} untied pairs.`,
|
|
1429
|
+
comparisons: []
|
|
1430
|
+
});
|
|
1431
|
+
else if (testable.length > 0 && !comparisons.some((c) => c.beats)) {
|
|
1432
|
+
const widest = Math.max(...comparisons.map((c) => Math.abs(c.medianDifference)));
|
|
1433
|
+
const basis = rule.resolution === void 0 ? "by the test only" : `by the test and the resolution ${rule.resolution}`;
|
|
1434
|
+
findings.push({
|
|
1435
|
+
kind: "no-spread",
|
|
1436
|
+
measure: rule.measure,
|
|
1437
|
+
text: `No pair of the ${comparisons.length} pairs of real entries separates on ${rule.measure}, ${basis}. The widest median difference is ${widest}.`,
|
|
1438
|
+
comparisons: [...comparisons].sort((a, b) => a.adjusted - b.adjusted).slice(0, 1)
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
return {
|
|
1443
|
+
check: {
|
|
1444
|
+
measure: rule.measure,
|
|
1445
|
+
sense: rule.sense,
|
|
1446
|
+
basis: rule.resolution === void 0 ? "test only" : "test and resolution",
|
|
1447
|
+
method: mixed ? void 0 : methods[0],
|
|
1448
|
+
mixed,
|
|
1449
|
+
guards,
|
|
1450
|
+
spread,
|
|
1451
|
+
entries: counts,
|
|
1452
|
+
partners: rule.partners,
|
|
1453
|
+
open: rule.noControl
|
|
1454
|
+
},
|
|
1455
|
+
findings,
|
|
1456
|
+
reads
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
/**
|
|
1460
|
+
* This function reads the controls and the spread of each measure of a bench.
|
|
1461
|
+
*
|
|
1462
|
+
* - A control wins a measure when no real entry beats it on the keys that both finished. A tie counts for the control.
|
|
1463
|
+
* - A real entry beats a control when the Holm-corrected sign test passes and the median paired difference is over the resolution.
|
|
1464
|
+
* - One Holm family holds the tests of one control on one measure over all real entries and all strata.
|
|
1465
|
+
* - One more Holm family holds the tests of one control against the real entries, with the same need and resolution. A control that beats none of them is weak.
|
|
1466
|
+
* - One Holm family holds the tests of the spread of one measure over all pairs of real entries and all strata.
|
|
1467
|
+
* - A test with fewer untied keys than `signTestPower` needs for that family reads `tooFew` and beats nothing.
|
|
1468
|
+
* - A test with enough shared keys and too few untied keys is `tied`. The two entries read as equal, so a constant measure is `no-spread` and not `too-few`.
|
|
1469
|
+
* - A control loses a partner only at the level divided by the count of its partners. One passed partner clears `uncaught`, so each partner is one more chance.
|
|
1470
|
+
*
|
|
1471
|
+
* The function throws a `RangeError` for a value that is not finite, a repeat of measure, group and key, or a key in two strata.
|
|
1472
|
+
* It also throws one for rules that contradict each other.
|
|
1473
|
+
* The rules, the controls and the date are stated, so the result depends only on the arguments.
|
|
1474
|
+
*/
|
|
1475
|
+
function checkMeasures(rows, options) {
|
|
1476
|
+
const alpha = options.alpha ?? DEFAULT_ALPHA;
|
|
1477
|
+
if (!(alpha > 0 && alpha < 1)) throw new RangeError(`checkMeasures: alpha is over 0 and under 1, not ${alpha}.`);
|
|
1478
|
+
const asOf = checkedDate("asOf", options.asOf);
|
|
1479
|
+
const rules = checkedRules(options);
|
|
1480
|
+
const data = groupedRows(rows);
|
|
1481
|
+
const controls = new Set(options.controls);
|
|
1482
|
+
const reading = {
|
|
1483
|
+
rules,
|
|
1484
|
+
data,
|
|
1485
|
+
real: [...new Set(rows.map((row) => row.group))].filter((group) => !controls.has(group)),
|
|
1486
|
+
alpha,
|
|
1487
|
+
mixed: new Set([...data.keys()].filter((measure) => methodsOf(data.get(measure)).length > 1)),
|
|
1488
|
+
memo: /* @__PURE__ */ new Map()
|
|
1489
|
+
};
|
|
1490
|
+
const findings = [];
|
|
1491
|
+
for (const measure of data.keys()) if (!rules.has(measure)) findings.push({
|
|
1492
|
+
kind: "no-rule",
|
|
1493
|
+
measure,
|
|
1494
|
+
text: `The measure ${measure} has rows and no rule.`,
|
|
1495
|
+
comparisons: []
|
|
1496
|
+
});
|
|
1497
|
+
const measures = [];
|
|
1498
|
+
const reads = /* @__PURE__ */ new Map();
|
|
1499
|
+
for (const rule of options.rules) {
|
|
1500
|
+
const result = checkRule(rule, reading, asOf);
|
|
1501
|
+
measures.push(result.check);
|
|
1502
|
+
findings.push(...result.findings);
|
|
1503
|
+
for (const read of result.reads) reads.set(JSON.stringify([
|
|
1504
|
+
read.measure,
|
|
1505
|
+
read.control,
|
|
1506
|
+
read.stratum ?? null
|
|
1507
|
+
]), read);
|
|
1508
|
+
}
|
|
1509
|
+
return {
|
|
1510
|
+
alpha,
|
|
1511
|
+
measures,
|
|
1512
|
+
reads: [...reads.values()],
|
|
1513
|
+
findings
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1516
|
+
/**
|
|
1517
|
+
* This function tells what a page says about the win of one real entry on one measure.
|
|
1518
|
+
* The claim is about this entry: only this entry beating the control, on this measure or on a partner, clears the entry. A win by another entry clears nothing.
|
|
1519
|
+
* With a stratum, the function reads that stratum. Without one, it reads the rows with no stratum.
|
|
1520
|
+
* The function throws a `RangeError` for a measure with no read and for an entry that the rows do not hold.
|
|
1521
|
+
*/
|
|
1522
|
+
function winClaim(check, measure, entry, stratum) {
|
|
1523
|
+
const read = check.measures.find((m) => m.measure === measure);
|
|
1524
|
+
if (read === void 0) throw new RangeError(`winClaim: the measure ${JSON.stringify(measure)} has no rule.`);
|
|
1525
|
+
const count = read.entries.find((c) => c.group === entry && c.stratum === stratum);
|
|
1526
|
+
if (count === void 0) throw new RangeError(`winClaim: the entry ${JSON.stringify(entry)} has no rows on ${JSON.stringify(measure)}${stratumText(stratum)}.`);
|
|
1527
|
+
const guards = read.guards.filter((g) => g.stratum === stratum);
|
|
1528
|
+
const winning = guards.filter((g) => g.status === "wins");
|
|
1529
|
+
const lostByControl = (guard) => guard.partners.filter((p) => p.status === "lost").map((p) => p.partner);
|
|
1530
|
+
const testsOf = (of, control) => check.reads.find((r) => r.measure === of && r.control === control && r.stratum === stratum)?.comparisons ?? [];
|
|
1531
|
+
const notBeaten = guards.filter((g) => !testsOf(measure, g.control).some((c) => c.better === entry && c.beats));
|
|
1532
|
+
const clearsOnPartner = (guard) => guard.partners.some((p) => testsOf(p.partner, guard.control).some((c) => c.better === entry && clearsPartner(c, guard.partners.length, check.alpha)));
|
|
1533
|
+
const countOf = (control) => read.entries.find((c) => c.group === control && c.stratum === stratum)?.censored ?? 0;
|
|
1534
|
+
const alone = guards.length > 0 && notBeaten.length === 0;
|
|
1535
|
+
return {
|
|
1536
|
+
alone,
|
|
1537
|
+
show: notBeaten.length > 0 && notBeaten.every((g) => g.status === "wins") ? [...new Set(winning.flatMap(lostByControl))] : alone ? [] : read.partners,
|
|
1538
|
+
likeControl: notBeaten.some((guard) => !clearsOnPartner(guard)),
|
|
1539
|
+
censored: {
|
|
1540
|
+
entry: count.censored,
|
|
1541
|
+
controls: notBeaten.map((g) => ({
|
|
1542
|
+
control: g.control,
|
|
1543
|
+
censored: countOf(g.control)
|
|
1544
|
+
}))
|
|
1545
|
+
},
|
|
1546
|
+
open: read.open
|
|
1547
|
+
};
|
|
1548
|
+
}
|
|
1092
1549
|
// src/power.ts
|
|
1093
1550
|
/**
|
|
1094
1551
|
* The power test of a time verdict. It draws seeded benches with a known speedup and counts the verdicts that find it.
|
|
@@ -1203,6 +1660,6 @@ function timeRounds(arms, options) {
|
|
|
1203
1660
|
sink
|
|
1204
1661
|
};
|
|
1205
1662
|
}
|
|
1206
|
-
export { BUDGET_LOOKS, LADDER_FORMAT, aaBand, aaSpread, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, checkLadder, checkPairedTime, controlFactor, firstArm, ladderVerdict, limitVerdict, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, rungVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower };
|
|
1663
|
+
export { BUDGET_LOOKS, LADDER_FORMAT, aaBand, aaSpread, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, checkLadder, checkMeasures, checkPairedTime, controlFactor, firstArm, ladderVerdict, limitVerdict, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, rungVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower, winClaim };
|
|
1207
1664
|
|
|
1208
1665
|
//# sourceMappingURL=index.js.map
|