@mark1russell7/rigor-bench 0.2.25 → 0.2.27

package/build/index.js

@@ -28,7 +28,7 @@ function caseBudget(name, values, aaValues, options, level) {
 	const aaMedian = aaValues.length > 0 ? median(aaValues) : void 0;
 	const difference = aaMedian === void 0 ? void 0 : Math.abs(middle - aaMedian);
 	const aaCoverage = aaValues.length > 0 ? medianInterval(aaValues, level).coverage : 0;
-	const facts = `median ${round$3(middle)} of ${values.length} rounds, interval ${round$3(interval.low)} to ${round$3(interval.high)}, limit ${limit}, A/A median ${aaMedian === void 0 ? "none" : round$3(aaMedian)} of ${aaValues.length} rounds, difference ${difference === void 0 ? "none" : round$3(difference)}`;
+	const facts = `median ${round$4(middle)} of ${values.length} rounds, interval ${round$4(interval.low)} to ${round$4(interval.high)}, limit ${limit}, A/A median ${aaMedian === void 0 ? "none" : round$4(aaMedian)} of ${aaValues.length} rounds, difference ${difference === void 0 ? "none" : round$4(difference)}`;
 	const decide = (outcome, why) => ({
 		case: name,
 		rounds: values.length,
@@ -180,7 +180,7 @@ function budgetLooksVerdict(rounds, aa, options) {
 	const looks = options.looks ?? BUDGET_LOOKS;
 	const level = options.level ?? .95;
 	const spent = budgetLookSpend(looks);
-	if (spent > 1 - level) throw new RangeError(`budgetLooksVerdict: the looks spend ${round$3(spent)}, over the ${round$3(1 - level)} of the level ${level}.`);
+	if (spent > 1 - level) throw new RangeError(`budgetLooksVerdict: the looks spend ${round$4(spent)}, over the ${round$4(1 - level)} of the level ${level}.`);
 	if (!(limit >= 0 && Number.isFinite(limit))) throw new RangeError(`budgetLooksVerdict: the limit is finite and 0 or more, not ${limit}.`);
 	if (!(maxDifference >= 0 && Number.isFinite(maxDifference))) throw new RangeError(`budgetLooksVerdict: the largest difference is finite and 0 or more, not ${maxDifference}.`);
 	const byRound = (list) => [...list].sort((a, b) => a.round - b.round);
@@ -209,7 +209,7 @@ function budgetLooksVerdict(rounds, aa, options) {
 		fails: verdict.cases.some((c) => c.outcome === "beyond")
 	};
 }
-function round$3(x) {
+function round$4(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
 // src/growth.ts
@@ -658,8 +658,22 @@ function checkLadder(ladder) {
 	});
 	return out;
 }
+/** This function gives the reason that a list has a NaN, with the index of the first one, or undefined. */
+function nanAt(name, values) {
+	const index = values.findIndex((value) => Number.isNaN(value));
+	return index < 0 ? void 0 : `${name} ${index} is NaN`;
+}
 /** This function gives the verdict of a rate: the Wilson interval at 95 % against the limit. */
 function rateResult(budget, failures, trials) {
+	const whole = (value) => Number.isSafeInteger(value) && value >= 0;
+	if (!(whole(failures) && whole(trials) && failures <= trials)) {
+		const why = `the counts are whole numbers of 0 or more with failures not above trials, not ${failures} failures of ${trials} trials`;
+		return {
+			measure: budget.measure,
+			outcome: "cannot decide",
+			reason: `The measure ${JSON.stringify(budget.measure)} is "cannot decide": ${why}.`
+		};
+	}
 	const interval = wilson(failures, trials);
 	const facts = `${failures} failures of ${trials} trials, interval ${interval.low} to ${interval.high}, limit ${budget.limit}`;
 	const outcome = trials === 0 ? "cannot decide" : interval.high <= budget.limit ? "within" : interval.low > budget.limit ? "beyond" : "cannot decide";
@@ -671,6 +685,12 @@ function rateResult(budget, failures, trials) {
 }
 /** This function gives the verdict of a set of drills: the largest value against the limit. */
 function worstResult(budget, values) {
+	const nan = nanAt("the drill", values);
+	if (nan !== void 0) return {
+		measure: budget.measure,
+		outcome: "cannot decide",
+		reason: `The measure ${JSON.stringify(budget.measure)} is "cannot decide": ${nan}.`
+	};
 	const largest = values.length === 0 ? void 0 : Math.max(...values);
 	const facts = `${values.length} drills, the largest ${largest ?? "none"} ${budget.unit}, limit ${budget.limit}`;
 	const outcome = largest !== void 0 && largest > budget.limit ? "beyond" : values.length < budget.minDrills ? "cannot decide" : "within";
@@ -695,7 +715,8 @@ function quantileResult(budget, values, censored) {
 	const open = censored.length > 0;
 	const bounds = [...values, ...censored];
 	if (bounds.length === 0) return say("cannot decide", `0 values, limit ${budget.limit} ${budget.unit}`);
-	if (values.some((value) => Number.isNaN(value))) return say("cannot decide", "the values have a NaN");
+	const nan = nanAt("the value", values);
+	if (nan !== void 0) return say("cannot decide", nan);
 	if (!censored.every((value) => Number.isFinite(value))) return say("cannot decide", "a censored value is not a finite number");
 	const lowest = quantileInterval(bounds, budget.quantile, level);
 	const highest = open ? quantileInterval([...values, ...censored.map(() => Infinity)], budget.quantile, level) : lowest;
@@ -935,14 +956,14 @@ function timeVerdict(candidate, aa, options = {}) {
 			noise
 		};
 		if (noise.noisy && verdict.band !== void 0) {
-			const reason = `${spread === void 0 ? "The A/A arm of the run has no cases, so the noise cap has no spread." : `The A/A spread of the run is ${round$2(spread)}, above the noise cap ${cap}.`} Without the cap, the outcome is "${verdict.outcome}".`;
+			const reason = `${spread === void 0 ? "The A/A arm of the run has no cases, so the noise cap has no spread." : `The A/A spread of the run is ${round$3(spread)}, above the noise cap ${cap}.`} Without the cap, the outcome is "${verdict.outcome}".`;
 			verdict = {
 				...verdict,
 				outcome: "cannot decide",
 				reason
 			};
 		} else if (noise.biased && verdict.band !== void 0) {
-			const reason = `The A/A median ratio of the run is ${round$2(aaMedian)}, out of [${round$2(Math.exp(-cap))}, ${round$2(Math.exp(cap))}] of the noise cap ${cap}. Without the cap, the outcome is "${verdict.outcome}".`;
+			const reason = `The A/A median ratio of the run is ${round$3(aaMedian)}, out of [${round$3(Math.exp(-cap))}, ${round$3(Math.exp(cap))}] of the noise cap ${cap}. Without the cap, the outcome is "${verdict.outcome}".`;
 			verdict = {
 				...verdict,
 				outcome: "cannot decide",
@@ -1010,44 +1031,59 @@ function ruleVerdict(candidate, aa, options, pooled) {
 	};
 	if (rule === "difference") {
 		const margin = marginOf(options, pooled);
-		const shown = pooled === void 0 ? margin : round$2(margin);
+		const shown = pooled === void 0 ? margin : round$3(margin);
 		const logs = cases.map((c) => Math.log(c.ratio));
 		const aaLogs = aaRatios.map((c) => Math.log(c.ratio));
-		const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.bootstrap);
+		const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.pValues === true ? {
+			...options.bootstrap,
+			against: 0
+		} : options.bootstrap);
+		const { level, runs, finite, seed } = d;
 		const interval = {
-			...d,
 			estimate: Math.exp(d.estimate),
 			low: Math.exp(d.low),
-			high: Math.exp(d.high)
+			high: Math.exp(d.high),
+			level,
+			runs,
+			finite,
+			seed
 		};
-		const span = `The interval of the candidate median over the A/A median is [${round$2(interval.low)}, ${round$2(interval.high)}].`;
+		const sides = options.pValues === true ? {
+			pSlower: d.pAbove ?? 1,
+			pFaster: d.pBelow ?? 1
+		} : {};
+		const span = `The interval of the candidate median over the A/A median is [${round$3(interval.low)}, ${round$3(interval.high)}].`;
 		if (d.low >= -margin && d.high <= margin) return {
 			...facts,
+			...sides,
 			interval,
 			outcome: "same",
 			reason: `${span} The interval is in the margin of ${shown}.`
 		};
 		if (interval.high < 1) return {
 			...facts,
+			...sides,
 			interval,
 			outcome: "faster",
 			reason: `${span} The interval is below 1.`
 		};
 		if (interval.low > 1) return {
 			...facts,
+			...sides,
 			interval,
 			outcome: "slower",
 			reason: `${span} The interval is above 1.`
 		};
 		return {
 			...facts,
+			...sides,
 			interval,
 			outcome: "cannot decide",
 			reason: `${span} The interval holds 1 and crosses the margin of ${shown}.`
 		};
 	}
 	const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);
-	const span = `The interval of the median ratio is [${round$2(interval.low)}, ${round$2(interval.high)}], and the A/A band is [${round$2(band.low)}, ${round$2(band.high)}].`;
+	const span = `The interval of the median ratio is [${round$3(interval.low)}, ${round$3(interval.high)}], and the A/A band is [${round$3(band.low)}, ${round$3(band.high)}].`;
 	if (interval.high < band.low) return {
 		...facts,
 		interval,
@@ -1073,7 +1109,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
 		reason: `${span} The interval crosses an end of the band.`
 	};
 }
-function round$2(x) {
+function round$3(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
 // src/limit.ts
@@ -1178,20 +1214,358 @@ function limitVerdict(candidate, aa, options) {
 		const rest = others.length === 0 ? "" : ` ${others.length} of ${cases.length} cases are not within${permit}.`;
 		return {
 			outcome: "within",
-			reason: `The largest bound of a case within is ${round$1(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.${rest}`,
+			reason: `The largest bound of a case within is ${round$2(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.${rest}`,
 			...facts
 		};
 	}
 	const top = cases.reduce((a, b) => b.bound > a.bound ? b : a);
 	return {
 		outcome: "cannot decide",
-		reason: `${others.length} of ${cases.length} cases are not within${permit}. The bound of the case ${JSON.stringify(top.case)} is ${round$1(top.bound)}, above the limit ${limit}, and too few floors are above it.`,
+		reason: `${others.length} of ${cases.length} cases are not within${permit}. The bound of the case ${JSON.stringify(top.case)} is ${round$2(top.bound)}, above the limit ${limit}, and too few floors are above it.`,
 		...facts
 	};
 }
-function round$1(x) {
+function round$2(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
+// src/merge.ts
+/**
+* The merge of the records of many jobs, and the pooled verdicts over them. Each job of a bench gives its own records.
+* A pooled verdict uses all jobs together, and the verdict of each job alone guards it.
+*/
+/**
+* This function removes records until the two orders of each case differ in count by 1 or less.
+* It removes the records of the larger order with the highest rounds first. The other records keep their order.
+* A job that stopped early, or a missing job, can leave a case with 2 more rounds of one order. `caseRatios` refuses such a case.
+*/
+function balanceOrders(records) {
+	const byCase = /* @__PURE__ */ new Map();
+	records.forEach((record, index) => {
+		const orders = byCase.get(record.case) ?? {
+			a: [],
+			b: []
+		};
+		orders[record.first === "b" ? "b" : "a"].push(index);
+		byCase.set(record.case, orders);
+	});
+	const drop = /* @__PURE__ */ new Set();
+	const highFirst = (x, y) => records[y].round - records[x].round;
+	for (const { a, b } of byCase.values()) {
+		const [more, fewer] = a.length > b.length ? [a, b] : [b, a];
+		const extra = more.length - fewer.length - 1;
+		if (extra > 0) for (const index of more.toSorted(highFirst).slice(0, extra)) drop.add(index);
+	}
+	return {
+		records: records.filter((_, index) => !drop.has(index)),
+		trimmed: drop.size
+	};
+}
+/**
+* This function gives the pool of the jobs. `pooledVerdict` and `pooledLimitVerdict` use it.
+* - It refuses two jobs with one label, and a record with a different job label. It also refuses two jobs with a record of one case and one round.
+* - It removes each case that a job skipped from all jobs, so all jobs have the same cases.
+* - It merges the records of the jobs, and it balances the merged records and the records of each job with `balanceOrders`.
+*/
+function poolRecords(jobs) {
+	return pool(jobs, "poolRecords");
+}
+/** The pool of `poolRecords`. Each error starts with the name `caller`. */
+function pool(jobs, caller) {
+	const labels = /* @__PURE__ */ new Set();
+	for (const { job } of jobs) {
+		if (labels.has(job)) throw new RangeError(`${caller}: two jobs have the label ${JSON.stringify(job)}.`);
+		labels.add(job);
+	}
+	for (const { job, candidate: c, aa: a } of jobs) for (const r of [...c, ...a]) if (r.job !== void 0 && r.job !== job) throw new RangeError(`${caller}: a record of the job ${JSON.stringify(job)} has the job ${JSON.stringify(r.job)}.`);
+	const dropped = [...new Set(jobs.flatMap((j) => (j.skipped ?? []).map((s) => s.case)))].sort();
+	const gone = new Set(dropped);
+	const keep = (records) => records.filter((r) => !gone.has(r.case));
+	const kept = jobs.map((j) => ({
+		job: j.job,
+		candidate: keep(j.candidate),
+		aa: keep(j.aa)
+	}));
+	uniqueRounds(kept, "candidate", caller);
+	uniqueRounds(kept, "aa", caller);
+	const candidate = balanceOrders(kept.flatMap((j) => j.candidate));
+	const aa = balanceOrders(kept.flatMap((j) => j.aa));
+	const each = kept.map(({ job, candidate: c, aa: a }) => ({
+		job,
+		candidate: balanceOrders(c).records,
+		aa: balanceOrders(a).records
+	}));
+	return {
+		candidate: candidate.records,
+		aa: aa.records,
+		jobs: each,
+		trimmed: candidate.trimmed + aa.trimmed,
+		dropped
+	};
+}
+/** This function refuses two jobs with a record of one case and one round in one pair. */
+function uniqueRounds(jobs, pair, caller) {
+	const seen = /* @__PURE__ */ new Map();
+	for (const { job, [pair]: records } of jobs) for (const r of records) {
+		const key = `${r.case}\u0000${r.round}`;
+		const other = seen.get(key);
+		if (other !== void 0 && other !== job) throw new RangeError(`${caller}: the jobs ${JSON.stringify(other)} and ${JSON.stringify(job)} both have the round ${r.round} of the case ${JSON.stringify(r.case)} in the ${pair} pair. Set roundOffset of each job to job * rounds.`);
+		seen.set(key, job);
+	}
+}
+/**
+* This function applies the guard of the jobs to a pooled verdict. A job whose outcome is `opposite` of the pooled outcome
+* makes the outcome `cannot decide`. Each other job does not change the outcome.
+*/
+function guard(pooled, jobs, opposite) {
+	const against = opposite[pooled.outcome];
+	const opposed = against === void 0 ? [] : jobs.filter((j) => j.verdict.outcome === against).map((j) => j.job);
+	if (opposed.length === 0) return {
+		outcome: pooled.outcome,
+		opposed,
+		reason: pooled.reason
+	};
+	const names = opposed.map((j) => JSON.stringify(j)).join(", ");
+	return {
+		outcome: "cannot decide",
+		opposed,
+		reason: `${pooled.reason} The pooled outcome is "${pooled.outcome}", but the jobs ${names} give "${against}". Thus the outcome is "cannot decide".`
+	};
+}
+const opposite = {
+	faster: "slower",
+	slower: "faster"
+};
+/**
+* This function gives the time verdict over the records of all jobs.
+* - It makes the pool of the jobs with `poolRecords`, and it gives the time verdict of the merged records.
+* - The verdict of each job alone uses the same options. A job with `cannot decide` or `same` does not change the outcome.
+* - A job that decides the opposite direction of the pooled outcome makes the outcome `cannot decide`.
+*/
+function pooledVerdict(jobs, options = {}) {
+	const { candidate, aa, jobs: each, trimmed, dropped } = pool(jobs, "pooledVerdict");
+	const pooled = timeVerdict(candidate, aa, options);
+	const verdicts = each.map(({ job, candidate: c, aa: a }) => ({
+		job,
+		verdict: timeVerdict(c, a, options)
+	}));
+	const { outcome, opposed, reason } = guard(pooled, verdicts, opposite);
+	return {
+		outcome,
+		pooled,
+		jobs: verdicts,
+		opposed,
+		trimmed,
+		dropped,
+		reason
+	};
+}
+const limitOpposite = {
+	within: "beyond",
+	beyond: "within"
+};
+/**
+* This function gives the limit verdict over the records of all jobs, for example for a speed gate on limits over a CI matrix.
+* - It makes the pool of the jobs with `poolRecords`, and it gives the limit verdict of the merged records.
+* - The verdict of each job alone uses the same options. A job with `cannot decide` does not change the outcome.
+* - A job with `beyond` against a pooled `within`, or the reverse, makes the outcome `cannot decide`.
+*/
+function pooledLimitVerdict(jobs, options) {
+	const { candidate, aa, jobs: each, trimmed, dropped } = pool(jobs, "pooledLimitVerdict");
+	const pooled = limitVerdict(candidate, aa, options);
+	const verdicts = each.map(({ job, candidate: c, aa: a }) => ({
+		job,
+		verdict: limitVerdict(c, a, options)
+	}));
+	const { outcome, opposed, reason } = guard(pooled, verdicts, limitOpposite);
+	return {
+		outcome,
+		pooled,
+		jobs: verdicts,
+		opposed,
+		trimmed,
+		dropped,
+		reason
+	};
+}
+/**
+* This function gives the verdict of each candidate pair of a run against its A/A pair `aaPair`.
+* It balances the records of each pair with `balanceOrders` first. It refuses an `aaPair` that the run does not have.
+*/
+function armVerdicts(run, aaPair, options = {}) {
+	const aaRecords = run.pairs[aaPair];
+	if (aaRecords === void 0) throw new RangeError(`armVerdicts: the run has no pair ${JSON.stringify(aaPair)}.`);
+	const aa = balanceOrders(aaRecords).records;
+	const out = {};
+	for (const [pair, records] of Object.entries(run.pairs)) if (pair !== aaPair) out[pair] = timeVerdict(balanceOrders(records).records, aa, options);
+	return out;
+}
+// src/family.ts
+/**
+* The read of a family of cases. A run reads many cases, and each case has its own interval. Without a correction, the
+* chance of one false `slower` case or more grows with the count of cases. At 16 cases and 0.025 for each side, this
+* chance can be 1 - 0.975^16, approximately 0.33. These reads are reports, and they do not change a verdict.
+*
+* - The Holm read (Holm, 1979) keeps the chance of one false claim or more on one side at most `level`.
+* - The read of each job keeps a `slower` or `faster` case only when each job gives that outcome alone.
+* - The fourth read keeps a case only when the Holm read and the read of each job both keep it.
+*/
+const pooledOf = (verdict) => "pooled" in verdict ? verdict : void 0;
+/** This function changes `slower` and `faster` to `cannot decide` when `keep` refuses the side. */
+function kept(outcome, keep) {
+	return (outcome === "slower" || outcome === "faster") && !keep(outcome) ? "cannot decide" : outcome;
+}
+/** This function gives the Holm read of one side, by the index of the case. A case with no p gets undefined. */
+function holmSide(ps, level) {
+	const index = ps.flatMap((p, i) => p === void 0 ? [] : [i]);
+	const corrected = holm(index.map((i) => ps[i]), level);
+	const out = new Array(ps.length).fill(void 0);
+	index.forEach((i, k) => out[i] = corrected[k].adjusted);
+	return out;
+}
+const count = (outcomes) => ({
+	slower: outcomes.filter((o) => o === "slower").length,
+	faster: outcomes.filter((o) => o === "faster").length
+});
+/**
+* This function gives the Holm read and the read of each job over a family of cases, as a report. The default `level` is 0.025 for each side.
+* It reads `pSlower` and `pFaster` of each time verdict, and of the pooled verdict of a pooled case. The option `pValues` of a verdict gives them.
+* A case with no p keeps its outcome in the Holm read, and a line gives the count of such cases.
+*/
+function familyRead(items, level = .025) {
+	if (!(level > 0 && level < 1)) throw new RangeError(`familyRead: the level is more than 0 and less than 1, not ${level}.`);
+	const times = items.map((item) => pooledOf(item.verdict)?.pooled ?? item.verdict);
+	const slower = holmSide(times.map((t) => t.pSlower), level);
+	const faster = holmSide(times.map((t) => t.pFaster), level);
+	const cases = items.map((item, i) => {
+		const outcome = item.verdict.outcome;
+		const adjusted = {
+			slower: slower[i],
+			faster: faster[i]
+		};
+		const pooled = pooledOf(item.verdict);
+		const byHolm = kept(outcome, (side) => adjusted[side] === void 0 || adjusted[side] < level);
+		const byJobs = pooled === void 0 ? outcome : kept(outcome, (side) => pooled.jobs.every((j) => j.verdict.outcome === side));
+		return {
+			case: item.case,
+			outcome,
+			holm: byHolm,
+			eachJob: byJobs,
+			eachJobHolm: byHolm === outcome ? byJobs : byHolm,
+			adjustedSlower: adjusted.slower,
+			adjustedFaster: adjusted.faster
+		};
+	});
+	const finite = times.filter((t) => t.pSlower !== void 0).map((t) => t.interval?.finite ?? 0);
+	const floor = finite.length === 0 ? void 0 : 1 / (Math.min(...finite) + 1);
+	const firstStep = finite.length === 0 ? void 0 : level / finite.length;
+	const plain = count(cases.map((c) => c.outcome));
+	const corrected = count(cases.map((c) => c.holm));
+	const eachJob = count(cases.map((c) => c.eachJob));
+	const eachJobHolm = count(cases.map((c) => c.eachJobHolm));
+	const counts = (side) => `${String(plain[side])}, with Holm ${String(corrected[side])}, with each job ${String(eachJob[side])}, with each job and Holm ${String(eachJobHolm[side])}`;
+	const lines = [`Family read (report only): ${String(cases.length)} cases. Slower: ${counts("slower")}. Faster: ${counts("faster")}.`];
+	const bare = times.filter((t) => t.pSlower === void 0).length;
+	if (bare > 0) lines.push(`${String(bare)} of ${String(times.length)} cases have no p, so the Holm read keeps their outcome. Give the verdicts the option pValues.`);
+	if (floor !== void 0 && firstStep !== void 0) lines.push(floor < firstStep ? `The smallest p of the draws is ${round$1(floor)}, below the first step ${round$1(firstStep)} of Holm at the level ${String(level)}.` : `The smallest p of the draws is ${round$1(floor)}, not below the first step ${round$1(firstStep)} of Holm. No case can read slower with Holm, so give the bootstrap more runs.`);
+	for (const c of cases) {
+		if (c.holm !== c.outcome) lines.push(`The case ${JSON.stringify(c.case)} reads ${c.outcome}, and with Holm it reads ${c.holm}: the adjusted p is ${round$1(c.outcome === "slower" ? c.adjustedSlower : c.adjustedFaster)}.`);
+		if (c.eachJob !== c.outcome) lines.push(`The case ${JSON.stringify(c.case)} reads ${c.outcome}, and a job alone does not read ${c.outcome}.`);
+	}
+	return {
+		level,
+		cases,
+		plain,
+		holm: corrected,
+		eachJob,
+		eachJobHolm,
+		floor,
+		firstStep,
+		lines
+	};
+}
+/**
+* This function draws `trials` families of pooled verdicts and gives the share of each read. Each arm of each round adds a
+* normal noise of `sigma` to the log time. Without `jobShift`, the jobs have no shift between them, so the read of each job has its best case.
+*/
+function familyPower(options) {
+	const { family } = options;
+	const ratio = options.ratio ?? 1;
+	const rounds = options.rounds ?? 5;
+	const jobs = options.jobs ?? 2;
+	const sigma = options.sigma ?? .1;
+	const trials = options.trials ?? 200;
+	const jobShift = options.jobShift ?? 0;
+	for (const [name, n] of [
+		["family", family],
+		["rounds", rounds],
+		["jobs", jobs],
+		["trials", trials]
+	]) if (!Number.isSafeInteger(n) || n < 1) throw new RangeError(`familyPower: ${name} is a whole number of 1 or more, not ${n}.`);
+	if (!(ratio > 0 && Number.isFinite(ratio))) throw new RangeError(`familyPower: the ratio is a finite number above 0, not ${ratio}.`);
+	if (!(jobShift >= 0 && Number.isFinite(jobShift))) throw new RangeError(`familyPower: jobShift is a finite number of 0 or more, not ${jobShift}.`);
+	const r = random(options.seed ?? 1);
+	const draw = (truth, job, c) => Array.from({ length: rounds }, (_, round) => ({
+		case: `${job}:${String(round)}`,
+		round: 0,
+		first: firstArm(round, c),
+		a: 10 * Math.exp(r.normal(0, sigma)),
+		b: 10 * truth * Math.exp(r.normal(0, sigma))
+	}));
+	const hits = {
+		false: {
+			plain: 0,
+			holm: 0,
+			eachJob: 0,
+			eachJobHolm: 0
+		},
+		first: {
+			plain: 0,
+			holm: 0,
+			eachJob: 0,
+			eachJobHolm: 0
+		}
+	};
+	for (let t = 0; t < trials; t++) {
+		const read = familyRead(Array.from({ length: family }, (_, c) => {
+			const records = Array.from({ length: jobs }, (_, j) => ({
+				job: `job ${String(j)}`,
+				candidate: draw((c === 0 ? ratio : 1) * (jobShift > 0 ? Math.exp(r.normal(0, jobShift)) : 1), String(j), c),
+				aa: draw(1, String(j), c)
+			}));
+			return {
+				case: `case ${String(c)}`,
+				verdict: pooledVerdict(records, {
+					...options.verdict,
+					pValues: true
+				})
+			};
+		}), options.level);
+		for (const key of [
+			"plain",
+			"holm",
+			"eachJob",
+			"eachJobHolm"
+		]) {
+			const outcome = (c) => key === "plain" ? c.outcome : c[key];
+			if (read.cases.some((c, i) => (i > 0 || ratio === 1) && outcome(c) === "slower")) hits.false[key]++;
+			if (outcome(read.cases[0]) === "slower") hits.first[key]++;
+		}
+	}
+	const share = (h) => ({
+		plain: h.plain / trials,
+		holm: h.holm / trials,
+		eachJob: h.eachJob / trials,
+		eachJobHolm: h.eachJobHolm / trials
+	});
+	return {
+		trials,
+		falseSlower: share(hits.false),
+		firstSlower: share(hits.first)
+	};
+}
+function round$1(x) {
+	return Math.round(x * 1e5) / 1e5;
+}
 // src/model.ts
 /**
 * The CPU model test. A limit from reads of many runners can hide a model that reads apart. A check of the pool
@@ -1589,177 +1963,6 @@ function tailFalseFails(aa, options) {
 function round(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
-// src/merge.ts
-/**
-* The merge of the records of many jobs, and the pooled verdicts over them. Each job of a bench gives its own records.
-* A pooled verdict uses all jobs together, and the verdict of each job alone guards it.
-*/
-/**
-* This function removes records until the two orders of each case differ in count by 1 or less.
-* It removes the records of the larger order with the highest rounds first. The other records keep their order.
-* A job that stopped early, or a missing job, can leave a case with 2 more rounds of one order. `caseRatios` refuses such a case.
-*/
-function balanceOrders(records) {
-	const byCase = /* @__PURE__ */ new Map();
-	records.forEach((record, index) => {
-		const orders = byCase.get(record.case) ?? {
-			a: [],
-			b: []
-		};
-		orders[record.first === "b" ? "b" : "a"].push(index);
-		byCase.set(record.case, orders);
-	});
-	const drop = /* @__PURE__ */ new Set();
-	const highFirst = (x, y) => records[y].round - records[x].round;
-	for (const { a, b } of byCase.values()) {
-		const [more, fewer] = a.length > b.length ? [a, b] : [b, a];
-		const extra = more.length - fewer.length - 1;
-		if (extra > 0) for (const index of more.toSorted(highFirst).slice(0, extra)) drop.add(index);
-	}
-	return {
-		records: records.filter((_, index) => !drop.has(index)),
-		trimmed: drop.size
-	};
-}
-/**
-* This function gives the pool of the jobs. `pooledVerdict` and `pooledLimitVerdict` use it.
-* - It refuses two jobs with one label, and a record with a different job label. It also refuses two jobs with a record of one case and one round.
-* - It removes each case that a job skipped from all jobs, so all jobs have the same cases.
-* - It merges the records of the jobs, and it balances the merged records and the records of each job with `balanceOrders`.
-*/
-function poolRecords(jobs) {
-	return pool(jobs, "poolRecords");
-}
-/** The pool of `poolRecords`. Each error starts with the name `caller`. */
-function pool(jobs, caller) {
-	const labels = /* @__PURE__ */ new Set();
-	for (const { job } of jobs) {
-		if (labels.has(job)) throw new RangeError(`${caller}: two jobs have the label ${JSON.stringify(job)}.`);
-		labels.add(job);
-	}
-	for (const { job, candidate: c, aa: a } of jobs) for (const r of [...c, ...a]) if (r.job !== void 0 && r.job !== job) throw new RangeError(`${caller}: a record of the job ${JSON.stringify(job)} has the job ${JSON.stringify(r.job)}.`);
-	const dropped = [...new Set(jobs.flatMap((j) => (j.skipped ?? []).map((s) => s.case)))].sort();
-	const gone = new Set(dropped);
-	const keep = (records) => records.filter((r) => !gone.has(r.case));
-	const kept = jobs.map((j) => ({
-		job: j.job,
-		candidate: keep(j.candidate),
-		aa: keep(j.aa)
-	}));
-	uniqueRounds(kept, "candidate", caller);
-	uniqueRounds(kept, "aa", caller);
-	const candidate = balanceOrders(kept.flatMap((j) => j.candidate));
-	const aa = balanceOrders(kept.flatMap((j) => j.aa));
-	const each = kept.map(({ job, candidate: c, aa: a }) => ({
-		job,
-		candidate: balanceOrders(c).records,
-		aa: balanceOrders(a).records
-	}));
-	return {
-		candidate: candidate.records,
-		aa: aa.records,
-		jobs: each,
-		trimmed: candidate.trimmed + aa.trimmed,
-		dropped
-	};
-}
-/** This function refuses two jobs with a record of one case and one round in one pair. */
-function uniqueRounds(jobs, pair, caller) {
-	const seen = /* @__PURE__ */ new Map();
-	for (const { job, [pair]: records } of jobs) for (const r of records) {
-		const key = `${r.case}\u0000${r.round}`;
-		const other = seen.get(key);
-		if (other !== void 0 && other !== job) throw new RangeError(`${caller}: the jobs ${JSON.stringify(other)} and ${JSON.stringify(job)} both have the round ${r.round} of the case ${JSON.stringify(r.case)} in the ${pair} pair. Set roundOffset of each job to job * rounds.`);
-		seen.set(key, job);
-	}
-}
-/**
-* This function applies the guard of the jobs to a pooled verdict. A job whose outcome is `opposite` of the pooled outcome
-* makes the outcome `cannot decide`. Each other job does not change the outcome.
-*/
-function guard(pooled, jobs, opposite) {
-	const against = opposite[pooled.outcome];
-	const opposed = against === void 0 ? [] : jobs.filter((j) => j.verdict.outcome === against).map((j) => j.job);
-	if (opposed.length === 0) return {
-		outcome: pooled.outcome,
-		opposed,
-		reason: pooled.reason
-	};
-	const names = opposed.map((j) => JSON.stringify(j)).join(", ");
-	return {
-		outcome: "cannot decide",
-		opposed,
-		reason: `${pooled.reason} The pooled outcome is "${pooled.outcome}", but the jobs ${names} give "${against}". Thus the outcome is "cannot decide".`
-	};
-}
-const opposite = {
-	faster: "slower",
-	slower: "faster"
-};
-/**
-* This function gives the time verdict over the records of all jobs.
-* - It makes the pool of the jobs with `poolRecords`, and it gives the time verdict of the merged records.
-* - The verdict of each job alone uses the same options. A job with `cannot decide` or `same` does not change the outcome.
-* - A job that decides the opposite direction of the pooled outcome makes the outcome `cannot decide`.
-*/
-function pooledVerdict(jobs, options = {}) {
-	const { candidate, aa, jobs: each, trimmed, dropped } = pool(jobs, "pooledVerdict");
-	const pooled = timeVerdict(candidate, aa, options);
-	const verdicts = each.map(({ job, candidate: c, aa: a }) => ({
-		job,
-		verdict: timeVerdict(c, a, options)
-	}));
-	const { outcome, opposed, reason } = guard(pooled, verdicts, opposite);
-	return {
-		outcome,
-		pooled,
-		jobs: verdicts,
-		opposed,
-		trimmed,
-		dropped,
-		reason
-	};
-}
-const limitOpposite = {
-	within: "beyond",
-	beyond: "within"
-};
-/**
-* This function gives the limit verdict over the records of all jobs, for example for a speed gate on limits over a CI matrix.
-* - It makes the pool of the jobs with `poolRecords`, and it gives the limit verdict of the merged records.
-* - The verdict of each job alone uses the same options. A job with `cannot decide` does not change the outcome.
-* - A job with `beyond` against a pooled `within`, or the reverse, makes the outcome `cannot decide`.
-*/
-function pooledLimitVerdict(jobs, options) {
-	const { candidate, aa, jobs: each, trimmed, dropped } = pool(jobs, "pooledLimitVerdict");
-	const pooled = limitVerdict(candidate, aa, options);
-	const verdicts = each.map(({ job, candidate: c, aa: a }) => ({
-		job,
-		verdict: limitVerdict(c, a, options)
-	}));
-	const { outcome, opposed, reason } = guard(pooled, verdicts, limitOpposite);
-	return {
-		outcome,
-		pooled,
-		jobs: verdicts,
-		opposed,
-		trimmed,
-		dropped,
-		reason
-	};
-}
-/**
-* This function gives the verdict of each candidate pair of a run against its A/A pair `aaPair`.
-* It balances the records of each pair with `balanceOrders` first. It refuses an `aaPair` that the run does not have.
-*/
-function armVerdicts(run, aaPair, options = {}) {
-	const aaRecords = run.pairs[aaPair];
-	if (aaRecords === void 0) throw new RangeError(`armVerdicts: the run has no pair ${JSON.stringify(aaPair)}.`);
-	const aa = balanceOrders(aaRecords).records;
-	const out = {};
-	for (const [pair, records] of Object.entries(run.pairs)) if (pair !== aaPair) out[pair] = timeVerdict(balanceOrders(records).records, aa, options);
-	return out;
-}
 // src/measures.ts
 /**
 * The check that a trick does not buy a measure. A control is a strategy made to win one measure and nothing else.
@@ -2542,6 +2745,6 @@ function timeRounds(arms, options) {
 		sink
 	};
 }
-export { BUDGET_LOOKS, CEILING_ONLY, GATE_LOOKS, LADDER_FORMAT, NARROW_SPREAD_SHARE, UNKNOWN_MODEL, aaBand, aaSpread, armModels, armOrder, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, caseTails, checkLadder, checkMeasures, checkPairedTime, controlFactor, firstArm, gateOf, growthVerdict, jointClaim, ladderVerdict, limitOnModel, limitVerdict, modelRowText, modelTest, notSlowerVerdict, orderSize, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, resultRows, resultVerdict, rungVerdict, tailFalseFails, tailLimitVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower, visibleShift, winClaim };
+export { BUDGET_LOOKS, CEILING_ONLY, GATE_LOOKS, LADDER_FORMAT, NARROW_SPREAD_SHARE, UNKNOWN_MODEL, aaBand, aaSpread, armModels, armOrder, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, caseTails, checkLadder, checkMeasures, checkPairedTime, controlFactor, familyPower, familyRead, firstArm, gateOf, growthVerdict, jointClaim, ladderVerdict, limitOnModel, limitVerdict, modelRowText, modelTest, notSlowerVerdict, orderSize, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, resultRows, resultVerdict, rungVerdict, tailFalseFails, tailLimitVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower, visibleShift, winClaim };
 
 //# sourceMappingURL=index.js.map