@mark1russell7/rigor-bench 0.1.0 → 0.1.2

package/README.md

@@ -37,8 +37,72 @@ v.comparison;                                     // the cases that gain and dro
 | `caseRatios(records)` | The ratio of each case. It refuses 2 records of one round, and an order that did not turn. |
 | `aaBand(aa, options)` | The band of the A/A arm, symmetric on a log scale. The kind `median` (the default) holds the A/A median, and the kind `case` holds one A/A case. It needs 5 cases or more. |
 | `timeVerdict(candidate, aa, options)` | The outcome of the rule `difference` (the default) or `band`. |
+| `aaSpread(aa, level)` | The A/A spread: the `level` quantile of `abs(log(ratio))` over the A/A cases. The default level is 0.9. |
+| `pooledBand(pool, level)` | The pooled band: the larger of the floor and the A/A spread of the records of earlier runs. |
+| `caseIntervals(records, level)` | The ratio of each case with a distribution-free interval of its median over the rounds. The level 0.95 needs 6 rounds or more. |
+| `limitVerdict(candidate, aa, options)` | The verdict of the claim "the ratio of each case is at most `limit`": `within`, `beyond` or `cannot decide`. |
+| `timeRounds(arms, options)` | The timed rounds of one case in one process: a warm-up, a batch of at least the target time, and the order of `firstArm`. |
 | `verdictPower(options)` | The share of each outcome over seeded benches with a known ratio: the power test of a rule. |
 
+## A claim with a limit
+
+`timeVerdict` compares the ratio with 1. A claim with a limit needs `limitVerdict`. An example is "the new code takes at most 2 times the time of the old code":
+
+```ts
+const v = limitVerdict(records, aaRecords, { limit: 2 });
+
+v.outcome;                                        // "within", "beyond" or "cannot decide"
+v.cases;                                          // the interval, the bound and the outcome of each case
+```
+
+- The A/A band widens the interval of each case. The bound is the high end of the interval times the high end of the band.
+- A case is `within` when its bound is at most the limit. It is `beyond` when its low end times the low end of the band is above the limit.
+- The verdict is `beyond` when a case is `beyond`, and `within` when each case is `within`.
+
+The verdict has no correction for many cases. Each case has its own interval at the level `caseLevel`. Thus with many cases, a verdict `beyond` can come from noise only. Set `caseLevel` higher when a bench has many cases. A correction of Holm is an option for a later version.
+
+## Time the rounds
+
+`timeRounds` times one case in one process. Start a fresh process for each case and each pair, and give the index of the case as `offset`:
+
+```ts
+const { records } = timeRounds({ a: () => oldCode(input), b: () => newCode(input) }, { case: "board-1", rounds: 31, offset: 0 });
+```
+
+Each arm gives a number. The engine cannot remove the work of a call, because the sum of these numbers uses it.
+
+## The noise cap and the pooled band
+
+Two options of `timeVerdict` keep a noisy run from a decision. Both are off by default, so the output stays the same as in 0.1.0.
+
+The option `noiseCap` caps the A/A spread of the run. The spread is the quantile `spreadLevel` (default 0.9) of `abs(log(ratio))` over the A/A cases. Above the cap, the outcome is `cannot decide`:
+
+```ts
+const v = timeVerdict(records, aaRecords, { noiseCap: 0.06 });
+
+v.noise;                                          // { spread, cap, noisy }
+v.reason;                                         // the spread, the cap and the outcome without the cap
+```
+
+The option `pool` gives a band from the A/A records of earlier quiet runs, with a floor. The caller keeps the pool, for example in a file of the repository:
+
+```ts
+const pool = { records: quietAaRecords, floor: Math.log(1.05) };   // each run has its own case names
+const v = timeVerdict(records, aaRecords, { noiseCap: 0.06, pool });
+
+v.band;                                           // the pooled band: its noise is at least the floor
+v.pool;                                           // { cases, runs, spread, floor, pooled }
+```
+
+- The noise of the pooled band is the larger of the floor and the spread of the pooled cases. The floor and the noise are on a log scale.
+- With fewer pooled cases than `minimum` (default 5), the band uses the floor alone.
+- The option `run` gives the run of each record, so `v.pool.runs` counts the runs. The cases of one run share one runner, so they are not independent.
+- The pooled band replaces the band of the run in the rule `band` and in `comparison`.
+- The rule `difference` uses the larger of `margin` and the noise of the pooled band as its margin.
+- The run still needs its own A/A arm of 5 cases or more. The noise cap uses that arm.
+
+For a fixed limit, for example 1.10 against a fixed base, use `limitVerdict`.
+
 ## The outcomes
 
 | Outcome | The rule `difference` (the default) | The rule `band` |
@@ -48,7 +112,7 @@ v.comparison;                                     // the cases that gain and dro
 | `same` | The interval is in `[exp(-margin), exp(margin)]`. The default margin is 0.05. | The interval is in the band. |
 | `cannot decide` | The interval holds 1 and crosses the margin. | The interval crosses an end of the band. |
 
-Both rules give `cannot decide` when the A/A arm has fewer than 5 cases, or when the candidate has no cases.
+Both rules give `cannot decide` when the A/A arm has fewer than 5 cases, or when the candidate has no cases. With `noiseCap`, both rules also give `cannot decide` when the A/A spread of the run is above the cap.
 
 ## The power test
 

package/build/index.d.ts

@@ -69,6 +69,44 @@ interface Band {
 }
 /** This function gives the band of the A/A arm. The band is symmetric on a log scale. It refuses fewer cases than `minimum`. */
 export declare function aaBand(aa: readonly PairedTime[], options?: BandOptions): Band;
+/**
+ * This function gives the A/A spread: the `level` quantile of `abs(log(ratio))` over the A/A cases. The default level is 0.9.
+ * It refuses an A/A arm with no cases.
+ */
+export declare function aaSpread(aa: readonly PairedTime[], level?: number): number;
+/** The options of a pooled band: the A/A records of earlier quiet runs, and a floor. The caller keeps the pool. */
+interface PoolOptions {
+  /** The A/A records of the earlier runs. Give each run its own case names, for example with the run id as a prefix. */
+  readonly records: readonly PairedTime[];
+  /** The smallest noise of the band, on a log scale. The default is 0. A floor of `Math.log(1.05)` gives at least the band `[1 / 1.05, 1.05]`. */
+  readonly floor?: number;
+  /** The fewest pooled cases that give a spread. With fewer cases, the noise of the band is the floor. The default is 5. */
+  readonly minimum?: number;
+  /**
+   * The run of a record, for example the id of its CI run. The verdict counts the runs of the pool with it.
+   * The cases of one run share one runner, so they are not independent. Many cases from few runs show the noise of few runners.
+   */
+  readonly run?: (record: PairedTime) => string;
+}
+/** The facts of the pooled band of a verdict. */
+interface PoolFacts {
+  /** The count of the pooled A/A cases. */
+  readonly cases: number;
+  /** The count of the runs of the pool, from the option `run`. It is undefined without that option. */
+  readonly runs: number | undefined;
+  /** The A/A spread of the pooled cases, or undefined when the pool has fewer cases than `minimum`. */
+  readonly spread: number | undefined;
+  readonly floor: number;
+  /** This value is `true` when the spread of the pool is larger than the floor, so the spread gives the noise of the band. */
+  readonly pooled: boolean;
+}
+/**
+ * This function gives the pooled band of the kind `case`. Its noise is the larger of the floor and the A/A spread of the pooled records.
+ * The spread is the `level` quantile of `abs(log(ratio))`, with the default level 0.9.
+ *
+ * A band with fewer pooled cases than `minimum` uses the floor alone. A band with no pooled cases has the median 1.
+ */
+export declare function pooledBand(pool: PoolOptions, level?: number): Band;
 /** The four outcomes of a time verdict. The candidate is the arm `b`, so `faster` is a ratio below 1. */
 type TimeOutcome = "faster" | "slower" | "same" | "cannot decide";
 /** The options of a time verdict. The bootstrap gives the interval of the median ratio of the candidate, and the band `median`. */
@@ -84,6 +122,26 @@ interface VerdictOptions extends BandOptions {
   readonly rule?: VerdictRule;
   /** The half width on a log scale of the ratios that the rule `difference` calls `same`. The default is 0.05. */
   readonly margin?: number;
+  /**
+   * The noise cap on the A/A spread of the run, on a log scale. Above the cap, the outcome is `cannot decide`.
+   * The default is no cap.
+   */
+  readonly noiseCap?: number;
+  /** The quantile of the A/A spread of the noise cap and of the pooled band. The default is 0.9. */
+  readonly spreadLevel?: number;
+  /**
+   * The pooled band. It replaces the band of the A/A arm of the run in the rule `band` and in `comparison`.
+   * The rule `difference` uses the larger of `margin` and the noise of the pooled band as its margin. The default is no pool.
+   */
+  readonly pool?: PoolOptions;
+}
+/** The noise cap of a verdict: the A/A spread of the run and the cap. */
+interface NoiseCheck {
+  /** The A/A spread of the run, or undefined when the A/A arm has no cases. */
+  readonly spread: number | undefined;
+  readonly cap: number;
+  /** This value is `true` when the spread is above the cap, or when the A/A arm has no cases. */
+  readonly noisy: boolean;
 }
 /** A time verdict, with the facts that it uses. */
 interface TimeVerdict {
@@ -106,6 +164,10 @@ interface TimeVerdict {
    * The comparison uses the value `1 + log(ratio)` of each case and the noise of the band, so its two sides are the ends of the band.
    */
   readonly comparison: Comparison | undefined;
+  /** The facts of the pooled band. The verdict has this field only with the option `pool`. */
+  readonly pool?: PoolFacts;
+  /** The noise cap. The verdict has this field only with the option `noiseCap`. */
+  readonly noise?: NoiseCheck;
 }
 /**
  * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.
@@ -113,8 +175,58 @@ interface TimeVerdict {
  *   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.
  * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.
  * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.
+ * - With `noiseCap`, an A/A spread of the run above the cap gives `cannot decide`.
+ * - With `pool`, the pooled band replaces the band of the A/A arm of the run. The run still needs `minimum` A/A cases.
  */
 export declare function timeVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options?: VerdictOptions): TimeVerdict;
+/** The ratio of one case with the interval of its median over the rounds. */
+interface CaseInterval extends CaseRatio {
+  /** The ends of the interval: `exp` of `medianInterval` over the values `log(b / a)` of the rounds. */
+  readonly low: number;
+  readonly high: number;
+  /** The coverage of the interval. It is at least the level. */
+  readonly coverage: number;
+}
+/**
+ * This function gives the ratio of each case with a distribution-free interval of its median, in the order of
+ * `caseRatios`. It needs enough rounds for the level: 6 rounds or more for the level 0.95.
+ */
+export declare function caseIntervals(records: readonly PairedTime[], level?: number): CaseInterval[];
+/** The three outcomes of a limit verdict. */
+type LimitOutcome = "within" | "beyond" | "cannot decide";
+/** The options of a limit verdict. */
+interface LimitOptions extends BandOptions {
+  /** The largest ratio `b / a` that the claim permits, for example 2. */
+  readonly limit: number;
+  /** The level of the interval of each case. The default is 0.95. */
+  readonly caseLevel?: number;
+}
+/** The verdict of one case against the limit. */
+interface CaseLimit extends CaseInterval {
+  /** The high end of the interval of the case, times the high end of the A/A band. */
+  readonly bound: number;
+  /** The low end of the interval of the case, times the low end of the A/A band. */
+  readonly floor: number;
+  readonly outcome: LimitOutcome;
+}
+/** A limit verdict, with the facts that it uses. */
+interface LimitVerdict {
+  readonly outcome: LimitOutcome;
+  /** The reason of the outcome, in one sentence. */
+  readonly reason: string;
+  readonly limit: number;
+  readonly cases: readonly CaseLimit[];
+  /** The band of the A/A arm, or undefined when the A/A arm has too few cases. */
+  readonly band: Band | undefined;
+}
+/**
+ * This function gives the verdict of the claim "the ratio `b / a` of each case is at most `limit`". The A/A band widens
+ * the interval of each case: the bound is the high end times the high end of the band.
+ * - A case is `within` when its bound is at most the limit, and `beyond` when its floor is above the limit.
+ * - The verdict is `beyond` when a case is `beyond`, and `within` when each case is `within`.
+ * - Each other verdict is `cannot decide`, and so is an A/A arm with too few cases or a candidate with no cases.
+ */
+export declare function limitVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: LimitOptions): LimitVerdict;
 /** The options of a power test. */
 interface PowerOptions {
   /** The count of the cases of the candidate and of the A/A arm. */
@@ -144,5 +256,34 @@ interface Power {
  * Each arm of each round adds a normal noise of `sigma` to the log time.
  */
 export declare function verdictPower(options: PowerOptions): Power;
-export type { Arm, Band, BandKind, BandOptions, CaseRatio, PairedTime, Power, PowerOptions, TimeOutcome, TimeVerdict, VerdictOptions, VerdictRule };
+/** The options of the timed rounds of one case. */
+interface RoundsOptions {
+  /** The name of the case. */
+  readonly case: string;
+  /** The count of the rounds. */
+  readonly rounds: number;
+  /** The offset of `firstArm`: give the index of the case, so half of the cases start with each arm. The default is 0. */
+  readonly offset?: number;
+  /** The least time in milliseconds of one batch of arm `a`. The default is 10. */
+  readonly targetMs?: number;
+  /** The time in milliseconds of the warm-up of the two arms. The default is 1000. */
+  readonly warmMs?: number;
+  /** The clock in milliseconds. The default is `performance.now`. */
+  readonly now?: () => number;
+}
+/** The result of the timed rounds of one case. */
+interface Rounds {
+  readonly records: PairedTime[];
+  /** The count of calls of each arm in one batch. */
+  readonly count: number;
+  /** The sum of the numbers that the arms gave. The engine cannot remove the work of a call, because this sum uses it. */
+  readonly sink: number;
+}
+/**
+ * This function times the arms `a` and `b` of one case, and gives one `PairedTime` record for each round. The time of
+ * a record is the time of one batch, in milliseconds. Each call of an arm gives a number, and the sum of these numbers
+ * keeps the work.
+ */
+export declare function timeRounds(arms: Readonly<Record<Arm, () => number>>, options: RoundsOptions): Rounds;
+export type { Arm, Band, BandKind, BandOptions, CaseInterval, CaseLimit, CaseRatio, LimitOptions, LimitOutcome, LimitVerdict, NoiseCheck, PairedTime, PoolFacts, PoolOptions, Power, PowerOptions, Rounds, RoundsOptions, TimeOutcome, TimeVerdict, VerdictOptions, VerdictRule };
 //# sourceMappingURL=index.d.ts.map

package/build/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/paired.ts","../src/verdict.ts","../src/power.ts"],"mappings":";;KAOY;;UAGK;;WAEN;;WAEA;WACA,OAAO;;WAEP;;WAEA;;;;;;wBAOK,SAAS,eAAe,kBAAa;;wBAOrC,gBAAgB,QAAQ,aAAa;;UAapC;WACN;;WAEA;WACA;;;;;;;wBAQK,WAAW,kBAAkB,eAAe;;;;;;KC7ChD;;UAGK;;WAEN,OAAO;;WAEP;;WAEA;;WAEA,YAAY;;;UAIN;;WAEN;WACA,MAAM;;;;;;WAMN;;WAEA;WACA;;WAEA;;;wBAIK,OAAO,aAAa,cAAc,UAAS,cAAmB;;KAiBlE;;;;;;;KAQA;;UAGK,uBAAuB;;WAE7B,OAAO;;WAEP;;;UAIM;WACN,SAAS;;WAET;WACA,gBAAgB;WAChB,MAAM;;;;;;;WAON,UAAU;;WAEV,MAAM;;;;;WAKN,YAAY;;;;;;;;;wBAUP,YAAY,oBAAoB,cAAc,aAAa,cAAc,UAAS,iBAAsB;;UCtGvG;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,UAAU;;;UAIJ;WACN;WACA,UAAU,SAAS,OAAO;;WAE1B;;;;;;wBAOK,aAAa,SAAS,eAAe"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/paired.ts","../src/verdict.ts","../src/limit.ts","../src/power.ts","../src/rounds.ts"],"mappings":";;KAOY;;UAGK;;WAEN;;WAEA;WACA,OAAO;;WAEP;;WAEA;;;;;;wBAOK,SAAS,eAAe,kBAAa;;wBAOrC,gBAAgB,QAAQ,aAAa;;UAapC;WACN;;WAEA;WACA;;;;;;;wBAQK,WAAW,kBAAkB,eAAe;;;;;;KC7ChD;;UAGK;;WAEN,OAAO;;WAEP;;WAEA;;WAEA,YAAY;;;UAIN;;WAEN;WACA,MAAM;;;;;;WAMN;;WAEA;WACA;;WAEA;;;wBAIK,OAAO,aAAa,cAAc,UAAS,cAAmB;;;;;wBAoB9D,SAAS,aAAa,cAAc;;UAWnC;;WAEN,kBAAkB;;WAElB;;WAEA;;;;;WAKA,OAAO,QAAQ;;;UAIT;;WAEN;;WAEA;;WAEA;WACA;;WAEA;;;;;;;;wBASK,WAAW,MAAM,aAAa,iBAAc;;KAmBhD;;;;;;;KAQA;;UAGK,uBAAuB;;WAE7B,OAAO;;WAEP;;;;;WAKA;;WAEA;;;;;WAKA,OAAO;;;UAID;;WAEN;WACA;;WAEA;;;UAIM;WACN,SAAS;;WAET;WACA,gBAAgB;WAChB,MAAM;;;;;;;WAON,UAAU;;WAEV,MAAM;;;;;WAKN,YAAY;;WAEZ,OAAO;;WAEP,QAAQ;;;;;;;;;;;wBAYH,YAAY,oBAAoB,cAAc,aAAa,cAAc,UAAS,iBAAsB;;UCnMvG,qBAAqB;;WAE3B;WACA;;WAEA;;;;;;wBAOK,cAAc,kBAAkB,cAAc,iBAAe;;KAcjE;;UAGK,qBAAqB;;WAE3B;;WAEA;;;UAIM,kBAAkB;;WAExB;;WAEA;WACA,SAAS;;;UAIH;WACN,SAAS;;WAET;WACA;WACA,gBAAgB;;WAEhB,MAAM;;;;;;;;;wBAUD,aAAa,oBAAoB,cAAc,aAAa,cAAc,SAAS,eAAe;;UC/DjG;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,UAAU;;;UAIJ;WACN;WACA,UAAU,SAAS,OAAO;;WAE1B;;;;;;wBAOK,aAAa,SAAS,eAAe;;UC7BpC;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN,SAAS;;WAET;;WAEA;;;;;;;wBAQK,WAAW,MAAM,SAAS,OAAO,qBAAqB,SAAS,gBAAgB"}

package/build/index.js

@@ -1,5 +1,6 @@
-import { bootstrapMedian, bootstrapTwo, compareRatios, median, quantile } from "@mark1russell7/rigor-statistics";
+import { bootstrapMedian, bootstrapTwo, compareRatios, median, medianInterval, quantile } from "@mark1russell7/rigor-statistics";
 import { random } from "@mark1russell7/rigor-generators";
+import { performance } from "node:perf_hooks";
 // src/paired.ts
 /**
 * The paired-time record of a bench. Each round runs the two arms of a pair on one case, one after the other, in a
@@ -83,13 +84,99 @@ function aaBand(aa, options = {}) {
 	};
 }
 /**
+* This function gives the A/A spread: the `level` quantile of `abs(log(ratio))` over the A/A cases. The default level is 0.9.
+* It refuses an A/A arm with no cases.
+*/
+function aaSpread(aa, level = .9) {
+	const ratios = caseRatios(aa);
+	if (ratios.length === 0) throw new RangeError("aaSpread: the A/A arm has no cases.");
+	return spreadOf(ratios, level);
+}
+function spreadOf(ratios, level) {
+	return quantile(ratios.map((r) => Math.abs(Math.log(r.ratio))), level);
+}
+/**
+* This function gives the pooled band of the kind `case`. Its noise is the larger of the floor and the A/A spread of the pooled records.
+* The spread is the `level` quantile of `abs(log(ratio))`, with the default level 0.9.
+*
+* A band with fewer pooled cases than `minimum` uses the floor alone. A band with no pooled cases has the median 1.
+*/
+function pooledBand(pool, level = .9) {
+	return poolOf(pool, level).band;
+}
+function poolOf(pool, level) {
+	const floor = pool.floor ?? 0;
+	const minimum = pool.minimum ?? 5;
+	if (!(floor >= 0 && Number.isFinite(floor))) throw new RangeError(`pooledBand: the floor is a finite number of 0 or more, not ${floor}.`);
+	if (!Number.isSafeInteger(minimum) || minimum < 1) throw new RangeError(`pooledBand: the minimum is a whole number of 1 or more, not ${minimum}.`);
+	const ratios = caseRatios(pool.records);
+	const spread = ratios.length >= minimum ? spreadOf(ratios, level) : void 0;
+	const noise = Math.max(floor, spread ?? 0);
+	const median = ratios.length === 0 ? 1 : Math.exp(quantile(ratios.map((r) => Math.log(r.ratio)), .5));
+	const band = {
+		cases: ratios.length,
+		kind: "case",
+		noise,
+		low: Math.exp(-noise),
+		high: Math.exp(noise),
+		median
+	};
+	const runs = pool.run === void 0 ? void 0 : new Set(pool.records.map(pool.run)).size;
+	return {
+		band,
+		facts: {
+			cases: ratios.length,
+			runs,
+			spread,
+			floor,
+			pooled: spread !== void 0 && spread > floor
+		}
+	};
+}
+/**
 * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.
 * - The rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.
 *   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.
 * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.
 * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.
+* - With `noiseCap`, an A/A spread of the run above the cap gives `cannot decide`.
+* - With `pool`, the pooled band replaces the band of the A/A arm of the run. The run still needs `minimum` A/A cases.
 */
 function timeVerdict(candidate, aa, options = {}) {
+	const level = options.spreadLevel ?? .9;
+	if (options.noiseCap !== void 0 && !(options.noiseCap >= 0 && Number.isFinite(options.noiseCap))) throw new RangeError(`timeVerdict: the noise cap is a finite number of 0 or more, not ${options.noiseCap}.`);
+	const pool = options.pool === void 0 ? void 0 : poolOf(options.pool, level);
+	let verdict = ruleVerdict(candidate, aa, options, pool?.band);
+	if (pool !== void 0) verdict = {
+		...verdict,
+		pool: pool.facts
+	};
+	if (options.noiseCap !== void 0) {
+		const cap = options.noiseCap;
+		const aaRatios = caseRatios(aa);
+		const spread = aaRatios.length === 0 ? void 0 : spreadOf(aaRatios, level);
+		const noise = {
+			spread,
+			cap,
+			noisy: spread === void 0 || spread > cap
+		};
+		verdict = {
+			...verdict,
+			noise
+		};
+		if (noise.noisy && verdict.band !== void 0) {
+			const reason = `${spread === void 0 ? "The A/A arm of the run has no cases, so the noise cap has no spread." : `The A/A spread of the run is ${round$1(spread)}, above the noise cap ${cap}.`} Without the cap, the outcome is "${verdict.outcome}".`;
+			verdict = {
+				...verdict,
+				outcome: "cannot decide",
+				reason
+			};
+		}
+	}
+	return verdict;
+}
+/** This function gives the outcome of the rule. A pooled band replaces the band of the A/A arm, and it widens the margin of the rule `difference`. */
+function ruleVerdict(candidate, aa, options, pooled) {
 	const rule = options.rule ?? "difference";
 	const cases = caseRatios(candidate);
 	const aaRatios = caseRatios(aa);
@@ -115,7 +202,7 @@ function timeVerdict(candidate, aa, options = {}) {
 			reason: `The A/A arm has ${aaRatios.length} cases, and a band needs ${minimum} or more.`
 		};
 	}
-	const band = aaBand(aa, options);
+	const band = pooled ?? aaBand(aa, options);
 	const facts = {
 		cases,
 		rule,
@@ -126,8 +213,10 @@ function timeVerdict(candidate, aa, options = {}) {
 		})
 	};
 	if (rule === "difference") {
-		const margin = options.margin ?? .05;
-		if (!(margin >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${margin}.`);
+		const given = options.margin ?? .05;
+		if (!(given >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${given}.`);
+		const margin = pooled === void 0 ? given : Math.max(given, pooled.noise);
+		const shown = pooled === void 0 ? margin : round$1(margin);
 		const logs = cases.map((c) => Math.log(c.ratio));
 		const aaLogs = aaRatios.map((c) => Math.log(c.ratio));
 		const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.bootstrap);
@@ -137,7 +226,7 @@ function timeVerdict(candidate, aa, options = {}) {
 			low: Math.exp(d.low),
 			high: Math.exp(d.high)
 		};
-		const span = `The interval of the candidate median over the A/A median is [${round(interval.low)}, ${round(interval.high)}].`;
+		const span = `The interval of the candidate median over the A/A median is [${round$1(interval.low)}, ${round$1(interval.high)}].`;
 		if (interval.high < 1) return {
 			...facts,
 			interval,
@@ -154,17 +243,17 @@ function timeVerdict(candidate, aa, options = {}) {
 			...facts,
 			interval,
 			outcome: "same",
-			reason: `${span} The interval is in the margin of ${margin}.`
+			reason: `${span} The interval is in the margin of ${shown}.`
 		};
 		return {
 			...facts,
 			interval,
 			outcome: "cannot decide",
-			reason: `${span} The interval holds 1 and crosses the margin of ${margin}.`
+			reason: `${span} The interval holds 1 and crosses the margin of ${shown}.`
 		};
 	}
 	const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);
-	const span = `The interval of the median ratio is [${round(interval.low)}, ${round(interval.high)}], and the A/A band is [${round(band.low)}, ${round(band.high)}].`;
+	const span = `The interval of the median ratio is [${round$1(interval.low)}, ${round$1(interval.high)}], and the A/A band is [${round$1(band.low)}, ${round$1(band.high)}].`;
 	if (interval.high < band.low) return {
 		...facts,
 		interval,
@@ -190,6 +279,106 @@ function timeVerdict(candidate, aa, options = {}) {
 		reason: `${span} The interval crosses an end of the band.`
 	};
 }
+function round$1(x) {
+	return Math.round(x * 1e4) / 1e4;
+}
+// src/limit.ts
+/**
+* The limit verdict of a bench: is the ratio of each case at most a limit? For example, the claim "the new code takes at most
+* 2 times the time of the old code" needs it. The time verdict compares only with 1.
+*/
+/**
+* This function gives the ratio of each case with a distribution-free interval of its median, in the order of
+* `caseRatios`. It needs enough rounds for the level: 6 rounds or more for the level 0.95.
+*/
+function caseIntervals(records, level = .95) {
+	const logs = /* @__PURE__ */ new Map();
+	for (const r of records) {
+		const list = logs.get(r.case) ?? [];
+		list.push(Math.log(r.b / r.a));
+		logs.set(r.case, list);
+	}
+	return caseRatios(records).map((c) => {
+		const interval = medianInterval(logs.get(c.case), level);
+		return {
+			...c,
+			low: Math.exp(interval.low),
+			high: Math.exp(interval.high),
+			coverage: interval.coverage
+		};
+	});
+}
+/**
+* This function gives the verdict of the claim "the ratio `b / a` of each case is at most `limit`". The A/A band widens
+* the interval of each case: the bound is the high end times the high end of the band.
+* - A case is `within` when its bound is at most the limit, and `beyond` when its floor is above the limit.
+* - The verdict is `beyond` when a case is `beyond`, and `within` when each case is `within`.
+* - Each other verdict is `cannot decide`, and so is an A/A arm with too few cases or a candidate with no cases.
+*/
+function limitVerdict(candidate, aa, options) {
+	const { limit } = options;
+	if (!(limit > 0 && Number.isFinite(limit))) throw new RangeError(`limitVerdict: the limit is a finite number above 0, not ${limit}.`);
+	const intervals = caseIntervals(candidate, options.caseLevel);
+	const minimum = options.minimum ?? 5;
+	if (intervals.length === 0) return {
+		outcome: "cannot decide",
+		reason: "The candidate has no cases.",
+		limit,
+		cases: [],
+		band: void 0
+	};
+	const aaCount = caseRatios(aa).length;
+	if (aaCount < minimum) {
+		const cases = intervals.map((c) => ({
+			...c,
+			bound: c.high,
+			floor: c.low,
+			outcome: "cannot decide"
+		}));
+		return {
+			outcome: "cannot decide",
+			reason: `The A/A arm has ${aaCount} cases, and a band needs ${minimum} or more.`,
+			limit,
+			cases,
+			band: void 0
+		};
+	}
+	const band = aaBand(aa, options);
+	const cases = intervals.map((c) => {
+		const bound = c.high * band.high;
+		const floor = c.low * band.low;
+		const outcome = bound <= limit ? "within" : floor > limit ? "beyond" : "cannot decide";
+		return {
+			...c,
+			bound,
+			floor,
+			outcome
+		};
+	});
+	const top = cases.reduce((a, b) => b.bound > a.bound ? b : a);
+	const beyond = cases.filter((c) => c.outcome === "beyond");
+	if (beyond.length > 0) return {
+		outcome: "beyond",
+		reason: `${beyond.length} of ${cases.length} cases are above the limit ${limit}, the first is ${JSON.stringify(beyond[0].case)}.`,
+		limit,
+		cases,
+		band
+	};
+	if (cases.every((c) => c.outcome === "within")) return {
+		outcome: "within",
+		reason: `The largest bound is ${round(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.`,
+		limit,
+		cases,
+		band
+	};
+	return {
+		outcome: "cannot decide",
+		reason: `The bound of the case ${JSON.stringify(top.case)} is ${round(top.bound)}, above the limit ${limit}, and its floor is not above it.`,
+		limit,
+		cases,
+		band
+	};
+}
 function round(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
@@ -246,6 +435,62 @@ function verdictPower(options) {
 		power: outcomes[truth]
 	};
 }
-export { aaBand, caseRatios, checkPairedTime, firstArm, timeVerdict, verdictPower };
+// src/rounds.ts
+/**
+* The timed rounds of one case in one process. The two arms first run for a warm-up time. Then the count of calls of
+* one batch grows until a batch of arm `a` takes the target time. Each round times one batch of each arm, in the order
+* of `firstArm`. Start this function in a fresh process for each case.
+*/
+/**
+* This function times the arms `a` and `b` of one case, and gives one `PairedTime` record for each round. The time of
+* a record is the time of one batch, in milliseconds. Each call of an arm gives a number, and the sum of these numbers
+* keeps the work.
+*/
+function timeRounds(arms, options) {
+	const { rounds } = options;
+	const offset = options.offset ?? 0;
+	const targetMs = options.targetMs ?? 10;
+	const warmMs = options.warmMs ?? 1e3;
+	const now = options.now ?? (() => performance.now());
+	if (!Number.isSafeInteger(rounds) || rounds < 1) throw new RangeError(`timeRounds: the rounds are a whole number of 1 or more, not ${rounds}.`);
+	if (!Number.isSafeInteger(offset)) throw new RangeError(`timeRounds: the offset is a whole number, not ${offset}.`);
+	if (!(targetMs > 0) || !(warmMs >= 0)) throw new RangeError("timeRounds: the target time is above 0, and the warm-up time is 0 or more.");
+	let sink = 0;
+	const batch = (arm, count) => {
+		const run = arms[arm];
+		const start = now();
+		for (let i = 0; i < count; i++) sink += run();
+		return now() - start;
+	};
+	let count = 1;
+	const warmUntil = now() + warmMs;
+	while (now() < warmUntil) {
+		if (batch("a", count) < targetMs / 4) count *= 2;
+		batch("b", count);
+	}
+	while (batch("a", count) < targetMs) count *= 2;
+	const records = [];
+	for (let round = 0; round < rounds; round++) {
+		const first = firstArm(round, offset);
+		const second = first === "a" ? "b" : "a";
+		const times = {
+			[first]: batch(first, count),
+			[second]: batch(second, count)
+		};
+		records.push({
+			case: options.case,
+			round,
+			first,
+			a: times.a,
+			b: times.b
+		});
+	}
+	return {
+		records,
+		count,
+		sink
+	};
+}
+export { aaBand, aaSpread, caseIntervals, caseRatios, checkPairedTime, firstArm, limitVerdict, pooledBand, timeRounds, timeVerdict, verdictPower };
 
 //# sourceMappingURL=index.js.map

package/build/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../src/paired.ts","../src/verdict.ts","../src/power.ts"],"sourcesContent":["/**\n * The paired-time record of a bench. Each round runs the two arms of a pair on one case, one after the other, in a\n * fresh process. The order turns each round, so a warm cache or a slow start does not always help the same arm.\n */\nimport { median } from \"@mark1russell7/rigor-statistics\";\n\n/** The two arms of a pair. Arm `a` is the baseline, and arm `b` is the candidate. */\nexport type Arm = \"a\" | \"b\";\n\n/** One round of one pair on one case: the times of the two arms in milliseconds, and the arm that ran first. */\nexport interface PairedTime {\n  /** The name of the case: one input of the bench. */\n  readonly case: string;\n  /** The number of the round, a whole number of 0 or more. */\n  readonly round: number;\n  readonly first: Arm;\n  /** The time of the baseline arm, in milliseconds. */\n  readonly a: number;\n  /** The time of the candidate arm, in milliseconds. */\n  readonly b: number;\n}\n\n/**\n * This function gives the arm that runs first in a round. The order turns each round. A case with an odd `offset`\n * starts with arm `b`, so half of the cases start with each arm.\n */\nexport function firstArm(round: number, offset = 0): Arm {\n  if (!Number.isSafeInteger(round) || round < 0) throw new RangeError(`firstArm: the round is a whole number of 0 or more, not ${round}.`);\n  if (!Number.isSafeInteger(offset)) throw new RangeError(`firstArm: the offset is a whole number, not ${offset}.`);\n  return (((round + offset) % 2) + 2) % 2 === 0 ? \"a\" : \"b\";\n}\n\n/** This function examines one record. A time that is not a finite number above 0 is an error. */\nexport function checkPairedTime(record: PairedTime): PairedTime {\n  const where = `the record of the case ${JSON.stringify(record.case)}, round ${record.round}`;\n  if (typeof record.case !== \"string\" || record.case === \"\") throw new RangeError(\"checkPairedTime: the case is a string that is not empty.\");\n  if (!Number.isSafeInteger(record.round) || record.round < 0) throw new RangeError(`checkPairedTime: in ${where}, the round is a whole number of 0 or more.`);\n  if (record.first !== \"a\" && record.first !== \"b\") throw new RangeError(`checkPairedTime: in ${where}, the first arm is \"a\" or \"b\", not ${String(record.first)}.`);\n  for (const arm of [\"a\", \"b\"] as const) {\n    const ms = record[arm];\n    if (!(typeof ms === \"number\" && Number.isFinite(ms) && ms > 0)) throw new RangeError(`checkPairedTime: in ${where}, the time of arm ${arm} is a finite number above 0, not ${String(ms)}.`);\n  }\n  return record;\n}\n\n/** The ratio of one case: the candidate time over the baseline time, over the rounds of the case. */\nexport interface CaseRatio {\n  readonly case: string;\n  /** The value `exp(median(log(b / a)))` over the rounds. For an odd count of rounds, it is the median of the ratios. */\n  readonly ratio: number;\n  readonly rounds: number;\n}\n\n/**\n * This function gives the ratio of each case, in the order of the first record of each case.\n * It examines each record. It refuses a case with 2 records of one round.\n * It also refuses a case whose two orders differ in count by more than 1, because that order did not turn each round.\n */\nexport function caseRatios(records: readonly PairedTime[]): CaseRatio[] {\n  const cases = new Map<string, PairedTime[]>();\n  for (const record of records) {\n    checkPairedTime(record);\n    const list = cases.get(record.case) ?? [];\n    list.push(record);\n    cases.set(record.case, list);\n  }\n  return [...cases].map(([name, list]) => {\n    const rounds = new Set(list.map((r) => r.round));\n    if (rounds.size !== list.length) throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} has 2 records of one round.`);\n    const aFirst = list.filter((r) => r.first === \"a\").length;\n    if (Math.abs(2 * aFirst - list.length) > 1) {\n      throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} starts with arm a in ${aFirst} of ${list.length} rounds. The order turns each round.`);\n    }\n    return { case: name, ratio: Math.exp(median(list.map((r) => Math.log(r.b / r.a)))), rounds: list.length };\n  });\n}\n","/**\n * The time verdict of a candidate against a baseline, with the noise of an A/A arm. An A/A pair runs the same code in\n * its two arms. Its ratios show how far apart two equal arms come on this machine. A candidate is faster or slower only\n * when its interval is out of that band.\n */\nimport { type BootstrapOptions, type Comparison, type Interval, bootstrapMedian, bootstrapTwo, compareRatios, median, quantile } from \"@mark1russell7/rigor-statistics\";\nimport { type CaseRatio, type PairedTime, caseRatios } from \"./paired.ts\";\n\n/**\n * The two kinds of an A/A band.\n * - The band `case` holds the ratio of one A/A case. It is wide, so it finds only a large change.\n * - The band `median` holds the median ratio of the A/A cases. It has the scale of the median of the candidate.\n */\nexport type BandKind = \"case\" | \"median\";\n\n/** The options of the A/A band. */\nexport interface BandOptions {\n  /** The kind of the band. The default is `median`, from the power test of `verdictPower`. */\n  readonly kind?: BandKind;\n  /** The quantile of the band `case`, or the level of the bootstrap of the band `median`. The default is 0.95. */\n  readonly level?: number;\n  /** The fewest A/A cases that give a band. The default is 5, the 5-seed rule. */\n  readonly minimum?: number;\n  /** The bootstrap of the band `median`. The level of the band replaces its level. */\n  readonly bootstrap?: BootstrapOptions;\n}\n\n/** The band of the A/A arm: two equal arms come out in this band of ratios. */\nexport interface Band {\n  /** The count of the A/A cases. */\n  readonly cases: number;\n  readonly kind: BandKind;\n  /**\n   * The half width of the band on a log scale.\n   * For the band `case`, it is the `level` quantile of `abs(log(ratio))` over the A/A cases.\n   * For the band `median`, it is the larger distance of an end of the bootstrap interval of the A/A median from 0.\n   */\n  readonly noise: number;\n  /** The ends of the band: `exp(-noise)` and `exp(noise)`. */\n  readonly low: number;\n  readonly high: number;\n  /** The median ratio of the A/A cases. A value far from 1 shows a bias of the order or of the arm slots. */\n  readonly median: number;\n}\n\n/** This function gives the band of the A/A arm. The band is symmetric on a log scale. It refuses fewer cases than `minimum`. */\nexport function aaBand(aa: readonly PairedTime[], options: BandOptions = {}): Band {\n  const ratios = caseRatios(aa);\n  const minimum = options.minimum ?? 5;\n  if (ratios.length < minimum) throw new RangeError(`aaBand: the A/A arm has ${ratios.length} cases, and a band needs ${minimum} or more.`);\n  const logs = ratios.map((r) => Math.log(r.ratio));\n  const kind = options.kind ?? \"median\";\n  const level = options.level ?? 0.95;\n  let noise: number;\n  if (kind === \"case\") noise = quantile(logs.map(Math.abs), level);\n  else {\n    const interval = bootstrapMedian(logs, { ...options.bootstrap, level });\n    noise = Math.max(Math.abs(interval.low), Math.abs(interval.high));\n  }\n  return { cases: ratios.length, kind, noise, low: Math.exp(-noise), high: Math.exp(noise), median: Math.exp(quantile(logs, 0.5)) };\n}\n\n/** The four outcomes of a time verdict. The candidate is the arm `b`, so `faster` is a ratio below 1. */\nexport type TimeOutcome = \"faster\" | \"slower\" | \"same\" | \"cannot decide\";\n\n/** The options of a time verdict. The bootstrap gives the interval of the median ratio of the candidate, and the band `median`. */\n/**\n * The two rules of a time verdict.\n * - The rule `band` compares the interval of the median ratio of the candidate with the A/A band.\n * - The rule `difference` compares the interval of the candidate median over the A/A median with 1 and with `margin`.\n */\nexport type VerdictRule = \"band\" | \"difference\";\n\n/** The options of a time verdict. The bootstrap gives each interval of the verdict. */\nexport interface VerdictOptions extends BandOptions {\n  /** The rule of the verdict. The default is `difference`, from the power test of `verdictPower`. */\n  readonly rule?: VerdictRule;\n  /** The half width on a log scale of the ratios that the rule `difference` calls `same`. The default is 0.05. */\n  readonly margin?: number;\n}\n\n/** A time verdict, with the facts that it uses. */\nexport interface TimeVerdict {\n  readonly outcome: TimeOutcome;\n  /** The reason of the outcome, in one sentence. */\n  readonly reason: string;\n  readonly cases: readonly CaseRatio[];\n  readonly rule: VerdictRule;\n  /**\n   * The interval that the rule compares, or undefined when there are no candidate cases.\n   * With too few A/A cases, it is the interval of the median ratio of the candidate.\n   * For the rule `band`, it is the bootstrap interval of the median ratio of the candidate.\n   * For the rule `difference`, it is the interval of the candidate median ratio over the A/A median ratio.\n   */\n  readonly interval: Interval | undefined;\n  /** The band of the A/A arm, or undefined when the A/A arm has too few cases. */\n  readonly band: Band | undefined;\n  /**\n   * The counts of the cases out of the band, or undefined without a band.\n   * The comparison uses the value `1 + log(ratio)` of each case and the noise of the band, so its two sides are the ends of the band.\n   */\n  readonly comparison: Comparison | undefined;\n}\n\n/**\n * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.\n * - The rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.\n *   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.\n * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.\n * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.\n */\nexport function timeVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: VerdictOptions = {}): TimeVerdict {\n  const rule = options.rule ?? \"difference\";\n  const cases = caseRatios(candidate);\n  const aaRatios = caseRatios(aa);\n  const minimum = options.minimum ?? 5;\n  const none = { cases, rule, interval: undefined, band: undefined, comparison: undefined };\n  if (cases.length === 0) return { ...none, outcome: \"cannot decide\", reason: \"The candidate has no cases.\" };\n  if (aaRatios.length < minimum) {\n    const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);\n    return { ...none, interval, outcome: \"cannot decide\", reason: `The A/A arm has ${aaRatios.length} cases, and a band needs ${minimum} or more.` };\n  }\n  const band = aaBand(aa, options);\n  const comparison = compareRatios(cases.map((c) => 1 + Math.log(c.ratio)), { better: \"lower\", noise: band.noise });\n  const facts = { cases, rule, band, comparison };\n  if (rule === \"difference\") {\n    const margin = options.margin ?? 0.05;\n    if (!(margin >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${margin}.`);\n    const logs = cases.map((c) => Math.log(c.ratio));\n    const aaLogs = aaRatios.map((c) => Math.log(c.ratio));\n    const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.bootstrap);\n    const interval: Interval = { ...d, estimate: Math.exp(d.estimate), low: Math.exp(d.low), high: Math.exp(d.high) };\n    const span = `The interval of the candidate median over the A/A median is [${round(interval.low)}, ${round(interval.high)}].`;\n    if (interval.high < 1) return { ...facts, interval, outcome: \"faster\", reason: `${span} The interval is below 1.` };\n    if (interval.low > 1) return { ...facts, interval, outcome: \"slower\", reason: `${span} The interval is above 1.` };\n    if (d.low >= -margin && d.high <= margin) return { ...facts, interval, outcome: \"same\", reason: `${span} The interval is in the margin of ${margin}.` };\n    return { ...facts, interval, outcome: \"cannot decide\", reason: `${span} The interval holds 1 and crosses the margin of ${margin}.` };\n  }\n  const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);\n  const span = `The interval of the median ratio is [${round(interval.low)}, ${round(interval.high)}], and the A/A band is [${round(band.low)}, ${round(band.high)}].`;\n  if (interval.high < band.low) return { ...facts, interval, outcome: \"faster\", reason: `${span} The interval is below the band.` };\n  if (interval.low > band.high) return { ...facts, interval, outcome: \"slower\", reason: `${span} The interval is above the band.` };\n  if (interval.low >= band.low && interval.high <= band.high) return { ...facts, interval, outcome: \"same\", reason: `${span} The interval is in the band.` };\n  return { ...facts, interval, outcome: \"cannot decide\", reason: `${span} The interval crosses an end of the band.` };\n}\n\nfunction round(x: number): number {\n  return Math.round(x * 10_000) / 10_000;\n}\n","/**\n * The power test of a time verdict. It draws seeded benches with a known speedup and counts the verdicts that find it.\n * It also draws benches with equal arms and counts the false verdicts.\n */\nimport { random } from \"@mark1russell7/rigor-generators\";\nimport { firstArm, type PairedTime } from \"./paired.ts\";\nimport { type TimeOutcome, type VerdictOptions, timeVerdict } from \"./verdict.ts\";\n\n/** The options of a power test. */\nexport interface PowerOptions {\n  /** The count of the cases of the candidate and of the A/A arm. */\n  readonly cases: number;\n  /** The count of the rounds of each case. The default is 5. */\n  readonly rounds?: number;\n  /** The true ratio of the candidate time to the baseline time, for example 0.93 for a speedup of 7%. */\n  readonly ratio: number;\n  /** The standard deviation of the log time of one arm in one round. The default is 0.1. */\n  readonly sigma?: number;\n  /** The count of the seeded benches. The default is 200. */\n  readonly trials?: number;\n  /** The seed of the draws. The default is 1. */\n  readonly seed?: number | string;\n  /** The options of the verdict of each bench. */\n  readonly verdict?: VerdictOptions;\n}\n\n/** The result of a power test: the share of each outcome over the trials. */\nexport interface Power {\n  readonly trials: number;\n  readonly outcomes: Readonly<Record<TimeOutcome, number>>;\n  /** The share of the trials whose outcome is the true one: `faster` for a ratio below 1, `slower` above 1, `same` at 1. */\n  readonly power: number;\n}\n\n/**\n * This function draws `trials` benches and gives the share of each verdict. Each case has its own log scale.\n * Each arm of each round adds a normal noise of `sigma` to the log time.\n */\nexport function verdictPower(options: PowerOptions): Power {\n  const rounds = options.rounds ?? 5;\n  const sigma = options.sigma ?? 0.1;\n  const trials = options.trials ?? 200;\n  if (!Number.isSafeInteger(options.cases) || options.cases < 1) throw new RangeError(`verdictPower: cases is a whole number of 1 or more, not ${options.cases}.`);\n  if (!(options.ratio > 0 && Number.isFinite(options.ratio))) throw new RangeError(`verdictPower: the ratio is a finite number above 0, not ${options.ratio}.`);\n  const r = random(options.seed ?? 1);\n  const bench = (ratio: number, prefix: string): PairedTime[] => {\n    const out: PairedTime[] = [];\n    for (let c = 0; c < options.cases; c++) {\n      const scale = 10 * Math.exp(r.normal());\n      for (let round = 0; round < rounds; round++) {\n        const a = scale * Math.exp(r.normal(0, sigma));\n        const b = scale * ratio * Math.exp(r.normal(0, sigma));\n        out.push({ case: `${prefix}${c}`, round, first: firstArm(round, c), a, b });\n      }\n    }\n    return out;\n  };\n  const truth: TimeOutcome = options.ratio < 1 ? \"faster\" : options.ratio > 1 ? \"slower\" : \"same\";\n  const counts: Record<TimeOutcome, number> = { faster: 0, slower: 0, same: 0, \"cannot decide\": 0 };\n  for (let t = 0; t < trials; t++) {\n    const aa = bench(1, \"aa\");\n    const candidate = bench(options.ratio, \"c\");\n    counts[timeVerdict(candidate, aa, options.verdict).outcome]++;\n  }\n  const outcomes = Object.fromEntries(Object.entries(counts).map(([k, n]) => [k, n / trials])) as Record<TimeOutcome, number>;\n  return { trials, outcomes, power: outcomes[truth] };\n}\n"],"mappings":";;;;;;;;;;;AA0BA,SAAgB,SAAS,OAAe,SAAS,GAAQ;CACvD,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG,MAAM,IAAI,WAAW,2DAA2D,MAAM,EAAE;CACvI,IAAI,CAAC,OAAO,cAAc,MAAM,GAAG,MAAM,IAAI,WAAW,+CAA+C,OAAO,EAAE;CAChH,SAAU,QAAQ,UAAU,IAAK,KAAK,MAAM,IAAI,MAAM;AACxD;;AAGA,SAAgB,gBAAgB,QAAgC;CAC9D,MAAM,QAAQ,0BAA0B,KAAK,UAAU,OAAO,IAAI,EAAE,UAAU,OAAO;CACrF,IAAI,OAAO,OAAO,SAAS,YAAY,OAAO,SAAS,IAAI,MAAM,IAAI,WAAW,0DAA0D;CAC1I,IAAI,CAAC,OAAO,cAAc,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG,MAAM,IAAI,WAAW,uBAAuB,MAAM,4CAA4C;CAC3J,IAAI,OAAO,UAAU,OAAO,OAAO,UAAU,KAAK,MAAM,IAAI,WAAW,uBAAuB,MAAM,qCAAqC,OAAO,OAAO,KAAK,EAAE,EAAE;CAChK,KAAK,MAAM,OAAO,CAAC,KAAK,GAAG,GAAY;EACrC,MAAM,KAAK,OAAO;EAClB,IAAI,EAAE,OAAO,OAAO,YAAY,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,MAAM,IAAI,WAAW,uBAAuB,MAAM,oBAAoB,IAAI,mCAAmC,OAAO,EAAE,EAAE,EAAE;CAC5L;CACA,OAAO;AACT;;;;;;AAeA,SAAgB,WAAW,SAA6C;CACtE,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,UAAU,SAAS;EAC5B,gBAAgB,MAAM;EACtB,MAAM,OAAO,MAAM,IAAI,OAAO,IAAI,KAAK,CAAC;EACxC,KAAK,KAAK,MAAM;EAChB,MAAM,IAAI,OAAO,MAAM,IAAI;CAC7B;CACA,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU;EAEtC,IAAI,IADe,IAAI,KAAK,KAAK,MAAM,EAAE,KAAK,CACrC,CAAC,CAAC,SAAS,KAAK,QAAQ,MAAM,IAAI,WAAW,wBAAwB,KAAK,UAAU,IAAI,EAAE,6BAA6B;EAChI,MAAM,SAAS,KAAK,QAAQ,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC;EACnD,IAAI,KAAK,IAAI,IAAI,SAAS,KAAK,MAAM,IAAI,GACvC,MAAM,IAAI,WAAW,wBAAwB,KAAK,UAAU,IAAI,EAAE,wBAAwB,OAAO,MAAM,KAAK,OAAO,qCAAqC;EAE1J,OAAO;GAAE,MAAM;GAAM,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,MAAM,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;GAAG,QAAQ,KAAK;EAAO;CAC1G,CAAC;AACH;;;;;;;;AC7BA,SAAgB,OAAO,IAA2B,UAAuB,CAAC,GAAS;CACjF,MAAM,SAAS,WAAW,EAAE;CAC5B,MAAM,UAAU,QAAQ,WAAW;CACnC,IAAI,OAAO,SAAS,SAAS,MAAM,IAAI,WAAW,2BAA2B,OAAO,OAAO,2BAA2B,QAAQ,UAAU;CACxI,MAAM,OAAO,OAAO,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;CAChD,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI;CACJ,IAAI,SAAS,QAAQ,QAAQ,SAAS,KAAK,IAAI,KAAK,GAAG,GAAG,KAAK;MAC1D;EACH,MAAM,WAAW,gBAAgB,MAAM;GAAE,GAAG,QAAQ;GAAW;EAAM,CAAC;EACtE,QAAQ,KAAK,IAAI,KAAK,IAAI,SAAS,GAAG,GAAG,KAAK,IAAI,SAAS,IAAI,CAAC;CAClE;CACA,OAAO;EAAE,OAAO,OAAO;EAAQ;EAAM;EAAO,KAAK,KAAK,IAAI,CAAC,KAAK;EAAG,MAAM,KAAK,IAAI,KAAK;EAAG,QAAQ,KAAK,IAAI,SAAS,MAAM,EAAG,CAAC;CAAE;AAClI;;;;;;;;AAmDA,SAAgB,YAAY,WAAkC,IAA2B,UAA0B,CAAC,GAAgB;CAClI,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,WAAW,SAAS;CAClC,MAAM,WAAW,WAAW,EAAE;CAC9B,MAAM,UAAU,QAAQ,WAAW;CACnC,MAAM,OAAO;EAAE;EAAO;EAAM,UAAU,KAAA;EAAW,MAAM,KAAA;EAAW,YAAY,KAAA;CAAU;CACxF,IAAI,MAAM,WAAW,GAAG,OAAO;EAAE,GAAG;EAAM,SAAS;EAAiB,QAAQ;CAA8B;CAC1G,IAAI,SAAS,SAAS,SAAS;EAC7B,MAAM,WAAW,gBAAgB,MAAM,KAAK,MAAM,EAAE,KAAK,GAAG,QAAQ,SAAS;EAC7E,OAAO;GAAE,GAAG;GAAM;GAAU,SAAS;GAAiB,QAAQ,mBAAmB,SAAS,OAAO,2BAA2B,QAAQ;EAAW;CACjJ;CACA,MAAM,OAAO,OAAO,IAAI,OAAO;CAE/B,MAAM,QAAQ;EAAE;EAAO;EAAM;EAAM,YADhB,cAAc,MAAM,KAAK,MAAM,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,GAAG;GAAE,QAAQ;GAAS,OAAO,KAAK;EAAM,CACnE;CAAE;CAC9C,IAAI,SAAS,cAAc;EACzB,MAAM,SAAS,QAAQ,UAAU;EACjC,IAAI,EAAE,UAAU,IAAI,MAAM,IAAI,WAAW,6CAA6C,OAAO,EAAE;EAC/F,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;EAC/C,MAAM,SAAS,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;EACpD,MAAM,IAAI,aAAa,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,QAAQ,SAAS;EACvF,MAAM,WAAqB;GAAE,GAAG;GAAG,UAAU,KAAK,IAAI,EAAE,QAAQ;GAAG,KAAK,KAAK,IAAI,EAAE,GAAG;GAAG,MAAM,KAAK,IAAI,EAAE,IAAI;EAAE;EAChH,MAAM,OAAO,gEAAgE,MAAM,SAAS,GAAG,EAAE,IAAI,MAAM,SAAS,IAAI,EAAE;EAC1H,IAAI,SAAS,OAAO,GAAG,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAU,QAAQ,GAAG,KAAK;EAA2B;EAClH,IAAI,SAAS,MAAM,GAAG,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAU,QAAQ,GAAG,KAAK;EAA2B;EACjH,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,QAAQ,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAQ,QAAQ,GAAG,KAAK,oCAAoC,OAAO;EAAG;EACtJ,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAiB,QAAQ,GAAG,KAAK,kDAAkD,OAAO;EAAG;CACrI;CACA,MAAM,WAAW,gBAAgB,MAAM,KAAK,MAAM,EAAE,KAAK,GAAG,QAAQ,SAAS;CAC7E,MAAM,OAAO,wCAAwC,MAAM,SAAS,GAAG,EAAE,IAAI,MAAM,SAAS,IAAI,EAAE,0BAA0B,MAAM,KAAK,GAAG,EAAE,IAAI,MAAM,KAAK,IAAI,EAAE;CACjK,IAAI,SAAS,OAAO,KAAK,KAAK,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAU,QAAQ,GAAG,KAAK;CAAkC;CAChI,IAAI,SAAS,MAAM,KAAK,MAAM,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAU,QAAQ,GAAG,KAAK;CAAkC;CAChI,IAAI,SAAS,OAAO,KAAK,OAAO,SAAS,QAAQ,KAAK,MAAM,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAQ,QAAQ,GAAG,KAAK;CAA+B;CACzJ,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAiB,QAAQ,GAAG,KAAK;CAA2C;AACpH;AAEA,SAAS,MAAM,GAAmB;CAChC,OAAO,KAAK,MAAM,IAAI,GAAM,IAAI;AAClC;;;;;;;;;;AC9GA,SAAgB,aAAa,SAA8B;CACzD,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,SAAS,QAAQ,UAAU;CACjC,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,KAAK,QAAQ,QAAQ,GAAG,MAAM,IAAI,WAAW,2DAA2D,QAAQ,MAAM,EAAE;CAC/J,IAAI,EAAE,QAAQ,QAAQ,KAAK,OAAO,SAAS,QAAQ,KAAK,IAAI,MAAM,IAAI,WAAW,2DAA2D,QAAQ,MAAM,EAAE;CAC5J,MAAM,IAAI,OAAO,QAAQ,QAAQ,CAAC;CAClC,MAAM,SAAS,OAAe,WAAiC;EAC7D,MAAM,MAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,OAAO,KAAK;GACtC,MAAM,QAAQ,KAAK,KAAK,IAAI,EAAE,OAAO,CAAC;GACtC,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,SAAS;IAC3C,MAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IAC7C,MAAM,IAAI,QAAQ,QAAQ,KAAK,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IACrD,IAAI,KAAK;KAAE,MAAM,GAAG,SAAS;KAAK;KAAO,OAAO,SAAS,OAAO,CAAC;KAAG;KAAG;IAAE,CAAC;GAC5E;EACF;EACA,OAAO;CACT;CACA,MAAM,QAAqB,QAAQ,QAAQ,IAAI,WAAW,QAAQ,QAAQ,IAAI,WAAW;CACzF,MAAM,SAAsC;EAAE,QAAQ;EAAG,QAAQ;EAAG,MAAM;EAAG,iBAAiB;CAAE;CAChG,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,KAAK;EAC/B,MAAM,KAAK,MAAM,GAAG,IAAI;EACxB,MAAM,YAAY,MAAM,QAAQ,OAAO,GAAG;EAC1C,OAAO,YAAY,WAAW,IAAI,QAAQ,OAAO,CAAC,CAAC,QAAQ;CAC7D;CACA,MAAM,WAAW,OAAO,YAAY,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;CAC3F,OAAO;EAAE;EAAQ;EAAU,OAAO,SAAS;CAAO;AACpD"}
+{"version":3,"file":"index.js","names":["round"],"sources":["../src/paired.ts","../src/verdict.ts","../src/limit.ts","../src/power.ts","../src/rounds.ts"],"sourcesContent":["/**\n * The paired-time record of a bench. Each round runs the two arms of a pair on one case, one after the other, in a\n * fresh process. The order turns each round, so a warm cache or a slow start does not always help the same arm.\n */\nimport { median } from \"@mark1russell7/rigor-statistics\";\n\n/** The two arms of a pair. Arm `a` is the baseline, and arm `b` is the candidate. */\nexport type Arm = \"a\" | \"b\";\n\n/** One round of one pair on one case: the times of the two arms in milliseconds, and the arm that ran first. */\nexport interface PairedTime {\n  /** The name of the case: one input of the bench. */\n  readonly case: string;\n  /** The number of the round, a whole number of 0 or more. */\n  readonly round: number;\n  readonly first: Arm;\n  /** The time of the baseline arm, in milliseconds. */\n  readonly a: number;\n  /** The time of the candidate arm, in milliseconds. */\n  readonly b: number;\n}\n\n/**\n * This function gives the arm that runs first in a round. The order turns each round. A case with an odd `offset`\n * starts with arm `b`, so half of the cases start with each arm.\n */\nexport function firstArm(round: number, offset = 0): Arm {\n  if (!Number.isSafeInteger(round) || round < 0) throw new RangeError(`firstArm: the round is a whole number of 0 or more, not ${round}.`);\n  if (!Number.isSafeInteger(offset)) throw new RangeError(`firstArm: the offset is a whole number, not ${offset}.`);\n  return (((round + offset) % 2) + 2) % 2 === 0 ? \"a\" : \"b\";\n}\n\n/** This function examines one record. A time that is not a finite number above 0 is an error. */\nexport function checkPairedTime(record: PairedTime): PairedTime {\n  const where = `the record of the case ${JSON.stringify(record.case)}, round ${record.round}`;\n  if (typeof record.case !== \"string\" || record.case === \"\") throw new RangeError(\"checkPairedTime: the case is a string that is not empty.\");\n  if (!Number.isSafeInteger(record.round) || record.round < 0) throw new RangeError(`checkPairedTime: in ${where}, the round is a whole number of 0 or more.`);\n  if (record.first !== \"a\" && record.first !== \"b\") throw new RangeError(`checkPairedTime: in ${where}, the first arm is \"a\" or \"b\", not ${String(record.first)}.`);\n  for (const arm of [\"a\", \"b\"] as const) {\n    const ms = record[arm];\n    if (!(typeof ms === \"number\" && Number.isFinite(ms) && ms > 0)) throw new RangeError(`checkPairedTime: in ${where}, the time of arm ${arm} is a finite number above 0, not ${String(ms)}.`);\n  }\n  return record;\n}\n\n/** The ratio of one case: the candidate time over the baseline time, over the rounds of the case. */\nexport interface CaseRatio {\n  readonly case: string;\n  /** The value `exp(median(log(b / a)))` over the rounds. For an odd count of rounds, it is the median of the ratios. */\n  readonly ratio: number;\n  readonly rounds: number;\n}\n\n/**\n * This function gives the ratio of each case, in the order of the first record of each case.\n * It examines each record. It refuses a case with 2 records of one round.\n * It also refuses a case whose two orders differ in count by more than 1, because that order did not turn each round.\n */\nexport function caseRatios(records: readonly PairedTime[]): CaseRatio[] {\n  const cases = new Map<string, PairedTime[]>();\n  for (const record of records) {\n    checkPairedTime(record);\n    const list = cases.get(record.case) ?? [];\n    list.push(record);\n    cases.set(record.case, list);\n  }\n  return [...cases].map(([name, list]) => {\n    const rounds = new Set(list.map((r) => r.round));\n    if (rounds.size !== list.length) throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} has 2 records of one round.`);\n    const aFirst = list.filter((r) => r.first === \"a\").length;\n    if (Math.abs(2 * aFirst - list.length) > 1) {\n      throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} starts with arm a in ${aFirst} of ${list.length} rounds. The order turns each round.`);\n    }\n    return { case: name, ratio: Math.exp(median(list.map((r) => Math.log(r.b / r.a)))), rounds: list.length };\n  });\n}\n","/**\n * The time verdict of a candidate against a baseline, with the noise of an A/A arm. An A/A pair runs the same code in\n * its two arms. Its ratios show how far apart two equal arms come on this machine. A candidate is faster or slower only\n * when its interval is out of that band.\n */\nimport { type BootstrapOptions, type Comparison, type Interval, bootstrapMedian, bootstrapTwo, compareRatios, median, quantile } from \"@mark1russell7/rigor-statistics\";\nimport { type CaseRatio, type PairedTime, caseRatios } from \"./paired.ts\";\n\n/**\n * The two kinds of an A/A band.\n * - The band `case` holds the ratio of one A/A case. It is wide, so it finds only a large change.\n * - The band `median` holds the median ratio of the A/A cases. It has the scale of the median of the candidate.\n */\nexport type BandKind = \"case\" | \"median\";\n\n/** The options of the A/A band. */\nexport interface BandOptions {\n  /** The kind of the band. The default is `median`, from the power test of `verdictPower`. */\n  readonly kind?: BandKind;\n  /** The quantile of the band `case`, or the level of the bootstrap of the band `median`. The default is 0.95. */\n  readonly level?: number;\n  /** The fewest A/A cases that give a band. The default is 5, the 5-seed rule. */\n  readonly minimum?: number;\n  /** The bootstrap of the band `median`. The level of the band replaces its level. */\n  readonly bootstrap?: BootstrapOptions;\n}\n\n/** The band of the A/A arm: two equal arms come out in this band of ratios. */\nexport interface Band {\n  /** The count of the A/A cases. */\n  readonly cases: number;\n  readonly kind: BandKind;\n  /**\n   * The half width of the band on a log scale.\n   * For the band `case`, it is the `level` quantile of `abs(log(ratio))` over the A/A cases.\n   * For the band `median`, it is the larger distance of an end of the bootstrap interval of the A/A median from 0.\n   */\n  readonly noise: number;\n  /** The ends of the band: `exp(-noise)` and `exp(noise)`. */\n  readonly low: number;\n  readonly high: number;\n  /** The median ratio of the A/A cases. A value far from 1 shows a bias of the order or of the arm slots. */\n  readonly median: number;\n}\n\n/** This function gives the band of the A/A arm. The band is symmetric on a log scale. It refuses fewer cases than `minimum`. */\nexport function aaBand(aa: readonly PairedTime[], options: BandOptions = {}): Band {\n  const ratios = caseRatios(aa);\n  const minimum = options.minimum ?? 5;\n  if (ratios.length < minimum) throw new RangeError(`aaBand: the A/A arm has ${ratios.length} cases, and a band needs ${minimum} or more.`);\n  const logs = ratios.map((r) => Math.log(r.ratio));\n  const kind = options.kind ?? \"median\";\n  const level = options.level ?? 0.95;\n  let noise: number;\n  if (kind === \"case\") noise = quantile(logs.map(Math.abs), level);\n  else {\n    const interval = bootstrapMedian(logs, { ...options.bootstrap, level });\n    noise = Math.max(Math.abs(interval.low), Math.abs(interval.high));\n  }\n  return { cases: ratios.length, kind, noise, low: Math.exp(-noise), high: Math.exp(noise), median: Math.exp(quantile(logs, 0.5)) };\n}\n\n/**\n * This function gives the A/A spread: the `level` quantile of `abs(log(ratio))` over the A/A cases. The default level is 0.9.\n * It refuses an A/A arm with no cases.\n */\nexport function aaSpread(aa: readonly PairedTime[], level = 0.9): number {\n  const ratios = caseRatios(aa);\n  if (ratios.length === 0) throw new RangeError(\"aaSpread: the A/A arm has no cases.\");\n  return spreadOf(ratios, level);\n}\n\nfunction spreadOf(ratios: readonly CaseRatio[], level: number): number {\n  return quantile(ratios.map((r) => Math.abs(Math.log(r.ratio))), level);\n}\n\n/** The options of a pooled band: the A/A records of earlier quiet runs, and a floor. The caller keeps the pool. */\nexport interface PoolOptions {\n  /** The A/A records of the earlier runs. Give each run its own case names, for example with the run id as a prefix. */\n  readonly records: readonly PairedTime[];\n  /** The smallest noise of the band, on a log scale. The default is 0. A floor of `Math.log(1.05)` gives at least the band `[1 / 1.05, 1.05]`. */\n  readonly floor?: number;\n  /** The fewest pooled cases that give a spread. With fewer cases, the noise of the band is the floor. The default is 5. */\n  readonly minimum?: number;\n  /**\n   * The run of a record, for example the id of its CI run. The verdict counts the runs of the pool with it.\n   * The cases of one run share one runner, so they are not independent. Many cases from few runs show the noise of few runners.\n   */\n  readonly run?: (record: PairedTime) => string;\n}\n\n/** The facts of the pooled band of a verdict. */\nexport interface PoolFacts {\n  /** The count of the pooled A/A cases. */\n  readonly cases: number;\n  /** The count of the runs of the pool, from the option `run`. It is undefined without that option. */\n  readonly runs: number | undefined;\n  /** The A/A spread of the pooled cases, or undefined when the pool has fewer cases than `minimum`. */\n  readonly spread: number | undefined;\n  readonly floor: number;\n  /** This value is `true` when the spread of the pool is larger than the floor, so the spread gives the noise of the band. */\n  readonly pooled: boolean;\n}\n\n/**\n * This function gives the pooled band of the kind `case`. Its noise is the larger of the floor and the A/A spread of the pooled records.\n * The spread is the `level` quantile of `abs(log(ratio))`, with the default level 0.9.\n *\n * A band with fewer pooled cases than `minimum` uses the floor alone. A band with no pooled cases has the median 1.\n */\nexport function pooledBand(pool: PoolOptions, level = 0.9): Band {\n  return poolOf(pool, level).band;\n}\n\nfunction poolOf(pool: PoolOptions, level: number): { band: Band; facts: PoolFacts } {\n  const floor = pool.floor ?? 0;\n  const minimum = pool.minimum ?? 5;\n  if (!(floor >= 0 && Number.isFinite(floor))) throw new RangeError(`pooledBand: the floor is a finite number of 0 or more, not ${floor}.`);\n  if (!Number.isSafeInteger(minimum) || minimum < 1) throw new RangeError(`pooledBand: the minimum is a whole number of 1 or more, not ${minimum}.`);\n  const ratios = caseRatios(pool.records);\n  const spread = ratios.length >= minimum ? spreadOf(ratios, level) : undefined;\n  const noise = Math.max(floor, spread ?? 0);\n  const median = ratios.length === 0 ? 1 : Math.exp(quantile(ratios.map((r) => Math.log(r.ratio)), 0.5));\n  const band: Band = { cases: ratios.length, kind: \"case\", noise, low: Math.exp(-noise), high: Math.exp(noise), median };\n  const runs = pool.run === undefined ? undefined : new Set(pool.records.map(pool.run)).size;\n  return { band, facts: { cases: ratios.length, runs, spread, floor, pooled: spread !== undefined && spread > floor } };\n}\n\n/** The four outcomes of a time verdict. The candidate is the arm `b`, so `faster` is a ratio below 1. */\nexport type TimeOutcome = \"faster\" | \"slower\" | \"same\" | \"cannot decide\";\n\n/** The options of a time verdict. The bootstrap gives the interval of the median ratio of the candidate, and the band `median`. */\n/**\n * The two rules of a time verdict.\n * - The rule `band` compares the interval of the median ratio of the candidate with the A/A band.\n * - The rule `difference` compares the interval of the candidate median over the A/A median with 1 and with `margin`.\n */\nexport type VerdictRule = \"band\" | \"difference\";\n\n/** The options of a time verdict. The bootstrap gives each interval of the verdict. */\nexport interface VerdictOptions extends BandOptions {\n  /** The rule of the verdict. The default is `difference`, from the power test of `verdictPower`. */\n  readonly rule?: VerdictRule;\n  /** The half width on a log scale of the ratios that the rule `difference` calls `same`. The default is 0.05. */\n  readonly margin?: number;\n  /**\n   * The noise cap on the A/A spread of the run, on a log scale. Above the cap, the outcome is `cannot decide`.\n   * The default is no cap.\n   */\n  readonly noiseCap?: number;\n  /** The quantile of the A/A spread of the noise cap and of the pooled band. The default is 0.9. */\n  readonly spreadLevel?: number;\n  /**\n   * The pooled band. It replaces the band of the A/A arm of the run in the rule `band` and in `comparison`.\n   * The rule `difference` uses the larger of `margin` and the noise of the pooled band as its margin. The default is no pool.\n   */\n  readonly pool?: PoolOptions;\n}\n\n/** The noise cap of a verdict: the A/A spread of the run and the cap. */\nexport interface NoiseCheck {\n  /** The A/A spread of the run, or undefined when the A/A arm has no cases. */\n  readonly spread: number | undefined;\n  readonly cap: number;\n  /** This value is `true` when the spread is above the cap, or when the A/A arm has no cases. */\n  readonly noisy: boolean;\n}\n\n/** A time verdict, with the facts that it uses. */\nexport interface TimeVerdict {\n  readonly outcome: TimeOutcome;\n  /** The reason of the outcome, in one sentence. */\n  readonly reason: string;\n  readonly cases: readonly CaseRatio[];\n  readonly rule: VerdictRule;\n  /**\n   * The interval that the rule compares, or undefined when there are no candidate cases.\n   * With too few A/A cases, it is the interval of the median ratio of the candidate.\n   * For the rule `band`, it is the bootstrap interval of the median ratio of the candidate.\n   * For the rule `difference`, it is the interval of the candidate median ratio over the A/A median ratio.\n   */\n  readonly interval: Interval | undefined;\n  /** The band of the A/A arm, or undefined when the A/A arm has too few cases. */\n  readonly band: Band | undefined;\n  /**\n   * The counts of the cases out of the band, or undefined without a band.\n   * The comparison uses the value `1 + log(ratio)` of each case and the noise of the band, so its two sides are the ends of the band.\n   */\n  readonly comparison: Comparison | undefined;\n  /** The facts of the pooled band. The verdict has this field only with the option `pool`. */\n  readonly pool?: PoolFacts;\n  /** The noise cap. The verdict has this field only with the option `noiseCap`. */\n  readonly noise?: NoiseCheck;\n}\n\n/**\n * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.\n * - The rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.\n *   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.\n * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.\n * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.\n * - With `noiseCap`, an A/A spread of the run above the cap gives `cannot decide`.\n * - With `pool`, the pooled band replaces the band of the A/A arm of the run. The run still needs `minimum` A/A cases.\n */\nexport function timeVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: VerdictOptions = {}): TimeVerdict {\n  const level = options.spreadLevel ?? 0.9;\n  if (options.noiseCap !== undefined && !(options.noiseCap >= 0 && Number.isFinite(options.noiseCap))) {\n    throw new RangeError(`timeVerdict: the noise cap is a finite number of 0 or more, not ${options.noiseCap}.`);\n  }\n  const pool = options.pool === undefined ? undefined : poolOf(options.pool, level);\n  let verdict = ruleVerdict(candidate, aa, options, pool?.band);\n  if (pool !== undefined) verdict = { ...verdict, pool: pool.facts };\n  if (options.noiseCap !== undefined) {\n    const cap = options.noiseCap;\n    const aaRatios = caseRatios(aa);\n    const spread = aaRatios.length === 0 ? undefined : spreadOf(aaRatios, level);\n    const noise: NoiseCheck = { spread, cap, noisy: spread === undefined || spread > cap };\n    verdict = { ...verdict, noise };\n    // The early outcomes (no candidate cases, too few A/A cases) keep their reason. Each other outcome gives way to the cap.\n    if (noise.noisy && verdict.band !== undefined) {\n      const why = spread === undefined ? \"The A/A arm of the run has no cases, so the noise cap has no spread.\" : `The A/A spread of the run is ${round(spread)}, above the noise cap ${cap}.`;\n      const reason = `${why} Without the cap, the outcome is \"${verdict.outcome}\".`;\n      verdict = { ...verdict, outcome: \"cannot decide\", reason };\n    }\n  }\n  return verdict;\n}\n\n/** This function gives the outcome of the rule. A pooled band replaces the band of the A/A arm, and it widens the margin of the rule `difference`. */\nfunction ruleVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: VerdictOptions, pooled: Band | undefined): TimeVerdict {\n  const rule = options.rule ?? \"difference\";\n  const cases = caseRatios(candidate);\n  const aaRatios = caseRatios(aa);\n  const minimum = options.minimum ?? 5;\n  const none = { cases, rule, interval: undefined, band: undefined, comparison: undefined };\n  if (cases.length === 0) return { ...none, outcome: \"cannot decide\", reason: \"The candidate has no cases.\" };\n  if (aaRatios.length < minimum) {\n    const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);\n    return { ...none, interval, outcome: \"cannot decide\", reason: `The A/A arm has ${aaRatios.length} cases, and a band needs ${minimum} or more.` };\n  }\n  const band = pooled ?? aaBand(aa, options);\n  const comparison = compareRatios(cases.map((c) => 1 + Math.log(c.ratio)), { better: \"lower\", noise: band.noise });\n  const facts = { cases, rule, band, comparison };\n  if (rule === \"difference\") {\n    const given = options.margin ?? 0.05;\n    if (!(given >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${given}.`);\n    const margin = pooled === undefined ? given : Math.max(given, pooled.noise);\n    const shown = pooled === undefined ? margin : round(margin);\n    const logs = cases.map((c) => Math.log(c.ratio));\n    const aaLogs = aaRatios.map((c) => Math.log(c.ratio));\n    const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.bootstrap);\n    const interval: Interval = { ...d, estimate: Math.exp(d.estimate), low: Math.exp(d.low), high: Math.exp(d.high) };\n    const span = `The interval of the candidate median over the A/A median is [${round(interval.low)}, ${round(interval.high)}].`;\n    if (interval.high < 1) return { ...facts, interval, outcome: \"faster\", reason: `${span} The interval is below 1.` };\n    if (interval.low > 1) return { ...facts, interval, outcome: \"slower\", reason: `${span} The interval is above 1.` };\n    if (d.low >= -margin && d.high <= margin) return { ...facts, interval, outcome: \"same\", reason: `${span} The interval is in the margin of ${shown}.` };\n    return { ...facts, interval, outcome: \"cannot decide\", reason: `${span} The interval holds 1 and crosses the margin of ${shown}.` };\n  }\n  const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);\n  const span = `The interval of the median ratio is [${round(interval.low)}, ${round(interval.high)}], and the A/A band is [${round(band.low)}, ${round(band.high)}].`;\n  if (interval.high < band.low) return { ...facts, interval, outcome: \"faster\", reason: `${span} The interval is below the band.` };\n  if (interval.low > band.high) return { ...facts, interval, outcome: \"slower\", reason: `${span} The interval is above the band.` };\n  if (interval.low >= band.low && interval.high <= band.high) return { ...facts, interval, outcome: \"same\", reason: `${span} The interval is in the band.` };\n  return { ...facts, interval, outcome: \"cannot decide\", reason: `${span} The interval crosses an end of the band.` };\n}\n\nfunction round(x: number): number {\n  return Math.round(x * 10_000) / 10_000;\n}\n","/**\n * The limit verdict of a bench: is the ratio of each case at most a limit? For example, the claim \"the new code takes at most\n * 2 times the time of the old code\" needs it. The time verdict compares only with 1.\n */\nimport { medianInterval } from \"@mark1russell7/rigor-statistics\";\nimport { type CaseRatio, type PairedTime, caseRatios } from \"./paired.ts\";\nimport { type Band, type BandOptions, aaBand } from \"./verdict.ts\";\n\n/** The ratio of one case with the interval of its median over the rounds. */\nexport interface CaseInterval extends CaseRatio {\n  /** The ends of the interval: `exp` of `medianInterval` over the values `log(b / a)` of the rounds. */\n  readonly low: number;\n  readonly high: number;\n  /** The coverage of the interval. It is at least the level. */\n  readonly coverage: number;\n}\n\n/**\n * This function gives the ratio of each case with a distribution-free interval of its median, in the order of\n * `caseRatios`. It needs enough rounds for the level: 6 rounds or more for the level 0.95.\n */\nexport function caseIntervals(records: readonly PairedTime[], level = 0.95): CaseInterval[] {\n  const logs = new Map<string, number[]>();\n  for (const r of records) {\n    const list = logs.get(r.case) ?? [];\n    list.push(Math.log(r.b / r.a));\n    logs.set(r.case, list);\n  }\n  return caseRatios(records).map((c) => {\n    const interval = medianInterval(logs.get(c.case)!, level);\n    return { ...c, low: Math.exp(interval.low), high: Math.exp(interval.high), coverage: interval.coverage };\n  });\n}\n\n/** The three outcomes of a limit verdict. */\nexport type LimitOutcome = \"within\" | \"beyond\" | \"cannot decide\";\n\n/** The options of a limit verdict. */\nexport interface LimitOptions extends BandOptions {\n  /** The largest ratio `b / a` that the claim permits, for example 2. */\n  readonly limit: number;\n  /** The level of the interval of each case. The default is 0.95. */\n  readonly caseLevel?: number;\n}\n\n/** The verdict of one case against the limit. */\nexport interface CaseLimit extends CaseInterval {\n  /** The high end of the interval of the case, times the high end of the A/A band. */\n  readonly bound: number;\n  /** The low end of the interval of the case, times the low end of the A/A band. */\n  readonly floor: number;\n  readonly outcome: LimitOutcome;\n}\n\n/** A limit verdict, with the facts that it uses. */\nexport interface LimitVerdict {\n  readonly outcome: LimitOutcome;\n  /** The reason of the outcome, in one sentence. */\n  readonly reason: string;\n  readonly limit: number;\n  readonly cases: readonly CaseLimit[];\n  /** The band of the A/A arm, or undefined when the A/A arm has too few cases. */\n  readonly band: Band | undefined;\n}\n\n/**\n * This function gives the verdict of the claim \"the ratio `b / a` of each case is at most `limit`\". The A/A band widens\n * the interval of each case: the bound is the high end times the high end of the band.\n * - A case is `within` when its bound is at most the limit, and `beyond` when its floor is above the limit.\n * - The verdict is `beyond` when a case is `beyond`, and `within` when each case is `within`.\n * - Each other verdict is `cannot decide`, and so is an A/A arm with too few cases or a candidate with no cases.\n */\nexport function limitVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: LimitOptions): LimitVerdict {\n  const { limit } = options;\n  if (!(limit > 0 && Number.isFinite(limit))) throw new RangeError(`limitVerdict: the limit is a finite number above 0, not ${limit}.`);\n  const intervals = caseIntervals(candidate, options.caseLevel);\n  const minimum = options.minimum ?? 5;\n  if (intervals.length === 0) return { outcome: \"cannot decide\", reason: \"The candidate has no cases.\", limit, cases: [], band: undefined };\n  const aaCount = caseRatios(aa).length;\n  if (aaCount < minimum) {\n    const cases = intervals.map((c) => ({ ...c, bound: c.high, floor: c.low, outcome: \"cannot decide\" as const }));\n    return { outcome: \"cannot decide\", reason: `The A/A arm has ${aaCount} cases, and a band needs ${minimum} or more.`, limit, cases, band: undefined };\n  }\n  const band = aaBand(aa, options);\n  const cases = intervals.map((c): CaseLimit => {\n    const bound = c.high * band.high;\n    const floor = c.low * band.low;\n    const outcome: LimitOutcome = bound <= limit ? \"within\" : floor > limit ? \"beyond\" : \"cannot decide\";\n    return { ...c, bound, floor, outcome };\n  });\n  const top = cases.reduce((a, b) => (b.bound > a.bound ? b : a));\n  const beyond = cases.filter((c) => c.outcome === \"beyond\");\n  if (beyond.length > 0) {\n    return { outcome: \"beyond\", reason: `${beyond.length} of ${cases.length} cases are above the limit ${limit}, the first is ${JSON.stringify(beyond[0]!.case)}.`, limit, cases, band };\n  }\n  if (cases.every((c) => c.outcome === \"within\")) {\n    return { outcome: \"within\", reason: `The largest bound is ${round(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.`, limit, cases, band };\n  }\n  return { outcome: \"cannot decide\", reason: `The bound of the case ${JSON.stringify(top.case)} is ${round(top.bound)}, above the limit ${limit}, and its floor is not above it.`, limit, cases, band };\n}\n\nfunction round(x: number): number {\n  return Math.round(x * 10_000) / 10_000;\n}\n","/**\n * The power test of a time verdict. It draws seeded benches with a known speedup and counts the verdicts that find it.\n * It also draws benches with equal arms and counts the false verdicts.\n */\nimport { random } from \"@mark1russell7/rigor-generators\";\nimport { firstArm, type PairedTime } from \"./paired.ts\";\nimport { type TimeOutcome, type VerdictOptions, timeVerdict } from \"./verdict.ts\";\n\n/** The options of a power test. */\nexport interface PowerOptions {\n  /** The count of the cases of the candidate and of the A/A arm. */\n  readonly cases: number;\n  /** The count of the rounds of each case. The default is 5. */\n  readonly rounds?: number;\n  /** The true ratio of the candidate time to the baseline time, for example 0.93 for a speedup of 7%. */\n  readonly ratio: number;\n  /** The standard deviation of the log time of one arm in one round. The default is 0.1. */\n  readonly sigma?: number;\n  /** The count of the seeded benches. The default is 200. */\n  readonly trials?: number;\n  /** The seed of the draws. The default is 1. */\n  readonly seed?: number | string;\n  /** The options of the verdict of each bench. */\n  readonly verdict?: VerdictOptions;\n}\n\n/** The result of a power test: the share of each outcome over the trials. */\nexport interface Power {\n  readonly trials: number;\n  readonly outcomes: Readonly<Record<TimeOutcome, number>>;\n  /** The share of the trials whose outcome is the true one: `faster` for a ratio below 1, `slower` above 1, `same` at 1. */\n  readonly power: number;\n}\n\n/**\n * This function draws `trials` benches and gives the share of each verdict. Each case has its own log scale.\n * Each arm of each round adds a normal noise of `sigma` to the log time.\n */\nexport function verdictPower(options: PowerOptions): Power {\n  const rounds = options.rounds ?? 5;\n  const sigma = options.sigma ?? 0.1;\n  const trials = options.trials ?? 200;\n  if (!Number.isSafeInteger(options.cases) || options.cases < 1) throw new RangeError(`verdictPower: cases is a whole number of 1 or more, not ${options.cases}.`);\n  if (!(options.ratio > 0 && Number.isFinite(options.ratio))) throw new RangeError(`verdictPower: the ratio is a finite number above 0, not ${options.ratio}.`);\n  const r = random(options.seed ?? 1);\n  const bench = (ratio: number, prefix: string): PairedTime[] => {\n    const out: PairedTime[] = [];\n    for (let c = 0; c < options.cases; c++) {\n      const scale = 10 * Math.exp(r.normal());\n      for (let round = 0; round < rounds; round++) {\n        const a = scale * Math.exp(r.normal(0, sigma));\n        const b = scale * ratio * Math.exp(r.normal(0, sigma));\n        out.push({ case: `${prefix}${c}`, round, first: firstArm(round, c), a, b });\n      }\n    }\n    return out;\n  };\n  const truth: TimeOutcome = options.ratio < 1 ? \"faster\" : options.ratio > 1 ? \"slower\" : \"same\";\n  const counts: Record<TimeOutcome, number> = { faster: 0, slower: 0, same: 0, \"cannot decide\": 0 };\n  for (let t = 0; t < trials; t++) {\n    const aa = bench(1, \"aa\");\n    const candidate = bench(options.ratio, \"c\");\n    counts[timeVerdict(candidate, aa, options.verdict).outcome]++;\n  }\n  const outcomes = Object.fromEntries(Object.entries(counts).map(([k, n]) => [k, n / trials])) as Record<TimeOutcome, number>;\n  return { trials, outcomes, power: outcomes[truth] };\n}\n","/**\n * The timed rounds of one case in one process. The two arms first run for a warm-up time. Then the count of calls of\n * one batch grows until a batch of arm `a` takes the target time. Each round times one batch of each arm, in the order\n * of `firstArm`. Start this function in a fresh process for each case.\n */\nimport { performance } from \"node:perf_hooks\";\nimport { type Arm, type PairedTime, firstArm } from \"./paired.ts\";\n\n/** The options of the timed rounds of one case. */\nexport interface RoundsOptions {\n  /** The name of the case. */\n  readonly case: string;\n  /** The count of the rounds. */\n  readonly rounds: number;\n  /** The offset of `firstArm`: give the index of the case, so half of the cases start with each arm. The default is 0. */\n  readonly offset?: number;\n  /** The least time in milliseconds of one batch of arm `a`. The default is 10. */\n  readonly targetMs?: number;\n  /** The time in milliseconds of the warm-up of the two arms. The default is 1000. */\n  readonly warmMs?: number;\n  /** The clock in milliseconds. The default is `performance.now`. */\n  readonly now?: () => number;\n}\n\n/** The result of the timed rounds of one case. */\nexport interface Rounds {\n  readonly records: PairedTime[];\n  /** The count of calls of each arm in one batch. */\n  readonly count: number;\n  /** The sum of the numbers that the arms gave. The engine cannot remove the work of a call, because this sum uses it. */\n  readonly sink: number;\n}\n\n/**\n * This function times the arms `a` and `b` of one case, and gives one `PairedTime` record for each round. The time of\n * a record is the time of one batch, in milliseconds. Each call of an arm gives a number, and the sum of these numbers\n * keeps the work.\n */\nexport function timeRounds(arms: Readonly<Record<Arm, () => number>>, options: RoundsOptions): Rounds {\n  const { rounds } = options;\n  const offset = options.offset ?? 0;\n  const targetMs = options.targetMs ?? 10;\n  const warmMs = options.warmMs ?? 1000;\n  const now = options.now ?? (() => performance.now());\n  if (!Number.isSafeInteger(rounds) || rounds < 1) throw new RangeError(`timeRounds: the rounds are a whole number of 1 or more, not ${rounds}.`);\n  if (!Number.isSafeInteger(offset)) throw new RangeError(`timeRounds: the offset is a whole number, not ${offset}.`);\n  if (!(targetMs > 0) || !(warmMs >= 0)) throw new RangeError(\"timeRounds: the target time is above 0, and the warm-up time is 0 or more.\");\n  let sink = 0;\n  const batch = (arm: Arm, count: number): number => {\n    const run = arms[arm];\n    const start = now();\n    for (let i = 0; i < count; i++) sink += run();\n    return now() - start;\n  };\n  let count = 1;\n  const warmUntil = now() + warmMs;\n  while (now() < warmUntil) {\n    if (batch(\"a\", count) < targetMs / 4) count *= 2;\n    batch(\"b\", count);\n  }\n  while (batch(\"a\", count) < targetMs) count *= 2;\n  const records: PairedTime[] = [];\n  for (let round = 0; round < rounds; round++) {\n    const first = firstArm(round, offset);\n    const second: Arm = first === \"a\" ? \"b\" : \"a\";\n    const times = { [first]: batch(first, count), [second]: batch(second, count) } as Record<Arm, number>;\n    records.push({ case: options.case, round, first, a: times.a, b: times.b });\n  }\n  return { records, count, sink };\n}\n"],"mappings":";;;;;;;;;;;;AA0BA,SAAgB,SAAS,OAAe,SAAS,GAAQ;CACvD,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG,MAAM,IAAI,WAAW,2DAA2D,MAAM,EAAE;CACvI,IAAI,CAAC,OAAO,cAAc,MAAM,GAAG,MAAM,IAAI,WAAW,+CAA+C,OAAO,EAAE;CAChH,SAAU,QAAQ,UAAU,IAAK,KAAK,MAAM,IAAI,MAAM;AACxD;;AAGA,SAAgB,gBAAgB,QAAgC;CAC9D,MAAM,QAAQ,0BAA0B,KAAK,UAAU,OAAO,IAAI,EAAE,UAAU,OAAO;CACrF,IAAI,OAAO,OAAO,SAAS,YAAY,OAAO,SAAS,IAAI,MAAM,IAAI,WAAW,0DAA0D;CAC1I,IAAI,CAAC,OAAO,cAAc,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG,MAAM,IAAI,WAAW,uBAAuB,MAAM,4CAA4C;CAC3J,IAAI,OAAO,UAAU,OAAO,OAAO,UAAU,KAAK,MAAM,IAAI,WAAW,uBAAuB,MAAM,qCAAqC,OAAO,OAAO,KAAK,EAAE,EAAE;CAChK,KAAK,MAAM,OAAO,CAAC,KAAK,GAAG,GAAY;EACrC,MAAM,KAAK,OAAO;EAClB,IAAI,EAAE,OAAO,OAAO,YAAY,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,MAAM,IAAI,WAAW,uBAAuB,MAAM,oBAAoB,IAAI,mCAAmC,OAAO,EAAE,EAAE,EAAE;CAC5L;CACA,OAAO;AACT;;;;;;AAeA,SAAgB,WAAW,SAA6C;CACtE,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,UAAU,SAAS;EAC5B,gBAAgB,MAAM;EACtB,MAAM,OAAO,MAAM,IAAI,OAAO,IAAI,KAAK,CAAC;EACxC,KAAK,KAAK,MAAM;EAChB,MAAM,IAAI,OAAO,MAAM,IAAI;CAC7B;CACA,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU;EAEtC,IAAI,IADe,IAAI,KAAK,KAAK,MAAM,EAAE,KAAK,CACrC,CAAC,CAAC,SAAS,KAAK,QAAQ,MAAM,IAAI,WAAW,wBAAwB,KAAK,UAAU,IAAI,EAAE,6BAA6B;EAChI,MAAM,SAAS,KAAK,QAAQ,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC;EACnD,IAAI,KAAK,IAAI,IAAI,SAAS,KAAK,MAAM,IAAI,GACvC,MAAM,IAAI,WAAW,wBAAwB,KAAK,UAAU,IAAI,EAAE,wBAAwB,OAAO,MAAM,KAAK,OAAO,qCAAqC;EAE1J,OAAO;GAAE,MAAM;GAAM,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,MAAM,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;GAAG,QAAQ,KAAK;EAAO;CAC1G,CAAC;AACH;;;;;;;;AC7BA,SAAgB,OAAO,IAA2B,UAAuB,CAAC,GAAS;CACjF,MAAM,SAAS,WAAW,EAAE;CAC5B,MAAM,UAAU,QAAQ,WAAW;CACnC,IAAI,OAAO,SAAS,SAAS,MAAM,IAAI,WAAW,2BAA2B,OAAO,OAAO,2BAA2B,QAAQ,UAAU;CACxI,MAAM,OAAO,OAAO,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;CAChD,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI;CACJ,IAAI,SAAS,QAAQ,QAAQ,SAAS,KAAK,IAAI,KAAK,GAAG,GAAG,KAAK;MAC1D;EACH,MAAM,WAAW,gBAAgB,MAAM;GAAE,GAAG,QAAQ;GAAW;EAAM,CAAC;EACtE,QAAQ,KAAK,IAAI,KAAK,IAAI,SAAS,GAAG,GAAG,KAAK,IAAI,SAAS,IAAI,CAAC;CAClE;CACA,OAAO;EAAE,OAAO,OAAO;EAAQ;EAAM;EAAO,KAAK,KAAK,IAAI,CAAC,KAAK;EAAG,MAAM,KAAK,IAAI,KAAK;EAAG,QAAQ,KAAK,IAAI,SAAS,MAAM,EAAG,CAAC;CAAE;AAClI;;;;;AAMA,SAAgB,SAAS,IAA2B,QAAQ,IAAa;CACvE,MAAM,SAAS,WAAW,EAAE;CAC5B,IAAI,OAAO,WAAW,GAAG,MAAM,IAAI,WAAW,qCAAqC;CACnF,OAAO,SAAS,QAAQ,KAAK;AAC/B;AAEA,SAAS,SAAS,QAA8B,OAAuB;CACrE,OAAO,SAAS,OAAO,KAAK,MAAM,KAAK,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,CAAC,GAAG,KAAK;AACvE;;;;;;;AAoCA,SAAgB,WAAW,MAAmB,QAAQ,IAAW;CAC/D,OAAO,OAAO,MAAM,KAAK,CAAC,CAAC;AAC7B;AAEA,SAAS,OAAO,MAAmB,OAAiD;CAClF,MAAM,QAAQ,KAAK,SAAS;CAC5B,MAAM,UAAU,KAAK,WAAW;CAChC,IAAI,EAAE,SAAS,KAAK,OAAO,SAAS,KAAK,IAAI,MAAM,IAAI,WAAW,8DAA8D,MAAM,EAAE;CACxI,IAAI,CAAC,OAAO,cAAc,OAAO,KAAK,UAAU,GAAG,MAAM,IAAI,WAAW,+DAA+D,QAAQ,EAAE;CACjJ,MAAM,SAAS,WAAW,KAAK,OAAO;CACtC,MAAM,SAAS,OAAO,UAAU,UAAU,SAAS,QAAQ,KAAK,IAAI,KAAA;CACpE,MAAM,QAAQ,KAAK,IAAI,OAAO,UAAU,CAAC;CACzC,MAAM,SAAS,OAAO,WAAW,IAAI,IAAI,KAAK,IAAI,SAAS,OAAO,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC,GAAG,EAAG,CAAC;CACrG,MAAM,OAAa;EAAE,OAAO,OAAO;EAAQ,MAAM;EAAQ;EAAO,KAAK,KAAK,IAAI,CAAC,KAAK;EAAG,MAAM,KAAK,IAAI,KAAK;EAAG;CAAO;CACrH,MAAM,OAAO,KAAK,QAAQ,KAAA,IAAY,KAAA,IAAY,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;CACtF,OAAO;EAAE;EAAM,OAAO;GAAE,OAAO,OAAO;GAAQ;GAAM;GAAQ;GAAO,QAAQ,WAAW,KAAA,KAAa,SAAS;EAAM;CAAE;AACtH;;;;;;;;;;AA8EA,SAAgB,YAAY,WAAkC,IAA2B,UAA0B,CAAC,GAAgB;CAClI,MAAM,QAAQ,QAAQ,eAAe;CACrC,IAAI,QAAQ,aAAa,KAAA,KAAa,EAAE,QAAQ,YAAY,KAAK,OAAO,SAAS,QAAQ,QAAQ,IAC/F,MAAM,IAAI,WAAW,mEAAmE,QAAQ,SAAS,EAAE;CAE7G,MAAM,OAAO,QAAQ,SAAS,KAAA,IAAY,KAAA,IAAY,OAAO,QAAQ,MAAM,KAAK;CAChF,IAAI,UAAU,YAAY,WAAW,IAAI,SAAS,MAAM,IAAI;CAC5D,IAAI,SAAS,KAAA,GAAW,UAAU;EAAE,GAAG;EAAS,MAAM,KAAK;CAAM;CACjE,IAAI,QAAQ,aAAa,KAAA,GAAW;EAClC,MAAM,MAAM,QAAQ;EACpB,MAAM,WAAW,WAAW,EAAE;EAC9B,MAAM,SAAS,SAAS,WAAW,IAAI,KAAA,IAAY,SAAS,UAAU,KAAK;EAC3E,MAAM,QAAoB;GAAE;GAAQ;GAAK,OAAO,WAAW,KAAA,KAAa,SAAS;EAAI;EACrF,UAAU;GAAE,GAAG;GAAS;EAAM;EAE9B,IAAI,MAAM,SAAS,QAAQ,SAAS,KAAA,GAAW;GAE7C,MAAM,SAAS,GADH,WAAW,KAAA,IAAY,yEAAyE,gCAAgCA,QAAM,MAAM,EAAE,wBAAwB,IAAI,GAChK,oCAAoC,QAAQ,QAAQ;GAC1E,UAAU;IAAE,GAAG;IAAS,SAAS;IAAiB;GAAO;EAC3D;CACF;CACA,OAAO;AACT;;AAGA,SAAS,YAAY,WAAkC,IAA2B,SAAyB,QAAuC;CAChJ,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,WAAW,SAAS;CAClC,MAAM,WAAW,WAAW,EAAE;CAC9B,MAAM,UAAU,QAAQ,WAAW;CACnC,MAAM,OAAO;EAAE;EAAO;EAAM,UAAU,KAAA;EAAW,MAAM,KAAA;EAAW,YAAY,KAAA;CAAU;CACxF,IAAI,MAAM,WAAW,GAAG,OAAO;EAAE,GAAG;EAAM,SAAS;EAAiB,QAAQ;CAA8B;CAC1G,IAAI,SAAS,SAAS,SAAS;EAC7B,MAAM,WAAW,gBAAgB,MAAM,KAAK,MAAM,EAAE,KAAK,GAAG,QAAQ,SAAS;EAC7E,OAAO;GAAE,GAAG;GAAM;GAAU,SAAS;GAAiB,QAAQ,mBAAmB,SAAS,OAAO,2BAA2B,QAAQ;EAAW;CACjJ;CACA,MAAM,OAAO,UAAU,OAAO,IAAI,OAAO;CAEzC,MAAM,QAAQ;EAAE;EAAO;EAAM;EAAM,YADhB,cAAc,MAAM,KAAK,MAAM,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,GAAG;GAAE,QAAQ;GAAS,OAAO,KAAK;EAAM,CACnE;CAAE;CAC9C,IAAI,SAAS,cAAc;EACzB,MAAM,QAAQ,QAAQ,UAAU;EAChC,IAAI,EAAE,SAAS,IAAI,MAAM,IAAI,WAAW,6CAA6C,MAAM,EAAE;EAC7F,MAAM,SAAS,WAAW,KAAA,IAAY,QAAQ,KAAK,IAAI,OAAO,OAAO,KAAK;EAC1E,MAAM,QAAQ,WAAW,KAAA,IAAY,SAASA,QAAM,MAAM;EAC1D,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;EAC/C,MAAM,SAAS,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;EACpD,MAAM,IAAI,aAAa,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,QAAQ,SAAS;EACvF,MAAM,WAAqB;GAAE,GAAG;GAAG,UAAU,KAAK,IAAI,EAAE,QAAQ;GAAG,KAAK,KAAK,IAAI,EAAE,GAAG;GAAG,MAAM,KAAK,IAAI,EAAE,IAAI;EAAE;EAChH,MAAM,OAAO,gEAAgEA,QAAM,SAAS,GAAG,EAAE,IAAIA,QAAM,SAAS,IAAI,EAAE;EAC1H,IAAI,SAAS,OAAO,GAAG,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAU,QAAQ,GAAG,KAAK;EAA2B;EAClH,IAAI,SAAS,MAAM,GAAG,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAU,QAAQ,GAAG,KAAK;EAA2B;EACjH,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,QAAQ,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAQ,QAAQ,GAAG,KAAK,oCAAoC,MAAM;EAAG;EACrJ,OAAO;GAAE,GAAG;GAAO;GAAU,SAAS;GAAiB,QAAQ,GAAG,KAAK,kDAAkD,MAAM;EAAG;CACpI;CACA,MAAM,WAAW,gBAAgB,MAAM,KAAK,MAAM,EAAE,KAAK,GAAG,QAAQ,SAAS;CAC7E,MAAM,OAAO,wCAAwCA,QAAM,SAAS,GAAG,EAAE,IAAIA,QAAM,SAAS,IAAI,EAAE,0BAA0BA,QAAM,KAAK,GAAG,EAAE,IAAIA,QAAM,KAAK,IAAI,EAAE;CACjK,IAAI,SAAS,OAAO,KAAK,KAAK,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAU,QAAQ,GAAG,KAAK;CAAkC;CAChI,IAAI,SAAS,MAAM,KAAK,MAAM,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAU,QAAQ,GAAG,KAAK;CAAkC;CAChI,IAAI,SAAS,OAAO,KAAK,OAAO,SAAS,QAAQ,KAAK,MAAM,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAQ,QAAQ,GAAG,KAAK;CAA+B;CACzJ,OAAO;EAAE,GAAG;EAAO;EAAU,SAAS;EAAiB,QAAQ,GAAG,KAAK;CAA2C;AACpH;AAEA,SAASA,QAAM,GAAmB;CAChC,OAAO,KAAK,MAAM,IAAI,GAAM,IAAI;AAClC;;;;;;;;;;ACvPA,SAAgB,cAAc,SAAgC,QAAQ,KAAsB;CAC1F,MAAM,uBAAO,IAAI,IAAsB;CACvC,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,OAAO,KAAK,IAAI,EAAE,IAAI,KAAK,CAAC;EAClC,KAAK,KAAK,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;EAC7B,KAAK,IAAI,EAAE,MAAM,IAAI;CACvB;CACA,OAAO,WAAW,OAAO,CAAC,CAAC,KAAK,MAAM;EACpC,MAAM,WAAW,eAAe,KAAK,IAAI,EAAE,IAAI,GAAI,KAAK;EACxD,OAAO;GAAE,GAAG;GAAG,KAAK,KAAK,IAAI,SAAS,GAAG;GAAG,MAAM,KAAK,IAAI,SAAS,IAAI;GAAG,UAAU,SAAS;EAAS;CACzG,CAAC;AACH;;;;;;;;AAwCA,SAAgB,aAAa,WAAkC,IAA2B,SAAqC;CAC7H,MAAM,EAAE,UAAU;CAClB,IAAI,EAAE,QAAQ,KAAK,OAAO,SAAS,KAAK,IAAI,MAAM,IAAI,WAAW,2DAA2D,MAAM,EAAE;CACpI,MAAM,YAAY,cAAc,WAAW,QAAQ,SAAS;CAC5D,MAAM,UAAU,QAAQ,WAAW;CACnC,IAAI,UAAU,WAAW,GAAG,OAAO;EAAE,SAAS;EAAiB,QAAQ;EAA+B;EAAO,OAAO,CAAC;EAAG,MAAM,KAAA;CAAU;CACxI,MAAM,UAAU,WAAW,EAAE,CAAC,CAAC;CAC/B,IAAI,UAAU,SAAS;EACrB,MAAM,QAAQ,UAAU,KAAK,OAAO;GAAE,GAAG;GAAG,OAAO,EAAE;GAAM,OAAO,EAAE;GAAK,SAAS;EAAyB,EAAE;EAC7G,OAAO;GAAE,SAAS;GAAiB,QAAQ,mBAAmB,QAAQ,2BAA2B,QAAQ;GAAY;GAAO;GAAO,MAAM,KAAA;EAAU;CACrJ;CACA,MAAM,OAAO,OAAO,IAAI,OAAO;CAC/B,MAAM,QAAQ,UAAU,KAAK,MAAiB;EAC5C,MAAM,QAAQ,EAAE,OAAO,KAAK;EAC5B,MAAM,QAAQ,EAAE,MAAM,KAAK;EAC3B,MAAM,UAAwB,SAAS,QAAQ,WAAW,QAAQ,QAAQ,WAAW;EACrF,OAAO;GAAE,GAAG;GAAG;GAAO;GAAO;EAAQ;CACvC,CAAC;CACD,MAAM,MAAM,MAAM,QAAQ,GAAG,MAAO,EAAE,QAAQ,EAAE,QAAQ,IAAI,CAAE;CAC9D,MAAM,SAAS,MAAM,QAAQ,MAAM,EAAE,YAAY,QAAQ;CACzD,IAAI,OAAO,SAAS,GAClB,OAAO;EAAE,SAAS;EAAU,QAAQ,GAAG,OAAO,OAAO,MAAM,MAAM,OAAO,6BAA6B,MAAM,iBAAiB,KAAK,UAAU,OAAO,EAAE,CAAE,IAAI,EAAE;EAAI;EAAO;EAAO;CAAK;CAErL,IAAI,MAAM,OAAO,MAAM,EAAE,YAAY,QAAQ,GAC3C,OAAO;EAAE,SAAS;EAAU,QAAQ,wBAAwB,MAAM,IAAI,KAAK,EAAE,eAAe,KAAK,UAAU,IAAI,IAAI,EAAE,gCAAgC,MAAM;EAAI;EAAO;EAAO;CAAK;CAEpL,OAAO;EAAE,SAAS;EAAiB,QAAQ,yBAAyB,KAAK,UAAU,IAAI,IAAI,EAAE,MAAM,MAAM,IAAI,KAAK,EAAE,oBAAoB,MAAM;EAAmC;EAAO;EAAO;CAAK;AACtM;AAEA,SAAS,MAAM,GAAmB;CAChC,OAAO,KAAK,MAAM,IAAI,GAAM,IAAI;AAClC;;;;;;;;;;ACjEA,SAAgB,aAAa,SAA8B;CACzD,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,SAAS,QAAQ,UAAU;CACjC,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,KAAK,QAAQ,QAAQ,GAAG,MAAM,IAAI,WAAW,2DAA2D,QAAQ,MAAM,EAAE;CAC/J,IAAI,EAAE,QAAQ,QAAQ,KAAK,OAAO,SAAS,QAAQ,KAAK,IAAI,MAAM,IAAI,WAAW,2DAA2D,QAAQ,MAAM,EAAE;CAC5J,MAAM,IAAI,OAAO,QAAQ,QAAQ,CAAC;CAClC,MAAM,SAAS,OAAe,WAAiC;EAC7D,MAAM,MAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,OAAO,KAAK;GACtC,MAAM,QAAQ,KAAK,KAAK,IAAI,EAAE,OAAO,CAAC;GACtC,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,SAAS;IAC3C,MAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IAC7C,MAAM,IAAI,QAAQ,QAAQ,KAAK,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IACrD,IAAI,KAAK;KAAE,MAAM,GAAG,SAAS;KAAK;KAAO,OAAO,SAAS,OAAO,CAAC;KAAG;KAAG;IAAE,CAAC;GAC5E;EACF;EACA,OAAO;CACT;CACA,MAAM,QAAqB,QAAQ,QAAQ,IAAI,WAAW,QAAQ,QAAQ,IAAI,WAAW;CACzF,MAAM,SAAsC;EAAE,QAAQ;EAAG,QAAQ;EAAG,MAAM;EAAG,iBAAiB;CAAE;CAChG,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,KAAK;EAC/B,MAAM,KAAK,MAAM,GAAG,IAAI;EACxB,MAAM,YAAY,MAAM,QAAQ,OAAO,GAAG;EAC1C,OAAO,YAAY,WAAW,IAAI,QAAQ,OAAO,CAAC,CAAC,QAAQ;CAC7D;CACA,MAAM,WAAW,OAAO,YAAY,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;CAC3F,OAAO;EAAE;EAAQ;EAAU,OAAO,SAAS;CAAO;AACpD;;;;;;;;;;;;AC5BA,SAAgB,WAAW,MAA2C,SAAgC;CACpG,MAAM,EAAE,WAAW;CACnB,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,MAAM,QAAQ,cAAc,YAAY,IAAI;CAClD,IAAI,CAAC,OAAO,cAAc,MAAM,KAAK,SAAS,GAAG,MAAM,IAAI,WAAW,+DAA+D,OAAO,EAAE;CAC9I,IAAI,CAAC,OAAO,cAAc,MAAM,GAAG,MAAM,IAAI,WAAW,iDAAiD,OAAO,EAAE;CAClH,IAAI,EAAE,WAAW,MAAM,EAAE,UAAU,IAAI,MAAM,IAAI,WAAW,4EAA4E;CACxI,IAAI,OAAO;CACX,MAAM,SAAS,KAAU,UAA0B;EACjD,MAAM,MAAM,KAAK;EACjB,MAAM,QAAQ,IAAI;EAClB,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,IAAI;EAC5C,OAAO,IAAI,IAAI;CACjB;CACA,IAAI,QAAQ;CACZ,MAAM,YAAY,IAAI,IAAI;CAC1B,OAAO,IAAI,IAAI,WAAW;EACxB,IAAI,MAAM,KAAK,KAAK,IAAI,WAAW,GAAG,SAAS;EAC/C,MAAM,KAAK,KAAK;CAClB;CACA,OAAO,MAAM,KAAK,KAAK,IAAI,UAAU,SAAS;CAC9C,MAAM,UAAwB,CAAC;CAC/B,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,SAAS;EAC3C,MAAM,QAAQ,SAAS,OAAO,MAAM;EACpC,MAAM,SAAc,UAAU,MAAM,MAAM;EAC1C,MAAM,QAAQ;IAAG,QAAQ,MAAM,OAAO,KAAK;IAAI,SAAS,MAAM,QAAQ,KAAK;EAAE;EAC7E,QAAQ,KAAK;GAAE,MAAM,QAAQ;GAAM;GAAO;GAAO,GAAG,MAAM;GAAG,GAAG,MAAM;EAAE,CAAC;CAC3E;CACA,OAAO;EAAE;EAAS;EAAO;CAAK;AAChC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/rigor-bench",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "The paired-time records of a bench: the order that turns each round, the ratio of each case, and the time verdict against the band of an A/A arm.",
   "keywords": [
     "benchmark",
@@ -35,10 +35,10 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@mark1russell7/rigor-statistics": "^0.1.3"
+    "@mark1russell7/rigor-statistics": "^0.1.5"
   },
   "devDependencies": {
-    "@mark1russell7/rigor-generators": "^0.1.0",
+    "@mark1russell7/rigor-generators": "^0.1.1",
     "@mark1russell7/cue": "^1.0.2",
     "@types/node": "^26.6.5",
     "tsdown": "^0.23.1",