@mark1russell7/rigor-bench 0.1.2 → 0.2.1

package/build/index.d.ts

@@ -1,38 +1,5 @@
+import { a as checkPairedTime, i as caseRatios, n as CaseRatio, o as firstArm, r as PairedTime, t as Arm } from "./paired.js";
 import { BootstrapOptions, Comparison, Interval } from "@mark1russell7/rigor-statistics";
-/** The two arms of a pair. Arm `a` is the baseline, and arm `b` is the candidate. */
-type Arm = "a" | "b";
-/** One round of one pair on one case: the times of the two arms in milliseconds, and the arm that ran first. */
-interface PairedTime {
-  /** The name of the case: one input of the bench. */
-  readonly case: string;
-  /** The number of the round, a whole number of 0 or more. */
-  readonly round: number;
-  readonly first: Arm;
-  /** The time of the baseline arm, in milliseconds. */
-  readonly a: number;
-  /** The time of the candidate arm, in milliseconds. */
-  readonly b: number;
-}
-/**
- * This function gives the arm that runs first in a round. The order turns each round. A case with an odd `offset`
- * starts with arm `b`, so half of the cases start with each arm.
- */
-export declare function firstArm(round: number, offset?: number): Arm;
-/** This function examines one record. A time that is not a finite number above 0 is an error. */
-export declare function checkPairedTime(record: PairedTime): PairedTime;
-/** The ratio of one case: the candidate time over the baseline time, over the rounds of the case. */
-interface CaseRatio {
-  readonly case: string;
-  /** The value `exp(median(log(b / a)))` over the rounds. For an odd count of rounds, it is the median of the ratios. */
-  readonly ratio: number;
-  readonly rounds: number;
-}
-/**
- * This function gives the ratio of each case, in the order of the first record of each case.
- * It examines each record. It refuses a case with 2 records of one round.
- * It also refuses a case whose two orders differ in count by more than 1, because that order did not turn each round.
- */
-export declare function caseRatios(records: readonly PairedTime[]): CaseRatio[];
 /**
  * The two kinds of an A/A band.
  * - The band `case` holds the ratio of one A/A case. It is wide, so it finds only a large change.
@@ -120,10 +87,14 @@ type VerdictRule = "band" | "difference";
 interface VerdictOptions extends BandOptions {
   /** The rule of the verdict. The default is `difference`, from the power test of `verdictPower`. */
   readonly rule?: VerdictRule;
-  /** The half width on a log scale of the ratios that the rule `difference` calls `same`. The default is 0.05. */
+  /**
+   * The half width on a log scale of the ratios that the rule `difference` calls `same`. The default is 0.05.
+   * An interval in the margin is `same`, also when it is wholly below 1.
+   */
   readonly margin?: number;
   /**
    * The noise cap on the A/A spread of the run, on a log scale. Above the cap, the outcome is `cannot decide`.
+   * The cap also limits the bias of the A/A arm: an A/A median ratio out of `[exp(-noiseCap), exp(noiseCap)]` gives `cannot decide`.
    * The default is no cap.
    */
   readonly noiseCap?: number;
@@ -135,13 +106,20 @@ interface VerdictOptions extends BandOptions {
    */
   readonly pool?: PoolOptions;
 }
-/** The noise cap of a verdict: the A/A spread of the run and the cap. */
+/** The noise cap of a verdict: the A/A spread of the run, the A/A median ratio of the run and the cap. */
 interface NoiseCheck {
   /** The A/A spread of the run, or undefined when the A/A arm has no cases. */
   readonly spread: number | undefined;
   readonly cap: number;
   /** This value is `true` when the spread is above the cap, or when the A/A arm has no cases. */
   readonly noisy: boolean;
+  /** The median ratio of the A/A cases of the run, or undefined when the A/A arm has no cases. A value far from 1 shows a bias of the order or of the arm slots. */
+  readonly aaMedian: number | undefined;
+  /**
+   * This value is `true` when the A/A median ratio of the run is out of `[exp(-cap), exp(cap)]`.
+   * With a spread level of 0.5 or more, a biased run is also noisy. The check decides alone only below that level.
+   */
+  readonly biased: boolean;
 }
 /** A time verdict, with the facts that it uses. */
 interface TimeVerdict {
@@ -171,11 +149,14 @@ interface TimeVerdict {
 }
 /**
  * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.
- * - The rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.
- *   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.
+ * - The rule `difference` gives `same` when the interval is in `[exp(-margin), exp(margin)]`. This test comes first, so
+ *   an interval in the margin and below 1 is `same`, not `faster`.
+ * - Out of the margin, the rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.
+ *   With the margin 0, each outcome stays the same as in 0.1.2.
  * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.
  * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.
  * - With `noiseCap`, an A/A spread of the run above the cap gives `cannot decide`.
+ * - With `noiseCap`, an A/A median ratio of the run out of `[exp(-noiseCap), exp(noiseCap)]` also gives `cannot decide`.
  * - With `pool`, the pooled band replaces the band of the A/A arm of the run. The run still needs `minimum` A/A cases.
  */
 export declare function timeVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options?: VerdictOptions): TimeVerdict;
@@ -227,6 +208,62 @@ interface LimitVerdict {
  * - Each other verdict is `cannot decide`, and so is an A/A arm with too few cases or a candidate with no cases.
  */
 export declare function limitVerdict(candidate: readonly PairedTime[], aa: readonly PairedTime[], options: LimitOptions): LimitVerdict;
+/** The records of each case with a turned order, and the count of the records that the balance removed. */
+interface Balanced<T extends PairedTime> {
+  readonly records: T[];
+  /** The count of the removed records. It is 0 when each case has a turned order. */
+  readonly trimmed: number;
+}
+/**
+ * This function removes records until the two orders of each case differ in count by 1 or less.
+ * It removes the records of the larger order with the highest rounds first. The other records keep their order.
+ * A job that stopped early, or a missing job, can leave a case with 2 more rounds of one order. `caseRatios` refuses such a case.
+ */
+export declare function balanceOrders<T extends PairedTime>(records: readonly T[]): Balanced<T>;
+/** The records of one job: the candidate pair and the A/A pair. */
+interface JobRecords {
+  /** The label of the job, for example the index of its CI job. A record with a field `job` has this label, or `pooledVerdict` refuses it. */
+  readonly job: string;
+  readonly candidate: readonly PairedTime[];
+  readonly aa: readonly PairedTime[];
+}
+/** The verdict of one job alone. */
+interface JobVerdict {
+  readonly job: string;
+  readonly verdict: TimeVerdict;
+}
+/** The pooled verdict over the jobs, with the verdict of each job alone. */
+interface PooledVerdict {
+  /** The outcome of the pooled verdict, or `cannot decide` when a job decides the opposite direction. */
+  readonly outcome: TimeOutcome;
+  /** The verdict of the merged and balanced records of all jobs. */
+  readonly pooled: TimeVerdict;
+  /** The verdict of each job alone, with the options of the pooled verdict. */
+  readonly jobs: readonly JobVerdict[];
+  /** The jobs whose own outcome is the opposite direction of the pooled outcome. */
+  readonly opposed: string[];
+  /** The count of the records that `balanceOrders` removed from the merged candidate and A/A records. */
+  readonly trimmed: number;
+  /** The reason of the pooled verdict, and the opposed jobs. */
+  readonly reason: string;
+}
+/**
+ * This function gives the time verdict over the records of all jobs.
+ * - It merges the records of the jobs, and it balances the merged candidate and A/A records with `balanceOrders`.
+ * - The verdict of each job alone uses the same options. A job with `cannot decide` or `same` does not change the outcome.
+ * - A job that decides the opposite direction of the pooled outcome makes the outcome `cannot decide`.
+ * - It refuses two jobs with one label, and two jobs with a record of one case and one round.
+ */
+export declare function pooledVerdict(jobs: readonly JobRecords[], options?: VerdictOptions): PooledVerdict;
+/** The pairs of a run: the records of each pair, by the id of the pair. */
+interface RunPairs {
+  readonly pairs: Readonly<Record<string, readonly PairedTime[]>>;
+}
+/**
+ * This function gives the verdict of each candidate pair of a run against its A/A pair `aaPair`.
+ * It balances the records of each pair with `balanceOrders` first. It refuses an `aaPair` that the run does not have.
+ */
+export declare function armVerdicts(run: RunPairs, aaPair: string, options?: VerdictOptions): Record<string, TimeVerdict>;
 /** The options of a power test. */
 interface PowerOptions {
   /** The count of the cases of the candidate and of the A/A arm. */
@@ -248,7 +285,11 @@ interface PowerOptions {
 interface Power {
   readonly trials: number;
   readonly outcomes: Readonly<Record<TimeOutcome, number>>;
-  /** The share of the trials whose outcome is the true one: `faster` for a ratio below 1, `slower` above 1, `same` at 1. */
+  /**
+   * The share of the trials whose outcome is the true one. For the rule `difference`, a ratio in the margin of the verdict has the true outcome `same`.
+   * That margin is the larger of `margin` and the noise of the pooled band. The pool is the same in each trial, so each trial has the same margin.
+   * Out of that margin, and for the rule `band`, the true outcome is `faster` below 1, `slower` above 1 and `same` at 1.
+   */
   readonly power: number;
 }
 /**
@@ -285,5 +326,5 @@ interface Rounds {
  * keeps the work.
  */
 export declare function timeRounds(arms: Readonly<Record<Arm, () => number>>, options: RoundsOptions): Rounds;
-export type { Arm, Band, BandKind, BandOptions, CaseInterval, CaseLimit, CaseRatio, LimitOptions, LimitOutcome, LimitVerdict, NoiseCheck, PairedTime, PoolFacts, PoolOptions, Power, PowerOptions, Rounds, RoundsOptions, TimeOutcome, TimeVerdict, VerdictOptions, VerdictRule };
+export { type Arm, type Balanced, type Band, type BandKind, type BandOptions, type CaseInterval, type CaseLimit, type CaseRatio, type JobRecords, type JobVerdict, type LimitOptions, type LimitOutcome, type LimitVerdict, type NoiseCheck, type PairedTime, type PoolFacts, type PoolOptions, type PooledVerdict, type Power, type PowerOptions, type Rounds, type RoundsOptions, type RunPairs, type TimeOutcome, type TimeVerdict, type VerdictOptions, type VerdictRule, caseRatios, checkPairedTime, firstArm };
 //# sourceMappingURL=index.d.ts.map

package/build/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/paired.ts","../src/verdict.ts","../src/limit.ts","../src/power.ts","../src/rounds.ts"],"mappings":";;KAOY;;UAGK;;WAEN;;WAEA;WACA,OAAO;;WAEP;;WAEA;;;;;;wBAOK,SAAS,eAAe,kBAAa;;wBAOrC,gBAAgB,QAAQ,aAAa;;UAapC;WACN;;WAEA;WACA;;;;;;;wBAQK,WAAW,kBAAkB,eAAe;;;;;;KC7ChD;;UAGK;;WAEN,OAAO;;WAEP;;WAEA;;WAEA,YAAY;;;UAIN;;WAEN;WACA,MAAM;;;;;;WAMN;;WAEA;WACA;;WAEA;;;wBAIK,OAAO,aAAa,cAAc,UAAS,cAAmB;;;;;wBAoB9D,SAAS,aAAa,cAAc;;UAWnC;;WAEN,kBAAkB;;WAElB;;WAEA;;;;;WAKA,OAAO,QAAQ;;;UAIT;;WAEN;;WAEA;;WAEA;WACA;;WAEA;;;;;;;;wBASK,WAAW,MAAM,aAAa,iBAAc;;KAmBhD;;;;;;;KAQA;;UAGK,uBAAuB;;WAE7B,OAAO;;WAEP;;;;;WAKA;;WAEA;;;;;WAKA,OAAO;;;UAID;;WAEN;WACA;;WAEA;;;UAIM;WACN,SAAS;;WAET;WACA,gBAAgB;WAChB,MAAM;;;;;;;WAON,UAAU;;WAEV,MAAM;;;;;WAKN,YAAY;;WAEZ,OAAO;;WAEP,QAAQ;;;;;;;;;;;wBAYH,YAAY,oBAAoB,cAAc,aAAa,cAAc,UAAS,iBAAsB;;UCnMvG,qBAAqB;;WAE3B;WACA;;WAEA;;;;;;wBAOK,cAAc,kBAAkB,cAAc,iBAAe;;KAcjE;;UAGK,qBAAqB;;WAE3B;;WAEA;;;UAIM,kBAAkB;;WAExB;;WAEA;WACA,SAAS;;;UAIH;WACN,SAAS;;WAET;WACA;WACA,gBAAgB;;WAEhB,MAAM;;;;;;;;;wBAUD,aAAa,oBAAoB,cAAc,aAAa,cAAc,SAAS,eAAe;;UC/DjG;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,UAAU;;;UAIJ;WACN;WACA,UAAU,SAAS,OAAO;;WAE1B;;;;;;wBAOK,aAAa,SAAS,eAAe;;UC7BpC;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN,SAAS;;WAET;;WAEA;;;;;;;wBAQK,WAAW,MAAM,SAAS,OAAO,qBAAqB,SAAS,gBAAgB"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/verdict.ts","../src/limit.ts","../src/merge.ts","../src/power.ts","../src/rounds.ts"],"mappings":";;;;;;;KAaY;;UAGK;;WAEN,OAAO;;WAEP;;WAEA;;WAEA,YAAY;;;UAIN;;WAEN;WACA,MAAM;;;;;;WAMN;;WAEA;WACA;;WAEA;;;wBAIK,OAAO,aAAa,cAAc,UAAS,cAAmB;;;;;wBAoB9D,SAAS,aAAa,cAAc;;UAWnC;;WAEN,kBAAkB;;WAElB;;WAEA;;;;;WAKA,OAAO,QAAQ;;;UAIT;;WAEN;;WAEA;;WAEA;WACA;;WAEA;;;;;;;;wBASK,WAAW,MAAM,aAAa,iBAAc;;KAmBhD;;;;;;;KAQA;;UAGK,uBAAuB;;WAE7B,OAAO;;;;;WAKP;;;;;;WAMA;;WAEA;;;;;WAKA,OAAO;;;UAID;;WAEN;WACA;;WAEA;;WAEA;;;;;WAKA;;;UAIM;WACN,SAAS;;WAET;WACA,gBAAgB;WAChB,MAAM;;;;;;;WAON,UAAU;;WAEV,MAAM;;;;;WAKN,YAAY;;WAEZ,OAAO;;WAEP,QAAQ;;;;;;;;;;;;;;wBAeH,YAAY,oBAAoB,cAAc,aAAa,cAAc,UAAS,iBAAsB;;UCjNvG,qBAAqB;;WAE3B;WACA;;WAEA;;;;;;wBAOK,cAAc,kBAAkB,cAAc,iBAAe;;KAcjE;;UAGK,qBAAqB;;WAE3B;;WAEA;;;UAIM,kBAAkB;;WAExB;;WAEA;WACA,SAAS;;;UAIH;WACN,SAAS;;WAET;WACA;WACA,gBAAgB;;WAEhB,MAAM;;;;;;;;;wBAUD,aAAa,oBAAoB,cAAc,aAAa,cAAc,SAAS,eAAe;;UChEjG,SAAS,UAAU;WACzB,SAAS;;WAET;;;;;;;wBAQK,cAAc,UAAU,YAAY,kBAAkB,MAAM,SAAS;;UAkBpE;;WAEN;WACA,oBAAoB;WACpB,aAAa;;;UAIP;WACN;WACA,SAAS;;;UAIH;;WAEN,SAAS;;WAET,QAAQ;;WAER,eAAe;;WAEf;;WAEA;;WAEA;;;;;;;;;wBAYK,cAAc,eAAe,cAAc,UAAS,iBAAsB;;UA4CzE;WACN,OAAO,SAAS,wBAAwB;;;;;;wBAOnC,YAAY,KAAK,UAAU,gBAAgB,UAAS,iBAAsB,eAAe;;UCtHxF;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,UAAU;;;UAIJ;WACN;WACA,UAAU,SAAS,OAAO;;;;;;WAM1B;;;;;;wBAOK,aAAa,SAAS,eAAe;;UCjCpC;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN,SAAS;;WAET;;WAEA;;;;;;;wBAQK,WAAW,MAAM,SAAS,OAAO,qBAAqB,SAAS,gBAAgB"}

package/build/index.js

@@ -1,56 +1,7 @@
+import { n as checkPairedTime, r as firstArm, t as caseRatios } from "./paired.js";
 import { bootstrapMedian, bootstrapTwo, compareRatios, median, medianInterval, quantile } from "@mark1russell7/rigor-statistics";
 import { random } from "@mark1russell7/rigor-generators";
 import { performance } from "node:perf_hooks";
-// src/paired.ts
-/**
-* The paired-time record of a bench. Each round runs the two arms of a pair on one case, one after the other, in a
-* fresh process. The order turns each round, so a warm cache or a slow start does not always help the same arm.
-*/
-/**
-* This function gives the arm that runs first in a round. The order turns each round. A case with an odd `offset`
-* starts with arm `b`, so half of the cases start with each arm.
-*/
-function firstArm(round, offset = 0) {
-	if (!Number.isSafeInteger(round) || round < 0) throw new RangeError(`firstArm: the round is a whole number of 0 or more, not ${round}.`);
-	if (!Number.isSafeInteger(offset)) throw new RangeError(`firstArm: the offset is a whole number, not ${offset}.`);
-	return ((round + offset) % 2 + 2) % 2 === 0 ? "a" : "b";
-}
-/** This function examines one record. A time that is not a finite number above 0 is an error. */
-function checkPairedTime(record) {
-	const where = `the record of the case ${JSON.stringify(record.case)}, round ${record.round}`;
-	if (typeof record.case !== "string" || record.case === "") throw new RangeError("checkPairedTime: the case is a string that is not empty.");
-	if (!Number.isSafeInteger(record.round) || record.round < 0) throw new RangeError(`checkPairedTime: in ${where}, the round is a whole number of 0 or more.`);
-	if (record.first !== "a" && record.first !== "b") throw new RangeError(`checkPairedTime: in ${where}, the first arm is "a" or "b", not ${String(record.first)}.`);
-	for (const arm of ["a", "b"]) {
-		const ms = record[arm];
-		if (!(typeof ms === "number" && Number.isFinite(ms) && ms > 0)) throw new RangeError(`checkPairedTime: in ${where}, the time of arm ${arm} is a finite number above 0, not ${String(ms)}.`);
-	}
-	return record;
-}
-/**
-* This function gives the ratio of each case, in the order of the first record of each case.
-* It examines each record. It refuses a case with 2 records of one round.
-* It also refuses a case whose two orders differ in count by more than 1, because that order did not turn each round.
-*/
-function caseRatios(records) {
-	const cases = /* @__PURE__ */ new Map();
-	for (const record of records) {
-		checkPairedTime(record);
-		const list = cases.get(record.case) ?? [];
-		list.push(record);
-		cases.set(record.case, list);
-	}
-	return [...cases].map(([name, list]) => {
-		if (new Set(list.map((r) => r.round)).size !== list.length) throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} has 2 records of one round.`);
-		const aFirst = list.filter((r) => r.first === "a").length;
-		if (Math.abs(2 * aFirst - list.length) > 1) throw new RangeError(`caseRatios: the case ${JSON.stringify(name)} starts with arm a in ${aFirst} of ${list.length} rounds. The order turns each round.`);
-		return {
-			case: name,
-			ratio: Math.exp(median(list.map((r) => Math.log(r.b / r.a)))),
-			rounds: list.length
-		};
-	});
-}
 // src/verdict.ts
 /**
 * The time verdict of a candidate against a baseline, with the noise of an A/A arm. An A/A pair runs the same code in
@@ -135,17 +86,20 @@ function poolOf(pool, level) {
 }
 /**
 * This function gives the time verdict of the candidate arm `b` against the baseline arm `a`.
-* - The rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.
-*   It gives `same` when the interval is in `[exp(-margin), exp(margin)]`.
+* - The rule `difference` gives `same` when the interval is in `[exp(-margin), exp(margin)]`. This test comes first, so
+*   an interval in the margin and below 1 is `same`, not `faster`.
+* - Out of the margin, the rule `difference` gives `faster` when the whole interval is below 1, and `slower` when it is above 1.
+*   With the margin 0, each outcome stays the same as in 0.1.2.
 * - The rule `band` gives `faster` below the band, `slower` above the band, and `same` in the band.
 * - Each other case is `cannot decide`, and so is an A/A arm with too few cases.
 * - With `noiseCap`, an A/A spread of the run above the cap gives `cannot decide`.
+* - With `noiseCap`, an A/A median ratio of the run out of `[exp(-noiseCap), exp(noiseCap)]` also gives `cannot decide`.
 * - With `pool`, the pooled band replaces the band of the A/A arm of the run. The run still needs `minimum` A/A cases.
 */
 function timeVerdict(candidate, aa, options = {}) {
 	const level = options.spreadLevel ?? .9;
 	if (options.noiseCap !== void 0 && !(options.noiseCap >= 0 && Number.isFinite(options.noiseCap))) throw new RangeError(`timeVerdict: the noise cap is a finite number of 0 or more, not ${options.noiseCap}.`);
-	const pool = options.pool === void 0 ? void 0 : poolOf(options.pool, level);
+	const pool = poolFor(options);
 	let verdict = ruleVerdict(candidate, aa, options, pool?.band);
 	if (pool !== void 0) verdict = {
 		...verdict,
@@ -155,10 +109,13 @@ function timeVerdict(candidate, aa, options = {}) {
 		const cap = options.noiseCap;
 		const aaRatios = caseRatios(aa);
 		const spread = aaRatios.length === 0 ? void 0 : spreadOf(aaRatios, level);
+		const aaMedian = aaRatios.length === 0 ? void 0 : Math.exp(quantile(aaRatios.map((r) => Math.log(r.ratio)), .5));
 		const noise = {
 			spread,
 			cap,
-			noisy: spread === void 0 || spread > cap
+			noisy: spread === void 0 || spread > cap,
+			aaMedian,
+			biased: aaMedian !== void 0 && Math.abs(Math.log(aaMedian)) > cap
 		};
 		verdict = {
 			...verdict,
@@ -171,10 +128,34 @@ function timeVerdict(candidate, aa, options = {}) {
 				outcome: "cannot decide",
 				reason
 			};
+		} else if (noise.biased && verdict.band !== void 0) {
+			const reason = `The A/A median ratio of the run is ${round$1(aaMedian)}, out of [${round$1(Math.exp(-cap))}, ${round$1(Math.exp(cap))}] of the noise cap ${cap}. Without the cap, the outcome is "${verdict.outcome}".`;
+			verdict = {
+				...verdict,
+				outcome: "cannot decide",
+				reason
+			};
 		}
 	}
 	return verdict;
 }
+/** This function gives the pooled band and its facts of the options, or undefined without the option `pool`. */
+function poolFor(options) {
+	return options.pool === void 0 ? void 0 : poolOf(options.pool, options.spreadLevel ?? .9);
+}
+/** This function gives the margin of the rule `difference`: the larger of `margin` and the noise of the pooled band. It refuses a margin below 0. */
+function marginOf(options, pooled) {
+	const given = options.margin ?? .05;
+	if (!(given >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${given}.`);
+	return pooled === void 0 ? given : Math.max(given, pooled.noise);
+}
+/**
+* This function gives the margin that the rule `difference` of `timeVerdict` uses with these options. It uses the same code as the verdict.
+* The pooled band comes from the records of the option `pool`, not from the records of the run.
+*/
+function verdictMargin(options = {}) {
+	return marginOf(options, poolFor(options)?.band);
+}
 /** This function gives the outcome of the rule. A pooled band replaces the band of the A/A arm, and it widens the margin of the rule `difference`. */
 function ruleVerdict(candidate, aa, options, pooled) {
 	const rule = options.rule ?? "difference";
@@ -213,9 +194,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
 		})
 	};
 	if (rule === "difference") {
-		const given = options.margin ?? .05;
-		if (!(given >= 0)) throw new RangeError(`timeVerdict: the margin is 0 or more, not ${given}.`);
-		const margin = pooled === void 0 ? given : Math.max(given, pooled.noise);
+		const margin = marginOf(options, pooled);
 		const shown = pooled === void 0 ? margin : round$1(margin);
 		const logs = cases.map((c) => Math.log(c.ratio));
 		const aaLogs = aaRatios.map((c) => Math.log(c.ratio));
@@ -227,6 +206,12 @@ function ruleVerdict(candidate, aa, options, pooled) {
 			high: Math.exp(d.high)
 		};
 		const span = `The interval of the candidate median over the A/A median is [${round$1(interval.low)}, ${round$1(interval.high)}].`;
+		if (d.low >= -margin && d.high <= margin) return {
+			...facts,
+			interval,
+			outcome: "same",
+			reason: `${span} The interval is in the margin of ${shown}.`
+		};
 		if (interval.high < 1) return {
 			...facts,
 			interval,
@@ -239,12 +224,6 @@ function ruleVerdict(candidate, aa, options, pooled) {
 			outcome: "slower",
 			reason: `${span} The interval is above 1.`
 		};
-		if (d.low >= -margin && d.high <= margin) return {
-			...facts,
-			interval,
-			outcome: "same",
-			reason: `${span} The interval is in the margin of ${shown}.`
-		};
 		return {
 			...facts,
 			interval,
@@ -382,6 +361,104 @@ function limitVerdict(candidate, aa, options) {
 function round(x) {
 	return Math.round(x * 1e4) / 1e4;
 }
+// src/merge.ts
+/**
+* This function removes records until the two orders of each case differ in count by 1 or less.
+* It removes the records of the larger order with the highest rounds first. The other records keep their order.
+* A job that stopped early, or a missing job, can leave a case with 2 more rounds of one order. `caseRatios` refuses such a case.
+*/
+function balanceOrders(records) {
+	const byCase = /* @__PURE__ */ new Map();
+	records.forEach((record, index) => {
+		const orders = byCase.get(record.case) ?? {
+			a: [],
+			b: []
+		};
+		orders[record.first === "b" ? "b" : "a"].push(index);
+		byCase.set(record.case, orders);
+	});
+	const drop = /* @__PURE__ */ new Set();
+	const highFirst = (x, y) => records[y].round - records[x].round;
+	for (const { a, b } of byCase.values()) {
+		const [more, fewer] = a.length > b.length ? [a, b] : [b, a];
+		const extra = more.length - fewer.length - 1;
+		if (extra > 0) for (const index of more.toSorted(highFirst).slice(0, extra)) drop.add(index);
+	}
+	return {
+		records: records.filter((_, index) => !drop.has(index)),
+		trimmed: drop.size
+	};
+}
+const opposite = {
+	faster: "slower",
+	slower: "faster"
+};
+/**
+* This function gives the time verdict over the records of all jobs.
+* - It merges the records of the jobs, and it balances the merged candidate and A/A records with `balanceOrders`.
+* - The verdict of each job alone uses the same options. A job with `cannot decide` or `same` does not change the outcome.
+* - A job that decides the opposite direction of the pooled outcome makes the outcome `cannot decide`.
+* - It refuses two jobs with one label, and two jobs with a record of one case and one round.
+*/
+function pooledVerdict(jobs, options = {}) {
+	const labels = /* @__PURE__ */ new Set();
+	for (const { job } of jobs) {
+		if (labels.has(job)) throw new RangeError(`pooledVerdict: two jobs have the label ${JSON.stringify(job)}.`);
+		labels.add(job);
+	}
+	for (const { job, candidate: c, aa: a } of jobs) for (const r of [...c, ...a]) if (r.job !== void 0 && r.job !== job) throw new RangeError(`pooledVerdict: a record of the job ${JSON.stringify(job)} has the job ${JSON.stringify(r.job)}.`);
+	uniqueRounds(jobs, "candidate");
+	uniqueRounds(jobs, "aa");
+	const candidate = balanceOrders(jobs.flatMap((j) => j.candidate));
+	const aa = balanceOrders(jobs.flatMap((j) => j.aa));
+	const pooled = timeVerdict(candidate.records, aa.records, options);
+	const each = jobs.map(({ job, candidate: c, aa: a }) => ({
+		job,
+		verdict: timeVerdict(balanceOrders(c).records, balanceOrders(a).records, options)
+	}));
+	const against = opposite[pooled.outcome];
+	const opposed = against === void 0 ? [] : each.filter((j) => j.verdict.outcome === against).map((j) => j.job);
+	const trimmed = candidate.trimmed + aa.trimmed;
+	if (opposed.length === 0) return {
+		outcome: pooled.outcome,
+		pooled,
+		jobs: each,
+		opposed,
+		trimmed,
+		reason: pooled.reason
+	};
+	const names = opposed.map((j) => JSON.stringify(j)).join(", ");
+	return {
+		outcome: "cannot decide",
+		pooled,
+		jobs: each,
+		opposed,
+		trimmed,
+		reason: `${pooled.reason} The pooled outcome is "${pooled.outcome}", but the jobs ${names} give "${against}". Thus the outcome is "cannot decide".`
+	};
+}
+/** This function refuses two jobs with a record of one case and one round in one pair. */
+function uniqueRounds(jobs, pair) {
+	const seen = /* @__PURE__ */ new Map();
+	for (const { job, [pair]: records } of jobs) for (const r of records) {
+		const key = `${r.case}\u0000${r.round}`;
+		const other = seen.get(key);
+		if (other !== void 0 && other !== job) throw new RangeError(`pooledVerdict: the jobs ${JSON.stringify(other)} and ${JSON.stringify(job)} both have the round ${r.round} of the case ${JSON.stringify(r.case)} in the ${pair} pair. Set roundOffset of each job to job * rounds.`);
+		seen.set(key, job);
+	}
+}
+/**
+* This function gives the verdict of each candidate pair of a run against its A/A pair `aaPair`.
+* It balances the records of each pair with `balanceOrders` first. It refuses an `aaPair` that the run does not have.
+*/
+function armVerdicts(run, aaPair, options = {}) {
+	const aaRecords = run.pairs[aaPair];
+	if (aaRecords === void 0) throw new RangeError(`armVerdicts: the run has no pair ${JSON.stringify(aaPair)}.`);
+	const aa = balanceOrders(aaRecords).records;
+	const out = {};
+	for (const [pair, records] of Object.entries(run.pairs)) if (pair !== aaPair) out[pair] = timeVerdict(balanceOrders(records).records, aa, options);
+	return out;
+}
 // src/power.ts
 /**
 * The power test of a time verdict. It draws seeded benches with a known speedup and counts the verdicts that find it.
@@ -416,7 +493,7 @@ function verdictPower(options) {
 		}
 		return out;
 	};
-	const truth = options.ratio < 1 ? "faster" : options.ratio > 1 ? "slower" : "same";
+	const truth = trueOutcome(options.ratio, options.verdict);
 	const counts = {
 		faster: 0,
 		slower: 0,
@@ -435,6 +512,11 @@ function verdictPower(options) {
 		power: outcomes[truth]
 	};
 }
+/** This function gives the true outcome of a ratio. The margin of the rule `difference` comes first, as in `timeVerdict`. */
+function trueOutcome(ratio, verdict = {}) {
+	if ((verdict.rule ?? "difference") === "difference" && Math.abs(Math.log(ratio)) <= verdictMargin(verdict)) return "same";
+	return ratio < 1 ? "faster" : ratio > 1 ? "slower" : "same";
+}
 // src/rounds.ts
 /**
 * The timed rounds of one case in one process. The two arms first run for a warm-up time. Then the count of calls of
@@ -491,6 +573,6 @@ function timeRounds(arms, options) {
 		sink
 	};
 }
-export { aaBand, aaSpread, caseIntervals, caseRatios, checkPairedTime, firstArm, limitVerdict, pooledBand, timeRounds, timeVerdict, verdictPower };
+export { aaBand, aaSpread, armVerdicts, balanceOrders, caseIntervals, caseRatios, checkPairedTime, firstArm, limitVerdict, pooledBand, pooledVerdict, timeRounds, timeVerdict, verdictPower };
 
 //# sourceMappingURL=index.js.map