@mark1russell7/rigor-bench 0.2.14 → 0.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -1
- package/build/index.d.ts +427 -234
- package/build/index.d.ts.map +1 -1
- package/build/index.js +414 -13
- package/build/index.js.map +1 -1
- package/package.json +2 -2
package/build/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":["Comparison"],"sources":["../src/budget.ts","../src/
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":["Comparison"],"sources":["../src/budget.ts","../src/verdict.ts","../src/limit.ts","../src/merge.ts","../src/gate.ts","../src/control.ts","../src/ladder.ts","../src/tail.ts","../src/measures.ts","../src/power.ts","../src/rounds.ts"],"mappings":";;;UAQiB;;WAEN;;WAEA;WACA;;;KAIC;;UAGK;;WAEN;;WAEA;;WAEA;;;UAIM;WACN;;WAEA;;WAEA;WACA;;WAEA;;WAEA;;WAEA;WACA;;WAEA;WACA,SAAS;;WAET;;;UAIM;WACN,SAAS;;WAET;WACA;WACA;WACA;WACA,gBAAgB;;;;;;;;;;;;wBAyDX,cAAc,iBAAiB,eAAe,aAAa,eAAe,SAAS,gBAAgB;;UAyBlG;WACN;WACA;;;;;;qBAOE,uBAAuB;;UAOnB;;WAEN;;WAEA;;WAEA,iBAAiB;;WAEjB;;;UAIM;WACN;WACA;;;UAIM,2BAA2B;;WAEjC;;WAEA,eAAe;;WAEf;;;;;;wBAOK,gBAAgB,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;wBAoDhC,mBAAmB,iBAAiB,eAAe,aAAa,eAAe,SAAS,qBAAqB;;;;;;KCnOjH;;UAGK;;WAEN,OAAO;;WAEP;;WAEA;;WAEA,YAAY;;;UAIN;;WAEN;WACA,MAAM;;;;;;WAMN;;WAEA;WACA;;WAEA;;;wBAIK,OAAO,aAAa,cAAc,UAAS,cAAmB;;;;;wBAoB9D,SAAS,aAAa,cAAc;;UAWnC;;WAEN,kBAAkB;;WAElB;;WAEA;;;;;WAKA,OAAO,QAAQ;;;UAIT;;WAEN;;WAEA;;WAEA;WACA;;WAEA;;;;;;;;wBASK,WAAW,MAAM,aAAa,iBAAc;;KAmBhD;;;;;;;KAQA;;UAGK,uBAAuB;;WAE7B,OAAO;;;;;WAKP;;;;;;WAMA;;WAEA;;;;;WAKA,OAAO;;;UAID;;WAEN;WACA;;WAEA;;WAEA;;;;;WAKA;;;UAIM;WACN,SAAS;;WAET;WACA,gBAAgB;WAChB,MAAM;;;;;;;WAON,UAAU;;WAEV,MAAM;;;;;WAKN,YAAYA;;WAEZ,OAAO;;WAEP,QAAQ;;;;;;;;;;;;;;wBAeH,YAAY,oBAAoB,cAAc,aAAa,cAAc,UAAS,iBAAsB;;;;;;;wBA+CxG,cAAc,UAAS;;UChQtB,qBAAqB;;WAE3B;WACA;;WAEA;;;;;;wBAOK,cAAc,kBAAkB,cAAc,iBAAe;;KAcjE;;UAGK,qBAAqB;;WAE3B;;WAEA;;;;;WAKA;;;UAIM,kBAAkB;;WAExB;;WAEA;WACA,SAAS;;;UAIH;WACN,SAAS;;WAET;WACA;WACA,gBAAgB;;WAEhB,MAAM;;WAEN;;WAEA;;;;;;;;;;wBAWK,aAAa,oBAAoB,cAAc,aAAa,cAAc,SAAS,eAAe;;UCzEjG,SAAS,UAAU;WACzB,SAAS;;WAET;;;;;;;wBAQK,cAAc,UAAU,YAAY,kBAAkB,MAAM,SAAS;;UAkBpE;;WAEN;WACA,oBAAoB;WACpB,aAAa;;WAEb;aAA8B;;;;UAIxB;WACN;WACA,WAAW;WACX,IAAI;;;UAIE;;WAEN,WAAW;;WAEX,IAAI;;WAEJ,eAAe;;WAEf;;WAEA;;;;;;;;wBASK,YAAY,eAAe,eAAe;;UAsEzC;WACN;WACA,SAAS;;;UAIH;;WAEN,SAAS;;WAET,QAAQ;;WAER,eAAe;;WAEf;;WAEA;;WAEA;;WAEA;;;;;;;;wBAWK,cAAc,eAAe,cAAc,UAAS,iBAAsB;;UASzE;WACN;WACA,SAAS;;;UAIH;;WAEN,SAAS;;WAET,QAAQ;;WAER,eAAe;;WAEf;;WAEA;;WAEA;;WAEA;;;;;;;;wBAWK,mBAAmB,eAAe,cAAc,SAAS,eAAe;;UASvE;WACN,OAAO,SAAS,wBAAwB;;;;;;wBAOnC,YAAY,KAAK,UAAU,gBAAgB,UAAS,iBAAsB,eAAe;;KC7N7F;;UAGK;WACN;WACA,SAAS;;;UAIH;WACN,SAAS;;WAET;;WAEA;;WAEA;;;;;;;;wBASK,OAAO,gBAAgB,oBAAoB;;KAW/C;;qBAKC;;UAGI;;WAEN,mBAAmB;;WAEnB;;WAEA;;WAEA;;;UAIM;WACN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM,iBAAiB;WACvB,MAAM;WACN;;WAEA;;WAEA,eAAe;;WAEf;;;UAIM,yBAAyB;WAC/B;WACA;;WAEA;WACA,gBAAgB;;WAEhB,eAAe;;;;;;;;;;;;;;;;;wBA8EV,iBAAiB,eAAe,cAAc,SAAS,mBAAmB;;UC7KzE;;WAEN;;WAEA;;WAEA;;WAEA;;;UAIM;;WAEN;;WAEA;;;UAIM;WACN;;WAEA;;WAEA;;WAEA;;;UAIM,uBAAuB;;WAE7B;;WAEA;WACA;;;UAIM,iBAAiB;WACvB;;WAEA;;;KAIC,cAAc,iBAAiB;;UAG1B;;WAEN;WACA,kBAAkB;;;;;;;;;;;;wBAuFb,cAAc,eAAe,cAAc,UAAS,iBAAsB;;qBC3I7E;;UAGI;WACN;;WAEA;;;UAIM;WACN;WACA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN;WACA;;WAEA;;;UAIM;WACN;WACA;WACA;;WAEA;;WAEA;;;;;;;;;;;;UAaM;WACN;WACA;;WAEA;;WAEA;;WAEA;;WAEA;;;KAIC,aAAa,eAAe,iBAAiB,aAAa;;UAGrD;;WAEN;;WAEA,IAAI,SAAS;WACb,kBAAkB;;WAElB;;WAEA;;;UAIM;WACN,eAAe;;WAEf;WACA,qBAAqB;WACrB,gBAAgB;;;KAIf;WACG;WAAyB;WAA0B,iBAAiB;WAAwB,aAAa;;WACzG;WAA2B;WAA0B;WAAoC;;WACzF;WAAuB;WAA0B;WAA2B;;WAC5E;WAAwB;WAA0B;;;UAGhD;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;WACA,mBAAmB;;;UAIb;WACN;WACA,SAAS;;WAET;;;UAIM;WACN;WACA;WACA;WACA,SAAS;;WAET;WACA,kBAAkB;;WAElB;WACA;WACA;;;UAIM;WACN;;WAEA;;WAEA;;WAEA;;WAEA;WACA,gBAAgB;;;UAIV;;WAEN;;WAEA;;;;;;;wBAkDK,YAAY,QAAQ;;wBAyFpB,YAAY,MAAM,MAAM,QAAQ,wBAAwB,SAAS,gBAAgB;;;;;;wBA6BjF,cAAc,QAAQ,QAAQ,kBAAkB,cAAc,SAAS,gBAAgB;;UClUtF;WACN;;WAEA;WACA;;;UAIM;;WAEN;;WAEA;;;UAIM;;WAEN,OAAO,cAAc,qBAAqB;;WAE1C;;;UAIM,yBAAyB;;;;;WAK/B;;WAEA;;;;;WAKA;;;UAIM,sBAAsB;;WAE5B;WACA,OAAO;;WAEP;;WAEA;WACA,SAAS;;;UAIH;WACN;;WAEA;;;UAIM;;WAEN;;WAEA,gBAAgB;;;UAIV;WACN,SAAS;;WAET;WACA,gBAAgB;;WAEhB;WACA;WACA;;;;;;wBAqBK,UAAU,kBAAkB,cAAc,gBAAW;;;;;;;;;;;;;;;wBAoCrD,iBAAiB,oBAAoB,cAAc,aAAa,cAAc,SAAS,mBAAmB;;;;;;;;;;wBA6C1G,eAAe,aAAa,cAAc,SAAS,kBAAkB;;UC9KpE;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;;WAEN;;WAEA;;;UAIM;WACN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,YAAY;;;UAIN;WACN,gBAAgB;;WAEhB;;WAEA;;WAEA;;;UAIM;WACN;WACA;WACA;;WAEA;;WAEA;WACA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN;WACA;WACA;;;;;WAKA;;WAEA;;WAEA,sBAAsB;;WAEtB;;WAEA;;WAEA,kBAAkB;;;UAIZ;WACN;WACA;WACA;WACA;;;UAIM;WACN;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN;;;;;WAKA;;;UAIM;WACN;WACA;WACA,QAAQ;WACR,mBAAmB;;;UAIb;WACN;WACA;;WAEA;;WAEA;;WAEA;WACA,iBAAiB;WACjB,iBAAiB;WACjB,kBAAkB;;WAElB;;WAEA,MAAM;;;KAIL;;UAGK;WACN,MAAM;WACN;WACA;WACA;;WAEA;;WAEA;;WAEA,sBAAsB;;;UAIhB;;WAEN;WACA,mBAAmB;;WAEnB,gBAAgB;WAChB,mBAAmB;;;UAIb;;WAEN;;WAEA;;WAEA;;WAEA;aAAqB;aAAwB;eAA8B;eAA0B;;;;WAErG,MAAM;;;;;;;;;;;;;;;;;;wBAuUD,cAAc,eAAe,cAAc,SAAS,sBAAsB;;;;;;;wBAgC1E,SAAS,OAAO,eAAe,iBAAiB,eAAe,mBAAmB;;UCnjBjF;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,UAAU;;;UAIJ;WACN;WACA,UAAU,SAAS,OAAO;;;;;;WAM1B;;;;;;wBAOK,aAAa,SAAS,eAAe;;UClCpC;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;UAIM;WACN,SAAS;;WAET;;WAEA;;;;;;;wBAQK,WAAW,MAAM,SAAS,OAAO,qBAAqB,SAAS,gBAAgB"}
|
package/build/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as checkPairedTime, r as firstArm, t as caseRatios } from "./paired.js";
|
|
2
|
-
import { bootstrapMedian, bootstrapTwo, compareRatios, holm, median, medianInterval, quantile, quantileInterval, quantileSamples, signTest, signTestPower, wilson } from "@mark1russell7/rigor-statistics";
|
|
2
|
+
import { bootstrapMedian, bootstrapTwo, compareRatios, holm, median, medianInterval, partialConjunction, quantile, quantileInterval, quantileSamples, signTest, signTestPower, wilson } from "@mark1russell7/rigor-statistics";
|
|
3
3
|
import { random } from "@mark1russell7/rigor-generators";
|
|
4
4
|
// src/budget.ts
|
|
5
5
|
/**
|
|
@@ -28,7 +28,7 @@ function caseBudget(name, values, aaValues, options, level) {
|
|
|
28
28
|
const aaMedian = aaValues.length > 0 ? median(aaValues) : void 0;
|
|
29
29
|
const difference = aaMedian === void 0 ? void 0 : Math.abs(middle - aaMedian);
|
|
30
30
|
const aaCoverage = aaValues.length > 0 ? medianInterval(aaValues, level).coverage : 0;
|
|
31
|
-
const facts = `median ${round$
|
|
31
|
+
const facts = `median ${round$3(middle)} of ${values.length} rounds, interval ${round$3(interval.low)} to ${round$3(interval.high)}, limit ${limit}, A/A median ${aaMedian === void 0 ? "none" : round$3(aaMedian)} of ${aaValues.length} rounds, difference ${difference === void 0 ? "none" : round$3(difference)}`;
|
|
32
32
|
const decide = (outcome, why) => ({
|
|
33
33
|
case: name,
|
|
34
34
|
rounds: values.length,
|
|
@@ -180,7 +180,7 @@ function budgetLooksVerdict(rounds, aa, options) {
|
|
|
180
180
|
const looks = options.looks ?? BUDGET_LOOKS;
|
|
181
181
|
const level = options.level ?? .95;
|
|
182
182
|
const spent = budgetLookSpend(looks);
|
|
183
|
-
if (spent > 1 - level) throw new RangeError(`budgetLooksVerdict: the looks spend ${round$
|
|
183
|
+
if (spent > 1 - level) throw new RangeError(`budgetLooksVerdict: the looks spend ${round$3(spent)}, over the ${round$3(1 - level)} of the level ${level}.`);
|
|
184
184
|
if (!(limit >= 0 && Number.isFinite(limit))) throw new RangeError(`budgetLooksVerdict: the limit is finite and 0 or more, not ${limit}.`);
|
|
185
185
|
if (!(maxDifference >= 0 && Number.isFinite(maxDifference))) throw new RangeError(`budgetLooksVerdict: the largest difference is finite and 0 or more, not ${maxDifference}.`);
|
|
186
186
|
const byRound = (list) => [...list].sort((a, b) => a.round - b.round);
|
|
@@ -209,9 +209,219 @@ function budgetLooksVerdict(rounds, aa, options) {
|
|
|
209
209
|
fails: verdict.cases.some((c) => c.outcome === "beyond")
|
|
210
210
|
};
|
|
211
211
|
}
|
|
212
|
-
function round$
|
|
212
|
+
function round$3(x) {
|
|
213
213
|
return Math.round(x * 1e4) / 1e4;
|
|
214
214
|
}
|
|
215
|
+
// src/gate.ts
|
|
216
|
+
/**
|
|
217
|
+
* The gate of a bench: the result that a check of a merge reads. A verdict gives one of the words of `LimitOutcome` for each
|
|
218
|
+
* case, and the gate gives "pass", "block" or "no read". A case with no outcome does not pass.
|
|
219
|
+
*/
|
|
220
|
+
/**
|
|
221
|
+
* This function gives the gate of a list of case outcomes.
|
|
222
|
+
* - The result is "block" when a case is `beyond`.
|
|
223
|
+
* - The result is "pass" when each case is `within`.
|
|
224
|
+
* - Each other result is "no read", and so is a list with no cases. A caller does not count "no read" as a pass.
|
|
225
|
+
*/
|
|
226
|
+
function gateOf(cases) {
|
|
227
|
+
const names = (outcome) => cases.filter((c) => c.outcome === outcome).map((c) => c.case);
|
|
228
|
+
const blocks = names("beyond");
|
|
229
|
+
const noRead = names("cannot decide");
|
|
230
|
+
if (cases.length === 0) return {
|
|
231
|
+
outcome: "no read",
|
|
232
|
+
reason: "The gate has no cases.",
|
|
233
|
+
blocks,
|
|
234
|
+
noRead
|
|
235
|
+
};
|
|
236
|
+
if (blocks.length > 0) return {
|
|
237
|
+
outcome: "block",
|
|
238
|
+
reason: `${blocks.length} of ${cases.length} cases are beyond the claim. The first is ${JSON.stringify(blocks[0])}.`,
|
|
239
|
+
blocks,
|
|
240
|
+
noRead
|
|
241
|
+
};
|
|
242
|
+
if (noRead.length > 0) return {
|
|
243
|
+
outcome: "no read",
|
|
244
|
+
reason: `${noRead.length} of ${cases.length} cases have no outcome. The first is ${JSON.stringify(noRead[0])}.`,
|
|
245
|
+
blocks,
|
|
246
|
+
noRead
|
|
247
|
+
};
|
|
248
|
+
return {
|
|
249
|
+
outcome: "pass",
|
|
250
|
+
reason: `Each of the ${cases.length} cases is within the claim.`,
|
|
251
|
+
blocks,
|
|
252
|
+
noRead
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
const READ_OF = {
|
|
256
|
+
within: "not slower",
|
|
257
|
+
beyond: "slower",
|
|
258
|
+
"cannot decide": "no read"
|
|
259
|
+
};
|
|
260
|
+
/** The looks of `notSlowerVerdict`: the rounds in each job. The level of the family has an equal share at each look. */
|
|
261
|
+
const GATE_LOOKS = [
|
|
262
|
+
12,
|
|
263
|
+
24,
|
|
264
|
+
48
|
|
265
|
+
];
|
|
266
|
+
/** This function examines the options. */
|
|
267
|
+
function checkOptions(level, looks, need) {
|
|
268
|
+
if (!(level > 0 && level < 1)) throw new RangeError(`notSlowerVerdict: the level is above 0 and below 1, not ${level}.`);
|
|
269
|
+
if (looks.length === 0) throw new RangeError("notSlowerVerdict: the list of looks is empty.");
|
|
270
|
+
let last = 0;
|
|
271
|
+
for (const rounds of looks) {
|
|
272
|
+
if (!(Number.isInteger(rounds) && rounds > last && rounds % 2 === 0)) throw new RangeError(`notSlowerVerdict: the looks are even whole numbers in rising order, not ${rounds} after ${last}.`);
|
|
273
|
+
last = rounds;
|
|
274
|
+
}
|
|
275
|
+
if (!(Number.isInteger(need) && need >= 1)) throw new RangeError(`notSlowerVerdict: the count of jobs is a whole number of 1 or more, not ${need}.`);
|
|
276
|
+
}
|
|
277
|
+
/** This function gives the records of one case of one job in the order of the rounds. It refuses a round that comes twice. */
|
|
278
|
+
function casesOf(records) {
|
|
279
|
+
const out = /* @__PURE__ */ new Map();
|
|
280
|
+
for (const record of records) {
|
|
281
|
+
checkPairedTime(record);
|
|
282
|
+
const list = out.get(record.case) ?? [];
|
|
283
|
+
if (list.some((r) => r.round === record.round)) throw new RangeError(`notSlowerVerdict: the round ${record.round} of ${JSON.stringify(record.case)} comes twice.`);
|
|
284
|
+
list.push(record);
|
|
285
|
+
out.set(record.case, list);
|
|
286
|
+
}
|
|
287
|
+
for (const list of out.values()) list.sort((a, b) => a.round - b.round);
|
|
288
|
+
return out;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* This function gives the differences of the first `rounds` rounds: log(b / a) of the candidate minus log(b / a) of the A/A arm.
|
|
292
|
+
* A pair has two records with the same first arm, because a difference of two orders holds the effect of the order.
|
|
293
|
+
*/
|
|
294
|
+
function differences(candidate, aa, rounds) {
|
|
295
|
+
const out = [];
|
|
296
|
+
const ofOrder = (list, first) => list.slice(0, rounds).filter((r) => r.first === first);
|
|
297
|
+
for (const first of ["a", "b"]) {
|
|
298
|
+
const left = ofOrder(candidate, first);
|
|
299
|
+
const right = ofOrder(aa, first);
|
|
300
|
+
for (let i = 0; i < Math.min(left.length, right.length); i++) {
|
|
301
|
+
const c = left[i];
|
|
302
|
+
const a = right[i];
|
|
303
|
+
out.push(Math.log(c.b / c.a) - Math.log(a.b / a.a));
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
/** This function gives the read of one job. A job that has not reached the look gives a p of 1 in both directions. */
|
|
309
|
+
function jobRead(job, diffs, margin, rounds) {
|
|
310
|
+
if (diffs.length < rounds) return {
|
|
311
|
+
job,
|
|
312
|
+
rounds: diffs.length,
|
|
313
|
+
below: 0,
|
|
314
|
+
above: 0,
|
|
315
|
+
notSlower: 1,
|
|
316
|
+
slower: 1,
|
|
317
|
+
centre: void 0
|
|
318
|
+
};
|
|
319
|
+
const below = diffs.filter((d) => d < margin).length;
|
|
320
|
+
const above = diffs.filter((d) => d > margin).length;
|
|
321
|
+
return {
|
|
322
|
+
job,
|
|
323
|
+
rounds: diffs.length,
|
|
324
|
+
below,
|
|
325
|
+
above,
|
|
326
|
+
notSlower: signTest(below, diffs.length - below),
|
|
327
|
+
slower: signTest(above, diffs.length - above),
|
|
328
|
+
centre: median(diffs)
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
/** This function gives the words of the facts of one job, for a reason. */
|
|
332
|
+
function jobFacts(read) {
|
|
333
|
+
if (read.centre === void 0) return `job ${read.job} has ${read.rounds} pairs`;
|
|
334
|
+
return `job ${read.job} has ${read.below} of ${read.rounds} pairs below the margin, centre ${Math.round(read.centre * 1e5) / 1e3} %`;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* This function gives the gate of the claim "the candidate is not slower than the A/A arm by more than the margin" for each
|
|
338
|
+
* case. A one-sided bound answers the claim. Thus a case with its whole interval under the margin reads "not slower", also
|
|
339
|
+
* when its interval holds 1.
|
|
340
|
+
*
|
|
341
|
+
* - In each job, the sign test compares the median of the differences with the margin. It uses no random numbers.
|
|
342
|
+
* - A case reads in `jobs` jobs or more: `partialConjunction` joins the p of each job, with no condition on how they depend.
|
|
343
|
+
* - The level is for the family. Each look has `level / looks` of it, and `holm` corrects each look over all cases.
|
|
344
|
+
* - The first look with an outcome decides a case. Later rounds do not change it. Each look uses the first rounds of a job.
|
|
345
|
+
* - The result is "block" when a case reads "slower", and "pass" when each case reads "not slower".
|
|
346
|
+
* - Each other result is "no read". The field `more` gives the next look of each case. After the last look, the caller adds jobs.
|
|
347
|
+
*
|
|
348
|
+
* The level needs independent rounds. Rounds with a shared cause, for example a slow computer for one job, do not have it.
|
|
349
|
+
* A job that a caller adds after the last look is one more look, and the level does not count it.
|
|
350
|
+
*/
|
|
351
|
+
function notSlowerVerdict(jobs, options) {
|
|
352
|
+
const level = options.level ?? .05;
|
|
353
|
+
const looks = options.looks ?? GATE_LOOKS;
|
|
354
|
+
const need = options.jobs ?? 2;
|
|
355
|
+
checkOptions(level, looks, need);
|
|
356
|
+
const marginOf = (name) => {
|
|
357
|
+
const margin = typeof options.margin === "number" ? options.margin : options.margin(name);
|
|
358
|
+
if (!(margin >= 0 && Number.isFinite(margin))) throw new RangeError(`notSlowerVerdict: the margin of ${JSON.stringify(name)} is finite and 0 or more, not ${margin}.`);
|
|
359
|
+
return margin;
|
|
360
|
+
};
|
|
361
|
+
const records = jobs.map((job) => ({
|
|
362
|
+
job: job.job,
|
|
363
|
+
candidate: casesOf(job.candidate),
|
|
364
|
+
aa: casesOf(job.aa)
|
|
365
|
+
}));
|
|
366
|
+
const names = [...new Set(records.flatMap((job) => [...job.candidate.keys()]))].sort();
|
|
367
|
+
const margins = names.map(marginOf);
|
|
368
|
+
const decided = /* @__PURE__ */ new Map();
|
|
369
|
+
const reached = /* @__PURE__ */ new Map();
|
|
370
|
+
const each = level / looks.length;
|
|
371
|
+
for (const rounds of looks) {
|
|
372
|
+
const reads = names.map((name, i) => records.map((job) => jobRead(job.job, differences(job.candidate.get(name) ?? [], job.aa.get(name) ?? [], rounds), margins[i], rounds)));
|
|
373
|
+
const joined = (pick) => reads.map((list) => list.length < need ? 1 : partialConjunction(list.map(pick), need));
|
|
374
|
+
const within = holm(joined((read) => read.notSlower), each);
|
|
375
|
+
const beyond = holm(joined((read) => read.slower), each);
|
|
376
|
+
names.forEach((name, i) => {
|
|
377
|
+
const list = reads[i];
|
|
378
|
+
if (!reached.has(name) || list.some((read) => read.rounds >= rounds)) reached.set(name, list);
|
|
379
|
+
if (decided.has(name)) return;
|
|
380
|
+
if (within[i]?.reject === true) decided.set(name, {
|
|
381
|
+
outcome: "within",
|
|
382
|
+
look: rounds,
|
|
383
|
+
reads: list
|
|
384
|
+
});
|
|
385
|
+
else if (beyond[i]?.reject === true) decided.set(name, {
|
|
386
|
+
outcome: "beyond",
|
|
387
|
+
look: rounds,
|
|
388
|
+
reads: list
|
|
389
|
+
});
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
const cases = names.map((name, i) => {
|
|
393
|
+
const decision = decided.get(name);
|
|
394
|
+
const outcome = decision?.outcome ?? "cannot decide";
|
|
395
|
+
const reads = decision?.reads ?? reached.get(name) ?? [];
|
|
396
|
+
const facts = reads.map(jobFacts).join(", ");
|
|
397
|
+
const where = decision === void 0 ? `no look of ${looks.join(", ")} rounds decides it` : `the look of ${decision.look} rounds decides it`;
|
|
398
|
+
return {
|
|
399
|
+
case: name,
|
|
400
|
+
outcome,
|
|
401
|
+
read: READ_OF[outcome],
|
|
402
|
+
margin: margins[i],
|
|
403
|
+
look: decision?.look,
|
|
404
|
+
jobs: reads,
|
|
405
|
+
reason: `The case ${JSON.stringify(name)} reads "${READ_OF[outcome]}": ${where} (${facts}).`
|
|
406
|
+
};
|
|
407
|
+
});
|
|
408
|
+
const more = cases.flatMap((c) => {
|
|
409
|
+
if (c.outcome !== "cannot decide") return [];
|
|
410
|
+
const next = looks.find((rounds) => records.some((job) => differences(job.candidate.get(c.case) ?? [], job.aa.get(c.case) ?? [], rounds).length < rounds));
|
|
411
|
+
return next === void 0 ? [] : [{
|
|
412
|
+
case: c.case,
|
|
413
|
+
rounds: next
|
|
414
|
+
}];
|
|
415
|
+
});
|
|
416
|
+
return {
|
|
417
|
+
...gateOf(cases),
|
|
418
|
+
level,
|
|
419
|
+
looks,
|
|
420
|
+
need,
|
|
421
|
+
cases,
|
|
422
|
+
more
|
|
423
|
+
};
|
|
424
|
+
}
|
|
215
425
|
// src/control.ts
|
|
216
426
|
/**
|
|
217
427
|
* The load factor of a run, read from a control. Runs on shared machines differ by load. A change does not touch a control.
|
|
@@ -661,14 +871,14 @@ function timeVerdict(candidate, aa, options = {}) {
|
|
|
661
871
|
noise
|
|
662
872
|
};
|
|
663
873
|
if (noise.noisy && verdict.band !== void 0) {
|
|
664
|
-
const reason = `${spread === void 0 ? "The A/A arm of the run has no cases, so the noise cap has no spread." : `The A/A spread of the run is ${round$
|
|
874
|
+
const reason = `${spread === void 0 ? "The A/A arm of the run has no cases, so the noise cap has no spread." : `The A/A spread of the run is ${round$2(spread)}, above the noise cap ${cap}.`} Without the cap, the outcome is "${verdict.outcome}".`;
|
|
665
875
|
verdict = {
|
|
666
876
|
...verdict,
|
|
667
877
|
outcome: "cannot decide",
|
|
668
878
|
reason
|
|
669
879
|
};
|
|
670
880
|
} else if (noise.biased && verdict.band !== void 0) {
|
|
671
|
-
const reason = `The A/A median ratio of the run is ${round$
|
|
881
|
+
const reason = `The A/A median ratio of the run is ${round$2(aaMedian)}, out of [${round$2(Math.exp(-cap))}, ${round$2(Math.exp(cap))}] of the noise cap ${cap}. Without the cap, the outcome is "${verdict.outcome}".`;
|
|
672
882
|
verdict = {
|
|
673
883
|
...verdict,
|
|
674
884
|
outcome: "cannot decide",
|
|
@@ -736,7 +946,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
|
|
|
736
946
|
};
|
|
737
947
|
if (rule === "difference") {
|
|
738
948
|
const margin = marginOf(options, pooled);
|
|
739
|
-
const shown = pooled === void 0 ? margin : round$
|
|
949
|
+
const shown = pooled === void 0 ? margin : round$2(margin);
|
|
740
950
|
const logs = cases.map((c) => Math.log(c.ratio));
|
|
741
951
|
const aaLogs = aaRatios.map((c) => Math.log(c.ratio));
|
|
742
952
|
const d = bootstrapTwo(logs, aaLogs, (x, y) => median(x) - median(y), options.bootstrap);
|
|
@@ -746,7 +956,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
|
|
|
746
956
|
low: Math.exp(d.low),
|
|
747
957
|
high: Math.exp(d.high)
|
|
748
958
|
};
|
|
749
|
-
const span = `The interval of the candidate median over the A/A median is [${round$
|
|
959
|
+
const span = `The interval of the candidate median over the A/A median is [${round$2(interval.low)}, ${round$2(interval.high)}].`;
|
|
750
960
|
if (d.low >= -margin && d.high <= margin) return {
|
|
751
961
|
...facts,
|
|
752
962
|
interval,
|
|
@@ -773,7 +983,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
|
|
|
773
983
|
};
|
|
774
984
|
}
|
|
775
985
|
const interval = bootstrapMedian(cases.map((c) => c.ratio), options.bootstrap);
|
|
776
|
-
const span = `The interval of the median ratio is [${round$
|
|
986
|
+
const span = `The interval of the median ratio is [${round$2(interval.low)}, ${round$2(interval.high)}], and the A/A band is [${round$2(band.low)}, ${round$2(band.high)}].`;
|
|
777
987
|
if (interval.high < band.low) return {
|
|
778
988
|
...facts,
|
|
779
989
|
interval,
|
|
@@ -799,7 +1009,7 @@ function ruleVerdict(candidate, aa, options, pooled) {
|
|
|
799
1009
|
reason: `${span} The interval crosses an end of the band.`
|
|
800
1010
|
};
|
|
801
1011
|
}
|
|
802
|
-
function round$
|
|
1012
|
+
function round$2(x) {
|
|
803
1013
|
return Math.round(x * 1e4) / 1e4;
|
|
804
1014
|
}
|
|
805
1015
|
// src/limit.ts
|
|
@@ -904,17 +1114,208 @@ function limitVerdict(candidate, aa, options) {
|
|
|
904
1114
|
const rest = others.length === 0 ? "" : ` ${others.length} of ${cases.length} cases are not within${permit}.`;
|
|
905
1115
|
return {
|
|
906
1116
|
outcome: "within",
|
|
907
|
-
reason: `The largest bound of a case within is ${round(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.${rest}`,
|
|
1117
|
+
reason: `The largest bound of a case within is ${round$1(top.bound)} in the case ${JSON.stringify(top.case)}, and it is at most the limit ${limit}.${rest}`,
|
|
908
1118
|
...facts
|
|
909
1119
|
};
|
|
910
1120
|
}
|
|
911
1121
|
const top = cases.reduce((a, b) => b.bound > a.bound ? b : a);
|
|
912
1122
|
return {
|
|
913
1123
|
outcome: "cannot decide",
|
|
914
|
-
reason: `${others.length} of ${cases.length} cases are not within${permit}. The bound of the case ${JSON.stringify(top.case)} is ${round(top.bound)}, above the limit ${limit}, and too few floors are above it.`,
|
|
1124
|
+
reason: `${others.length} of ${cases.length} cases are not within${permit}. The bound of the case ${JSON.stringify(top.case)} is ${round$1(top.bound)}, above the limit ${limit}, and too few floors are above it.`,
|
|
915
1125
|
...facts
|
|
916
1126
|
};
|
|
917
1127
|
}
|
|
1128
|
+
function round$1(x) {
|
|
1129
|
+
return Math.round(x * 1e4) / 1e4;
|
|
1130
|
+
}
|
|
1131
|
+
// src/tail.ts
|
|
1132
|
+
/**
|
|
1133
|
+
* The tail limit verdict of a bench with few rounds. The median of each case has one limit, and a high value of its rounds
|
|
1134
|
+
* has another limit. The verdict `limitVerdict` needs an interval of the median, and a few rounds give a wide interval.
|
|
1135
|
+
* This verdict reads each value as it is, and the A/A arm of the same run tells which tail is noise.
|
|
1136
|
+
*/
|
|
1137
|
+
function ratiosOf(records) {
|
|
1138
|
+
const ratios = /* @__PURE__ */ new Map();
|
|
1139
|
+
for (const r of records) {
|
|
1140
|
+
const list = ratios.get(r.case) ?? [];
|
|
1141
|
+
list.push(r.b / r.a);
|
|
1142
|
+
ratios.set(r.case, list);
|
|
1143
|
+
}
|
|
1144
|
+
return ratios;
|
|
1145
|
+
}
|
|
1146
|
+
function checkRank(rank) {
|
|
1147
|
+
if (!Number.isSafeInteger(rank) || rank < 1) throw new RangeError(`tail: the rank is a whole number of 1 or more, not ${rank}.`);
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* This function gives the value at one rank of the ratios of each case, from the largest, in the order of `caseRatios`. It
|
|
1151
|
+
* examines the records as `caseRatios` does. A case needs `2 * rank + 1` rounds, so that the value is above the median.
|
|
1152
|
+
*/
|
|
1153
|
+
function caseTails(records, rank = 2) {
|
|
1154
|
+
checkRank(rank);
|
|
1155
|
+
const ratios = ratiosOf(records);
|
|
1156
|
+
return caseRatios(records).map((c) => {
|
|
1157
|
+
const sorted = (ratios.get(c.case) ?? []).toSorted((x, y) => y - x);
|
|
1158
|
+
return {
|
|
1159
|
+
case: c.case,
|
|
1160
|
+
rounds: c.rounds,
|
|
1161
|
+
tail: c.rounds >= 2 * rank + 1 ? sorted[rank - 1] : void 0
|
|
1162
|
+
};
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
function limitOf(options, name) {
|
|
1166
|
+
const limit = typeof options.limit === "function" ? options.limit(name) : options.limit;
|
|
1167
|
+
for (const value of [limit.median, limit.tail]) if (!(value > 0 && Number.isFinite(value))) throw new RangeError(`tail: each limit is a finite number above 0, not ${value} for the case ${JSON.stringify(name)}.`);
|
|
1168
|
+
return limit;
|
|
1169
|
+
}
|
|
1170
|
+
function largestTail(tails) {
|
|
1171
|
+
const values = tails.flatMap((c) => c.tail === void 0 ? [] : [c.tail]);
|
|
1172
|
+
return values.length === 0 ? void 0 : Math.max(...values);
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* This function gives the verdict of the claim "the median and the tail of each case are at most their limits".
|
|
1176
|
+
* - A case is `beyond` when its median is over its limit. It is also `beyond` when its tail is over its limit and over the
|
|
1177
|
+
* largest tail of the A/A arm of the run.
|
|
1178
|
+
* - A case with too few rounds for the rank is `cannot decide`, unless its median is over its limit.
|
|
1179
|
+
* - A case of `expected` with no record is `cannot decide`.
|
|
1180
|
+
* - The verdict is `beyond` when a case is `beyond`, and `within` when each case is `within`.
|
|
1181
|
+
* - Each case and the verdict are `cannot decide` when the largest tail of the A/A arm is over `aaCap`. The same applies
|
|
1182
|
+
* when the A/A arm has fewer than `minimum` cases with a tail. This is not a pass, so the caller takes another read.
|
|
1183
|
+
*
|
|
1184
|
+
* With the default rank, a real cost in 1 round cannot fail the tail limit, because 1 round cannot be told from noise. A
|
|
1185
|
+
* cost in 2 rounds or more can fail it. The verdict has no interval, so a limit needs a margin from the spread of earlier
|
|
1186
|
+
* runs.
|
|
1187
|
+
*/
|
|
1188
|
+
function tailLimitVerdict(candidate, aa, options) {
|
|
1189
|
+
const { aaCap } = options;
|
|
1190
|
+
const rank = options.rank ?? 2;
|
|
1191
|
+
const minimum = options.minimum ?? 5;
|
|
1192
|
+
checkRank(rank);
|
|
1193
|
+
if (!(aaCap > 0 && Number.isFinite(aaCap))) throw new RangeError(`tailLimitVerdict: the cap is a finite number above 0, not ${aaCap}.`);
|
|
1194
|
+
if (!Number.isSafeInteger(minimum) || minimum < 1) throw new RangeError(`tailLimitVerdict: the minimum is a whole number of 1 or more, not ${minimum}.`);
|
|
1195
|
+
const aaTails = caseTails(aa, rank).filter((c) => c.tail !== void 0);
|
|
1196
|
+
const aaTail = aaTails.length < minimum ? void 0 : largestTail(aaTails);
|
|
1197
|
+
const tails = new Map(caseTails(candidate, rank).map((c) => [c.case, c]));
|
|
1198
|
+
const facts = {
|
|
1199
|
+
aaTail,
|
|
1200
|
+
aaCap,
|
|
1201
|
+
rank
|
|
1202
|
+
};
|
|
1203
|
+
const present = caseRatios(candidate).map((c) => {
|
|
1204
|
+
const { tail } = tails.get(c.case) ?? { tail: void 0 };
|
|
1205
|
+
const limit = limitOf(options, c.case);
|
|
1206
|
+
if (aaTail === void 0 || aaTail > aaCap) return {
|
|
1207
|
+
case: c.case,
|
|
1208
|
+
tail,
|
|
1209
|
+
rounds: c.rounds,
|
|
1210
|
+
ratio: c.ratio,
|
|
1211
|
+
limit,
|
|
1212
|
+
overMedian: false,
|
|
1213
|
+
overTail: false,
|
|
1214
|
+
outcome: "cannot decide"
|
|
1215
|
+
};
|
|
1216
|
+
const overMedian = c.ratio > limit.median;
|
|
1217
|
+
const overTail = tail !== void 0 && tail > limit.tail && tail > aaTail;
|
|
1218
|
+
const outcome = overMedian || overTail ? "beyond" : tail === void 0 ? "cannot decide" : "within";
|
|
1219
|
+
return {
|
|
1220
|
+
case: c.case,
|
|
1221
|
+
tail,
|
|
1222
|
+
rounds: c.rounds,
|
|
1223
|
+
ratio: c.ratio,
|
|
1224
|
+
limit,
|
|
1225
|
+
overMedian,
|
|
1226
|
+
overTail,
|
|
1227
|
+
outcome
|
|
1228
|
+
};
|
|
1229
|
+
});
|
|
1230
|
+
const found = new Set(present.map((c) => c.case));
|
|
1231
|
+
const missing = [...new Set(options.expected ?? [])].filter((name) => !found.has(name)).map((name) => ({
|
|
1232
|
+
case: name,
|
|
1233
|
+
tail: void 0,
|
|
1234
|
+
rounds: 0,
|
|
1235
|
+
ratio: void 0,
|
|
1236
|
+
limit: limitOf(options, name),
|
|
1237
|
+
overMedian: false,
|
|
1238
|
+
overTail: false,
|
|
1239
|
+
outcome: "cannot decide"
|
|
1240
|
+
}));
|
|
1241
|
+
const cases = [...present, ...missing];
|
|
1242
|
+
if (aaTail === void 0) return {
|
|
1243
|
+
...facts,
|
|
1244
|
+
outcome: "cannot decide",
|
|
1245
|
+
cases,
|
|
1246
|
+
reason: `The A/A arm has ${aaTails.length} cases with a tail, and the verdict needs ${minimum} or more.`
|
|
1247
|
+
};
|
|
1248
|
+
if (aaTail > aaCap) return {
|
|
1249
|
+
...facts,
|
|
1250
|
+
outcome: "cannot decide",
|
|
1251
|
+
cases,
|
|
1252
|
+
reason: `The largest tail ${round(aaTail)} of the A/A arm is over the cap ${aaCap}, so this run has no power.`
|
|
1253
|
+
};
|
|
1254
|
+
if (cases.length === 0) return {
|
|
1255
|
+
...facts,
|
|
1256
|
+
outcome: "cannot decide",
|
|
1257
|
+
cases,
|
|
1258
|
+
reason: "The candidate has no cases."
|
|
1259
|
+
};
|
|
1260
|
+
const beyond = cases.filter((c) => c.outcome === "beyond");
|
|
1261
|
+
if (beyond.length > 0) return {
|
|
1262
|
+
...facts,
|
|
1263
|
+
outcome: "beyond",
|
|
1264
|
+
cases,
|
|
1265
|
+
reason: `${beyond.length} of ${cases.length} cases are over a limit. The first is ${JSON.stringify(beyond[0].case)}.`
|
|
1266
|
+
};
|
|
1267
|
+
if (missing.length > 0) return {
|
|
1268
|
+
...facts,
|
|
1269
|
+
outcome: "cannot decide",
|
|
1270
|
+
cases,
|
|
1271
|
+
reason: `${missing.length} of ${cases.length} cases have no record. The first is ${JSON.stringify(missing[0].case)}.`
|
|
1272
|
+
};
|
|
1273
|
+
const open = cases.filter((c) => c.outcome === "cannot decide");
|
|
1274
|
+
if (open.length > 0) return {
|
|
1275
|
+
...facts,
|
|
1276
|
+
outcome: "cannot decide",
|
|
1277
|
+
cases,
|
|
1278
|
+
reason: `${open.length} of ${cases.length} cases have too few rounds for the rank ${rank}. The first is ${JSON.stringify(open[0].case)}.`
|
|
1279
|
+
};
|
|
1280
|
+
return {
|
|
1281
|
+
...facts,
|
|
1282
|
+
outcome: "within",
|
|
1283
|
+
cases,
|
|
1284
|
+
reason: `Each of the ${cases.length} cases is at most its limits, or its tail is at most the tail ${round(aaTail)} of the A/A arm.`
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* This function applies the rule of `tailLimitVerdict` to each case of the A/A arm, and gives the cases that fail. Each
|
|
1289
|
+
* case reads the tail of the other cases, because a case cannot be the noise of itself. An arm with one case gives no
|
|
1290
|
+
* failed case.
|
|
1291
|
+
*
|
|
1292
|
+
* A limit under 1 states a gain, and an A/A pair shows no gain. So the A/A arm has no read of that case, and the case goes
|
|
1293
|
+
* in `gains` with that reason. The tail of that case stays in the noise of the other cases. The false fails of a gain limit
|
|
1294
|
+
* come from repeat reads of an unchanged tree against its fixed base.
|
|
1295
|
+
*/
|
|
1296
|
+
function tailFalseFails(aa, options) {
|
|
1297
|
+
const tails = caseTails(aa, options.rank ?? 2);
|
|
1298
|
+
const ratios = new Map(caseRatios(aa).map((c) => [c.case, c.ratio]));
|
|
1299
|
+
const gains = tails.flatMap((c) => {
|
|
1300
|
+
const limit = limitOf(options, c.case);
|
|
1301
|
+
const under = Math.min(limit.median, limit.tail);
|
|
1302
|
+
return under < 1 ? [{
|
|
1303
|
+
case: c.case,
|
|
1304
|
+
reason: `The limit ${under} is under 1, and an A/A pair shows no gain, so the A/A arm has no read of this case.`
|
|
1305
|
+
}] : [];
|
|
1306
|
+
});
|
|
1307
|
+
const noRead = new Set(gains.map((c) => c.case));
|
|
1308
|
+
return {
|
|
1309
|
+
fails: tails.filter((c) => {
|
|
1310
|
+
if (noRead.has(c.case)) return false;
|
|
1311
|
+
const others = largestTail(tails.filter((other) => other !== c));
|
|
1312
|
+
if (others === void 0) return false;
|
|
1313
|
+
const limit = limitOf(options, c.case);
|
|
1314
|
+
return (ratios.get(c.case) ?? 0) > limit.median || c.tail !== void 0 && c.tail > limit.tail && c.tail > others;
|
|
1315
|
+
}).map((c) => c.case),
|
|
1316
|
+
gains
|
|
1317
|
+
};
|
|
1318
|
+
}
|
|
918
1319
|
function round(x) {
|
|
919
1320
|
return Math.round(x * 1e4) / 1e4;
|
|
920
1321
|
}
|
|
@@ -1660,6 +2061,6 @@ function timeRounds(arms, options) {
|
|
|
1660
2061
|
sink
|
|
1661
2062
|
};
|
|
1662
2063
|
}
|
|
1663
|
-
export { BUDGET_LOOKS, LADDER_FORMAT, aaBand, aaSpread, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, checkLadder, checkMeasures, checkPairedTime, controlFactor, firstArm, ladderVerdict, limitVerdict, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, rungVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower, winClaim };
|
|
2064
|
+
export { BUDGET_LOOKS, GATE_LOOKS, LADDER_FORMAT, aaBand, aaSpread, armVerdicts, balanceOrders, budgetLookSpend, budgetLooksVerdict, budgetVerdict, caseIntervals, caseRatios, caseTails, checkLadder, checkMeasures, checkPairedTime, controlFactor, firstArm, gateOf, ladderVerdict, limitVerdict, notSlowerVerdict, poolRecords, pooledBand, pooledLimitVerdict, pooledVerdict, rungVerdict, tailFalseFails, tailLimitVerdict, timeRounds, timeVerdict, verdictMargin, verdictPower, winClaim };
|
|
1664
2065
|
|
|
1665
2066
|
//# sourceMappingURL=index.js.map
|