@cosyte/bench 0.0.0-stage → 0.1.2

package/dist/index.d.ts

@@ -0,0 +1,1120 @@
+/**
+ * The benchmark constants, frozen.
+ *
+ * Every value here is fixed by the performance standard,
+ * `documentation/decisions/0005-the-performance-standard.md` in the `config` repo, which says for
+ * each one whether it was measured or chosen, and on what basis. The warmup rule is not here: it
+ * is `PERF_CONTRACT` in `@cosyte/test-utils/perf` (ADR 0001 section 2), which `measure.ts` imports
+ * and the build bundles, so the scaling gate and the benchmarks warm up by one rule kept in one
+ * place.
+ *
+ * They are not tuning knobs and cannot be overridden per package. A comparison's false alarm rate
+ * was measured with these values; change one and the measurement no longer describes the tool.
+ *
+ * @packageDocumentation
+ */
+/** The benchmark constants. See {@link BENCH_CONTRACT}. */
+interface BenchContractShape {
+    /** Timed samples per case per process. Each sample is whole passes over the corpus. */
+    readonly SAMPLES: number;
+    /** Minimum timed length of one sample, in milliseconds. A sample runs whole passes until it is reached. */
+    readonly SAMPLE_MIN_MS: number;
+    /** Rounds in one base against head comparison. Each round runs both sides once, in fresh processes. */
+    readonly ROUNDS: number;
+    /** One-sided error allowed at each end of the comparison's interval on the median ratio. */
+    readonly ALPHA: number;
+    /** The smallest slowdown a comparison will call a regression, as a fraction: 0.03 is 3%. */
+    readonly THRESHOLD: number;
+    /** Time budget for the latency calls of one case, in milliseconds. */
+    readonly LATENCY_BUDGET_MS: number;
+    /** Most latency calls taken for one case. */
+    readonly LATENCY_MAX_CALLS: number;
+    /** Fewest latency calls taken for one case, whatever the budget says. */
+    readonly LATENCY_MIN_CALLS: number;
+    /** A percentile q is reported only when at least this many calls lie above it: n(1 - q) >= 10. */
+    readonly LATENCY_TAIL_CALLS: number;
+    /** The median call must be at least this many timer reads long, or latency is skipped. */
+    readonly LATENCY_TIMER_MULTIPLE: number;
+    /** Most operations in the first allocation window. Halved while a garbage collection lands in a window. */
+    readonly ALLOC_WINDOW_OPS: number;
+    /** The first allocation window is cut to about this many milliseconds of operations, from the warmup's pace. */
+    readonly ALLOC_WINDOW_MS: number;
+    /** The young generation size the allocation process runs with, in MiB, so a window rarely collects. */
+    readonly ALLOC_SEMI_SPACE_MIB: number;
+    /** Calls measured one by one for the peak heap. */
+    readonly PEAK_CALLS: number;
+    /** Rounds in the long run that checks the heap stays flat. */
+    readonly LONG_RUN_ROUNDS: number;
+    /** Rounds at the start of the long run left out of the slope, while caches and compiled code settle. */
+    readonly LONG_RUN_SETTLE_ROUNDS: number;
+    /** Minimum length of one long run round, in milliseconds, fixed as a number of operations by the first. */
+    readonly LONG_RUN_ROUND_MS: number;
+    /** A long run whose fitted growth over the measured rounds reaches this many bytes is reported as growing. */
+    readonly LONG_RUN_GROWTH_BYTES: number;
+    /** A garbage collection round that reclaims less than this, in bytes, counts as settled (ADR 0001 section 8). */
+    readonly GC_SETTLE_BYTES: number;
+    /** Consecutive settled rounds needed (ADR 0001 section 8). */
+    readonly GC_SETTLE_ROUNDS: number;
+    /** Most garbage collection rounds before giving up (ADR 0001 section 8). */
+    readonly GC_MAX_ROUNDS: number;
+    /** Fresh processes timed for the cold import, for the target and for an empty module each. */
+    readonly IMPORT_RUNS: number;
+    /**
+     * A case whose last three warmup batches each took at least this many milliseconds a pass runs on
+     * the large-case plan: every batch is one pass, so the warmup rule's aggregation cannot happen.
+     */
+    readonly LARGE_PASS_MS: number;
+    /** Passes in each of the two windows whose medians the large-case warmup compares. */
+    readonly LARGE_WARMUP_WINDOW: number;
+    /** Hard cap on the large-case warmup, in milliseconds, from the start of the warmup. */
+    readonly LARGE_WARMUP_MAX_MS: number;
+    /**
+     * The long run also runs on a benchmark's largest size whose rounds fit in this many milliseconds,
+     * a round being at least one pass.
+     */
+    readonly LONG_RUN_MAX_MS: number;
+    /** The timing comparison may become a required check when A/A false alarms are under this rate. */
+    readonly REQUIRED_FALSE_ALARM_RATE: number;
+    /** The confidence that the false alarm rate is under the line, for the rule above. */
+    readonly REQUIRED_CONFIDENCE: number;
+}
+/**
+ * The frozen benchmark constants.
+ *
+ * @example
+ * ```ts
+ * import { BENCH_CONTRACT } from "@cosyte/bench";
+ *
+ * BENCH_CONTRACT.ROUNDS; // => 12
+ * BENCH_CONTRACT.THRESHOLD; // => 0.03
+ * ```
+ */
+declare const BENCH_CONTRACT: BenchContractShape;
+
+/**
+ * How a package describes its benchmarks.
+ *
+ * A suite is a module whose default export is {@link defineSuite}'s result. It is handed the module
+ * under test (`bind`) rather than importing it, so one suite file measures the base build and the
+ * head build of a comparison. Every corpus is built in code (`build`): the runner never takes a data
+ * file, which is how its inputs stay synthetic by construction.
+ *
+ * @packageDocumentation
+ */
+/** One corpus: the inputs one pass runs over, built outside every timed region. */
+interface BenchCorpus<TInput> {
+    /** The inputs. One pass calls the measured function once on each. */
+    readonly inputs: readonly TInput[];
+    /** Bytes one pass handles, when it means something. Turns operations a second into bytes a second. */
+    readonly bytes?: number;
+    /** Called once the process has finished with this corpus, for example to remove a temporary directory. */
+    readonly teardown?: () => void;
+}
+/** One input size of a benchmark: small, large, or anything between. */
+interface BenchSize<TInput> {
+    /** How a reader names this size, for example `"1 KiB"`. Unique within its benchmark. */
+    readonly label: string;
+    /** Build the corpus. Called once per process, before anything is timed. */
+    readonly build: () => BenchCorpus<TInput>;
+    /**
+     * Whether the base against head comparison runs this size. Defaults to `true`. Set `false` for a
+     * size whose processes are too slow for a pull request's time budget; a reading still measures it.
+     */
+    readonly compare?: boolean;
+}
+/** One benchmark: a function measured at one or more input sizes. */
+interface Benchmark<TTarget, TInput, TResult> {
+    /** Unique within the suite. */
+    readonly name: string;
+    /** The sizes, smallest first. Each size is measured on its own. */
+    readonly sizes: readonly BenchSize<TInput>[];
+    /** Take the imported module under test and return the function to measure. */
+    readonly bind: (target: TTarget) => (input: TInput) => TResult;
+    /**
+     * Reduce a result to a number summed into a sink the runner checks, so the compiler cannot delete
+     * the measured call. Defaults to 1 for each result that is not `null` or `undefined`; supply it
+     * when the function returns nothing. The sum must grow over a measurement, so return a positive
+     * number for at least some results; a run whose sink did not grow is refused.
+     */
+    readonly weigh?: (result: TResult) => number;
+    /** The size label the long run (is the heap flat?) uses. Defaults to the first size. */
+    readonly longRun?: string;
+}
+/** A benchmark with its input and result types erased, as a suite holds it. */
+type AnyBenchmark<TTarget> = Benchmark<TTarget, unknown, unknown>;
+/** A suite: the benchmarks of one package. */
+interface BenchSuite<TTarget> {
+    /** Shown in every report, for example `"@cosyte/script-utils phi-scan"`. */
+    readonly name: string;
+    /** At least one. */
+    readonly benchmarks: readonly AnyBenchmark<TTarget>[];
+}
+/**
+ * Type one benchmark. Only a type helper: it returns its argument, with the input and result types
+ * erased so benchmarks of different shapes can sit in one suite.
+ *
+ * @example
+ * ```ts
+ * import { defineBenchmark } from "@cosyte/bench";
+ *
+ * const b = defineBenchmark({
+ *   name: "split",
+ *   sizes: [{ label: "1 KiB", build: () => ({ inputs: ["a|b".repeat(341)] }) }],
+ *   bind: () => (s: string) => s.split("|"),
+ * });
+ * b.name; // => "split"
+ * ```
+ */
+declare function defineBenchmark<TTarget, TInput, TResult>(benchmark: Benchmark<TTarget, TInput, TResult>): AnyBenchmark<TTarget>;
+/**
+ * Check a suite and freeze it. Throws a `TypeError` naming the problem: an empty name, no
+ * benchmarks, a repeated name or label, a benchmark with no sizes, or a `longRun` naming no size.
+ *
+ * @example
+ * ```ts
+ * import { defineBenchmark, defineSuite } from "@cosyte/bench";
+ *
+ * export default defineSuite({
+ *   name: "example",
+ *   benchmarks: [
+ *     defineBenchmark({
+ *       name: "split",
+ *       sizes: [{ label: "1 KiB", build: () => ({ inputs: ["a|b".repeat(341)] }) }],
+ *       bind: () => (s: string) => s.split("|"),
+ *     }),
+ *   ],
+ * });
+ * ```
+ */
+declare function defineSuite<TTarget>(suite: BenchSuite<TTarget>): BenchSuite<TTarget>;
+/** One case: a benchmark at one size. */
+interface CaseRef {
+    readonly benchmark: string;
+    readonly size: string;
+}
+
+/**
+ * The timing primitives: the warmup, the throughput samples and the latency calls. They run inside
+ * a measuring process, one case per process, and take their clocks as arguments so their tests can
+ * drive them with exact arithmetic instead of a machine's noise.
+ *
+ * @packageDocumentation
+ */
+/**
+ * The warmup rule, read from `PERF_CONTRACT` in `@cosyte/test-utils/perf` (ADR 0001 section 2) and
+ * renamed, so the scaling gate and the benchmarks warm up by one rule kept in one place.
+ *
+ * @internal
+ */
+declare const WARMUP: Readonly<{
+    MIN_MS: number;
+    BATCH_MS: number;
+    MAX_MS: number;
+    STABLE_TOL: number;
+    STABLE_BATCHES: number;
+}>;
+/**
+ * Which warmup rule a process ended on. `standard` is ADR 0001 section 2's. `large` is the
+ * large-case plan, for a case whose every batch is a single pass: see {@link warmUp}.
+ */
+type WarmupPlan = "standard" | "large";
+/** What the warmup did. */
+interface WarmupReport {
+    /**
+     * Whether the warmup settled: on the standard plan, three batches in a row within 5% of their
+     * median before 5 s ran out; on the large-case plan, two windows of passes with medians within 5%
+     * of each other before 30 s ran out.
+     */
+    readonly stable: boolean;
+    /** Wall time spent warming, in milliseconds. */
+    readonly elapsedMs: number;
+    /** Passes over the corpus. */
+    readonly passes: number;
+    /** Each batch's milliseconds per pass. On the large-case plan every batch after the switch is one pass. */
+    readonly batches: readonly number[];
+    /** The rule the warmup ended on. */
+    readonly plan: WarmupPlan;
+}
+/** Per-call latency, or why there is none. */
+type LatencyReport = {
+    readonly status: "measured";
+    readonly calls: number;
+    readonly timerOverheadNs: number;
+    /** `null` when fewer than `LATENCY_TAIL_CALLS` calls lie above the percentile. */
+    readonly p50Ns: number | null;
+    readonly p95Ns: number | null;
+    readonly p99Ns: number | null;
+    readonly maxNs: number;
+} | {
+    readonly status: "skipped";
+    readonly reason: "latency-below-timer-resolution";
+    readonly calls: number;
+    readonly timerOverheadNs: number;
+    readonly p50Ns: number;
+};
+
+/**
+ * The memory measurements: allocations per operation, peak heap per call, and whether the heap stays
+ * flat over a long run. Each runs in a measuring process started with `--expose-gc`, one case per
+ * process.
+ *
+ * They read the heap through a {@link HeapProbe}, so their tests can drive them with a scripted heap.
+ * The real probe follows ADR 0001 section 8: a zero-argument `gc()` only, settled by rule rather than
+ * by a count. The measuring process also runs with `--no-concurrent-array-buffer-sweeping`, so a
+ * collection frees the ArrayBuffers it found dead before it ends, and the external memory a
+ * collection reports before and after is what it held and what it kept.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Why a memory measurement has no number. `allocation-window-gc` is no longer written: a case whose
+ * every window collects is now counted with its collections. It stays here because documents
+ * written before carry it.
+ */
+type MemorySkipReason = "gc-unavailable" | "gc-unsettled" | "allocation-window-gc";
+/**
+ * How allocations were counted. `no-gc-window`: the heap's growth over windows in which no
+ * collection ran. `gc-accounted`: every window collected, so the growth over each window plus what
+ * its collections reclaimed.
+ */
+type AllocationMethod = "no-gc-window" | "gc-accounted";
+/** Allocations per operation. */
+type AllocationReport = {
+    readonly status: "measured";
+    /** How they were counted. Absent in a document written before it was recorded: `no-gc-window`. */
+    readonly method?: AllocationMethod;
+    /** Collections that ran in the two windows counted: 0 for `no-gc-window`. */
+    readonly collections?: number;
+    /** The smaller of the two windows, in operations. */
+    readonly windowOps: number;
+    /** Bytes allocated on the V8 heap per operation. */
+    readonly heapBytesPerOp: number;
+    /** Bytes allocated outside it (Buffers, ArrayBuffers) per operation. */
+    readonly externalBytesPerOp: number;
+} | {
+    readonly status: "skipped";
+    readonly reason: MemorySkipReason;
+};
+/** Peak memory per call: the V8 heap, external memory, and the two together. */
+type PeakReport = {
+    readonly status: "measured";
+    readonly calls: number;
+    /** The highest V8 heap peak of any call, in bytes above the settled heap before it. */
+    readonly peakHeapBytes: number;
+    /** The median call's V8 heap peak. */
+    readonly medianPeakHeapBytes: number;
+    /**
+     * The highest external peak of any call (Buffers, ArrayBuffers), above the external memory
+     * held before it. Absent in a document written before external memory was read.
+     */
+    readonly peakExternalBytes?: number;
+    /** The median call's external peak. */
+    readonly medianPeakExternalBytes?: number;
+    /**
+     * The highest peak of the V8 heap and external memory together, above the two before the call:
+     * the figure a memory budget reads. Not the sum of the two peaks above, which need not come at
+     * one moment.
+     */
+    readonly peakTotalBytes?: number;
+    /** The median call's peak of the two together. */
+    readonly medianPeakTotalBytes?: number;
+} | {
+    readonly status: "skipped";
+    readonly reason: MemorySkipReason;
+};
+/** Whether the heap stayed flat over a long run. */
+type LongRunReport = {
+    readonly status: "measured";
+    /** `growing` when the fitted growth reaches `LONG_RUN_GROWTH_BYTES`. */
+    readonly verdict: "flat" | "growing";
+    /** Operations in all. */
+    readonly ops: number;
+    /** Operations per round, fixed by the first round. */
+    readonly opsPerRound: number;
+    /** The settled heap after each round, in bytes. */
+    readonly settledHeap: readonly number[];
+    /** Theil-Sen slope of the settled heap against operations, after the settling rounds. */
+    readonly slopeBytesPerOp: number;
+    /** The slope times the operations it was fitted over: the growth the verdict reads. */
+    readonly growthBytes: number;
+    /** The smallest leak, in bytes per operation, this run could call growing: the bound over the span. */
+    readonly detectableBytesPerOp: number;
+} | {
+    readonly status: "skipped";
+    readonly reason: MemorySkipReason;
+};
+
+/** An interval on a median, from the sign test. */
+interface MedianInterval {
+    /** The `rank`-th smallest value. */
+    readonly low: number;
+    /** The `rank`-th largest value. */
+    readonly high: number;
+    /** Which order statistics bound it. */
+    readonly rank: number;
+    /** The chance that each end alone misses the median, exactly: `P(Binomial(n, 1/2) <= rank - 1)`. */
+    readonly tail: number;
+}
+
+/**
+ * The base against head comparison, and its calibration.
+ *
+ * A comparison is `ROUNDS` rounds. In each round every case runs once on the base build and once on
+ * the head build, each in a fresh process, back to back; even rounds run the base first, odd rounds
+ * the head. A drift in the machine over the run lands on both sides of a round, seconds apart, and
+ * cancels in their ratio; an order effect lands on the head in half the rounds and on the base in
+ * the other half, so it widens the spread of the ratios instead of biasing their median.
+ *
+ * Each round gives one ratio per case, `head / base` of the two processes' fastest samples. The
+ * fastest, not the median, because the noise on a shared machine is one-sided (another tenant only
+ * ever slows a sample down), and ADR 0001 section 1 measured `min` as the one estimator that keeps a
+ * stall on one side out of a ratio. The verdict reads the sign test's interval on the median of the
+ * per-round ratios, which assumes nothing about the shape of the noise, only that rounds are
+ * independent.
+ *
+ * @packageDocumentation
+ */
+
+/** What a comparison says about one case. */
+type Verdict = "regression" | "improvement" | "no-regression" | "inconclusive" | "skipped";
+/**
+ * Judge one case from its per-round ratios.
+ *
+ * - `regression`: the whole interval is above `1 + THRESHOLD`. The head is slower by more than the
+ *   threshold, with each end of the interval wrong at most `ALPHA` of the time.
+ * - `improvement`: the whole interval is below `1 / (1 + THRESHOLD)`.
+ * - `no-regression`: the whole interval is at or below `1 + THRESHOLD`: a slowdown larger than
+ *   the threshold is ruled out.
+ * - `inconclusive`: the interval straddles `1 + THRESHOLD`, or there are too few rounds for one.
+ *   Not a pass and not a failure: the rounds disagreed too much to tell.
+ *
+ * @example
+ * ```ts
+ * import { judge } from "@cosyte/bench";
+ *
+ * judge([1.1, 1.12, 1.09, 1.11, 1.1, 1.13, 1.1, 1.08, 1.12, 1.1, 1.11, 1.09]).verdict; // => "regression"
+ * judge([1, 1.01, 0.99, 1, 1.02, 0.98, 1, 1.01, 0.99, 1, 1, 1.01]).verdict; // => "no-regression"
+ * ```
+ */
+declare function judge(ratios: readonly number[], threshold?: number, alpha?: number): {
+    verdict: Exclude<Verdict, "skipped">;
+    interval: MedianInterval | null;
+    medianRatio: number;
+};
+/** One case of a comparison. */
+interface CaseComparison extends CaseRef {
+    /** Each round's base process: the fastest of its samples, milliseconds per operation. */
+    readonly baseMsPerOp: readonly number[];
+    /** Each round's head process: the fastest of its samples, milliseconds per operation. */
+    readonly headMsPerOp: readonly number[];
+    /** Each round's base samples, whole, so another estimator can be judged on the same runs. */
+    readonly baseSamples: readonly (readonly number[])[];
+    /** Each round's head samples, whole. */
+    readonly headSamples: readonly (readonly number[])[];
+    /**
+     * Each round's base process: the warmup rule it ended on. Absent in a document written before it
+     * was recorded.
+     */
+    readonly basePlans?: readonly WarmupPlan[];
+    /** Each round's head process: the warmup rule it ended on. */
+    readonly headPlans?: readonly WarmupPlan[];
+    /** Each round's `head / base`. */
+    readonly ratios: readonly number[];
+    readonly medianRatio: number | null;
+    readonly interval: MedianInterval | null;
+    readonly verdict: Verdict;
+    /** Why the case was skipped, when it was. A skip is not a pass. */
+    readonly skipReason: string | null;
+}
+/** A whole comparison. */
+interface ComparisonBody {
+    readonly rounds: number;
+    readonly threshold: number;
+    readonly alpha: number;
+    /** The slowdown injected on the head side, for calibration. Zero in a real comparison. */
+    readonly inject: number;
+    readonly cases: readonly CaseComparison[];
+}
+/** A rate with its exact upper bound. */
+interface CountedRate {
+    readonly events: number;
+    readonly trials: number;
+    readonly rate: number;
+    /** The one-sided upper bound at `REQUIRED_CONFIDENCE` (Clopper-Pearson). */
+    readonly upperBound: number;
+}
+/** The ratio one case was observed at over a whole calibration. */
+interface ObservedRatio extends CaseRef {
+    /** Measured rounds, pooled over every comparison that measured the case. */
+    readonly rounds: number;
+    /**
+     * The median of those rounds' `head / base` ratios, or `null` when no round measured. Under an
+     * injected slowdown this is the slowdown the head side was actually measured at, which is not
+     * `1 + inject` (see `sampleThroughput`): a detection rate is read against it.
+     */
+    readonly medianRatio: number | null;
+}
+/** A calibration: many comparisons of known outcome. */
+interface CalibrationBody {
+    /** The slowdown injected on the head side. Zero is an A/A calibration: every regression is a false alarm. */
+    readonly inject: number;
+    readonly comparisons: number;
+    /** Each comparison's cases, kept whole so a different rule can be judged against the same runs. */
+    readonly runs: readonly (readonly CaseComparison[])[];
+    /** Case verdicts of `regression`, out of the case verdicts that measured. A skipped case is not a trial. */
+    readonly perCase: CountedRate;
+    /**
+     * Comparisons with at least one `regression`, out of the comparisons whose outcome is known: every
+     * case measured, or a regression was called. A comparison that skipped a case and called no
+     * regression is neither a pass nor a trial.
+     */
+    readonly perComparison: CountedRate;
+    /** Comparisons with at least one skipped case, out of all comparisons. A required check is red on each (exit 2). */
+    readonly skippedComparisons: CountedRate;
+    /** Case verdicts by kind. */
+    readonly verdicts: Readonly<Record<Verdict, number>>;
+    /** Each case's observed ratio, in the order the cases first appear. */
+    readonly observed: readonly ObservedRatio[];
+    /** For an A/A calibration: whether the timing comparison may now be a required check, and why. */
+    readonly requiredRule: {
+        readonly applies: boolean;
+        readonly met: boolean;
+        readonly reason: string;
+    };
+}
+
+/**
+ * Cold import time and the bytes an import loads.
+ *
+ * Each timed import runs in a fresh process with no compile cache, so nothing is compiled before it
+ * starts. The same number of fresh processes import an empty module, interleaved with the target's,
+ * so a reader can see how much of the time is the import machinery rather than the package.
+ *
+ * The bytes are the files Node actually loaded for the import, found by a module hook in one further
+ * process that is not timed (the hook itself costs time). This is what a Node consumer loads; it is
+ * not what a bundler would keep after tree shaking, which depends on what the consumer imports.
+ *
+ * @packageDocumentation
+ */
+
+/** One file an import loaded. */
+interface LoadedFile {
+    /** Relative to the working directory, so no report carries a home directory. */
+    readonly path: string;
+    readonly bytes: number;
+    /** Gzip at level 9: roughly what it costs to download. */
+    readonly gzipBytes: number;
+}
+/** Cold import time and loaded bytes. */
+interface ImportCostReport {
+    readonly runs: number;
+    /** One per fresh process, milliseconds, in the order taken. */
+    readonly coldImportMs: readonly number[];
+    readonly medianMs: number;
+    readonly p95Ms: number;
+    /** The same for an empty module: the import machinery alone. */
+    readonly emptyImportMs: readonly number[];
+    readonly emptyMedianMs: number;
+    readonly files: readonly LoadedFile[];
+    readonly totalBytes: number;
+    readonly totalGzipBytes: number;
+}
+
+/**
+ * A reading: how fast, how much memory and how heavy one build is, on this machine.
+ *
+ * A reading is not a guarantee and travels with its machine: it is what the results repository keeps
+ * over time, compared only within one runner class.
+ *
+ * @packageDocumentation
+ */
+
+/** Throughput of one case. */
+type Throughput = {
+    readonly status: "measured";
+    /** One entry per process: its samples and their median, milliseconds per operation. */
+    readonly processes: readonly {
+        readonly samplesMsPerOp: readonly number[];
+        readonly medianMsPerOp: number;
+        /** The warmup rule the process ended on. Absent in a document written before it was recorded. */
+        readonly plan?: WarmupPlan;
+    }[];
+    /** The median of the process medians. */
+    readonly medianMsPerOp: number;
+    readonly opsPerSecond: number;
+    /** `null` when the suite gave no byte count for the corpus. */
+    readonly bytesPerSecond: number | null;
+} | {
+    readonly status: "skipped";
+    readonly reason: string;
+};
+/** Everything a reading measured for one case. */
+interface CaseReading extends CaseRef {
+    readonly inputsPerPass: number;
+    readonly bytesPerPass: number | null;
+    readonly throughput: Throughput;
+    readonly latency: LatencyReport | null;
+    readonly allocations: AllocationReport | null;
+    readonly peak: PeakReport | null;
+    readonly longRun: LongRunReport | null;
+}
+/** A whole reading. */
+interface ReadingBody {
+    /** Fresh processes timed per case. */
+    readonly processes: number;
+    readonly cases: readonly CaseReading[];
+    /** `null` when import cost was not asked for. */
+    readonly importCost: ImportCostReport | null;
+}
+
+/**
+ * The machine a measurement was taken on. Every results document carries it, because a number
+ * means nothing apart from its machine: ADR 0001 measured a significant shift from a runner image
+ * update with no code change.
+ *
+ * Nothing here names a person: no host name, no user, no absolute path.
+ *
+ * @packageDocumentation
+ */
+/** Where a measurement ran. */
+interface Environment {
+    readonly node: string;
+    readonly v8: string;
+    readonly platform: string;
+    readonly arch: string;
+    readonly osRelease: string;
+    readonly cpuModel: string | null;
+    readonly logicalCpus: number;
+    readonly totalMemoryBytes: number;
+    /** The cgroup CPU quota (`cpu.max`), when Linux reports one: a throttled container measures differently. */
+    readonly cgroupCpuMax: string | null;
+    /** Whether `NODE_OPTIONS` was set. Its value is not recorded; it can hold paths. */
+    readonly nodeOptionsSet: boolean;
+    /**
+     * The 1, 5 and 15 minute load averages when the document was written (zeros where the platform
+     * has none). A busy machine is a noisy one: this is how a reader tells.
+     */
+    readonly loadAverage: readonly number[];
+    readonly runner: Runner;
+}
+/** The CI runner, when there is one. */
+interface Runner {
+    readonly ci: boolean;
+    /** `github-actions`, or `null` off CI. */
+    readonly provider: string | null;
+    /** `github-hosted` or `self-hosted` (`RUNNER_ENVIRONMENT`). */
+    readonly environment: string | null;
+    /** The runner image (`ImageOS`) and its version (`ImageVersion`). */
+    readonly imageOS: string | null;
+    readonly imageVersion: string | null;
+    /**
+     * A key for "the same kind of machine": the provider, image and architecture on CI, or
+     * `local-<platform>-<arch>` off it. Results are only compared within one class.
+     */
+    readonly class: string;
+}
+
+/**
+ * The results document: one JSON file per reading, comparison or calibration, in a versioned shape a
+ * results repository can keep for years and read back.
+ *
+ * The shape is fixed by a JSON Schema ({@link RESULTS_SCHEMA}, also shipped as
+ * `schema/results-v1.schema.json`). Version 1 only grows: a new field may appear, no field changes
+ * meaning or goes away, and readers ignore fields they do not know. Anything else is version 2.
+ *
+ * @packageDocumentation
+ */
+
+/** The version this package writes and reads. */
+declare const SCHEMA_VERSION = 1;
+/** Which build a document describes. */
+interface BuildRef {
+    /** The entry measured, relative to the working directory. */
+    readonly target: string;
+    /** The commit it was built from, when known. */
+    readonly commit: string | null;
+}
+interface DocumentBase {
+    readonly schemaVersion: 1;
+    readonly tool: {
+        readonly name: "@cosyte/bench";
+        readonly version: string;
+    };
+    /** When the document was written, ISO 8601 UTC. */
+    readonly createdAt: string;
+    readonly suite: {
+        readonly name: string;
+        readonly file: string;
+    };
+    readonly environment: Environment;
+    /** The constants in force, so a document can be judged against the method that made it. */
+    readonly contract: {
+        readonly bench: BenchContractShape;
+        readonly warmup: typeof WARMUP;
+    };
+}
+/** A results document. */
+type ResultsDocument = DocumentBase & ({
+    readonly kind: "reading";
+    readonly subject: BuildRef;
+    readonly reading: ReadingBody;
+} | {
+    readonly kind: "comparison";
+    readonly base: BuildRef;
+    readonly head: BuildRef;
+    readonly comparison: ComparisonBody;
+} | {
+    readonly kind: "calibration";
+    readonly base: BuildRef;
+    readonly head: BuildRef;
+    readonly calibration: CalibrationBody;
+});
+/** The JSON Schema keywords the results schema uses. */
+interface Schema {
+    readonly $ref?: string;
+    readonly $defs?: Readonly<Record<string, Schema>>;
+    readonly const?: unknown;
+    readonly enum?: readonly unknown[];
+    readonly type?: string | readonly string[];
+    readonly minimum?: number;
+    readonly required?: readonly string[];
+    readonly properties?: Readonly<Record<string, Schema>>;
+    readonly items?: Schema;
+    readonly oneOf?: readonly Schema[];
+}
+/**
+ * The results schema, version 1, as JSON Schema (draft 2020-12). It holds what every reader relies
+ * on; a document may carry more.
+ *
+ * @example
+ * ```ts
+ * import { RESULTS_SCHEMA } from "@cosyte/bench";
+ *
+ * RESULTS_SCHEMA.$id; // => "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json"
+ * ```
+ */
+declare const RESULTS_SCHEMA: {
+    readonly $schema: "https://json-schema.org/draft/2020-12/schema";
+    readonly $id: "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json";
+    readonly title: "@cosyte/bench results, version 1";
+    readonly type: "object";
+    readonly required: readonly ["schemaVersion", "kind", "tool", "createdAt", "suite", "environment", "contract"];
+    readonly properties: {
+        readonly schemaVersion: {
+            readonly const: 1;
+        };
+        readonly kind: {
+            readonly enum: readonly ["reading", "comparison", "calibration"];
+        };
+        readonly tool: {
+            readonly type: "object";
+            readonly required: readonly ["name", "version"];
+            readonly properties: {
+                readonly name: {
+                    readonly const: "@cosyte/bench";
+                };
+                readonly version: {
+                    readonly type: "string";
+                };
+            };
+        };
+        readonly createdAt: {
+            readonly type: "string";
+        };
+        readonly suite: {
+            readonly type: "object";
+            readonly required: readonly ["name", "file"];
+            readonly properties: {
+                readonly name: {
+                    readonly type: "string";
+                };
+                readonly file: {
+                    readonly type: "string";
+                };
+            };
+        };
+        readonly environment: {
+            readonly $ref: "#/$defs/environment";
+        };
+        readonly contract: {
+            readonly type: "object";
+            readonly required: readonly ["bench", "warmup"];
+            readonly properties: {
+                readonly bench: {
+                    readonly type: "object";
+                };
+                readonly warmup: {
+                    readonly type: "object";
+                };
+            };
+        };
+    };
+    readonly oneOf: readonly [{
+        readonly required: readonly ["subject", "reading"];
+        readonly properties: {
+            readonly kind: {
+                readonly const: "reading";
+            };
+            readonly subject: {
+                readonly $ref: "#/$defs/build";
+            };
+            readonly reading: {
+                readonly $ref: "#/$defs/reading";
+            };
+        };
+    }, {
+        readonly required: readonly ["base", "head", "comparison"];
+        readonly properties: {
+            readonly kind: {
+                readonly const: "comparison";
+            };
+            readonly base: {
+                readonly $ref: "#/$defs/build";
+            };
+            readonly head: {
+                readonly $ref: "#/$defs/build";
+            };
+            readonly comparison: {
+                readonly $ref: "#/$defs/comparison";
+            };
+        };
+    }, {
+        readonly required: readonly ["base", "head", "calibration"];
+        readonly properties: {
+            readonly kind: {
+                readonly const: "calibration";
+            };
+            readonly base: {
+                readonly $ref: "#/$defs/build";
+            };
+            readonly head: {
+                readonly $ref: "#/$defs/build";
+            };
+            readonly calibration: {
+                readonly $ref: "#/$defs/calibration";
+            };
+        };
+    }];
+    readonly $defs: {
+        readonly build: {
+            readonly type: "object";
+            readonly required: readonly ["target", "commit"];
+            readonly properties: {
+                readonly target: {
+                    readonly type: "string";
+                };
+                readonly commit: {
+                    readonly type: readonly ["string", "null"];
+                };
+            };
+        };
+        readonly environment: {
+            readonly type: "object";
+            readonly required: readonly ["node", "v8", "platform", "arch", "osRelease", "cpuModel", "logicalCpus", "totalMemoryBytes", "cgroupCpuMax", "runner"];
+            readonly properties: {
+                readonly node: {
+                    readonly type: "string";
+                };
+                readonly v8: {
+                    readonly type: "string";
+                };
+                readonly platform: {
+                    readonly type: "string";
+                };
+                readonly arch: {
+                    readonly type: "string";
+                };
+                readonly osRelease: {
+                    readonly type: "string";
+                };
+                readonly cpuModel: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly logicalCpus: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly totalMemoryBytes: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly cgroupCpuMax: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly loadAverage: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "number";
+                    };
+                };
+                readonly runner: {
+                    readonly type: "object";
+                    readonly required: readonly ["ci", "provider", "class"];
+                    readonly properties: {
+                        readonly ci: {
+                            readonly type: "boolean";
+                        };
+                        readonly provider: {
+                            readonly type: readonly ["string", "null"];
+                        };
+                        readonly class: {
+                            readonly type: "string";
+                        };
+                    };
+                };
+            };
+        };
+        readonly interval: Schema;
+        readonly caseComparison: {
+            readonly type: "object";
+            readonly required: readonly ["benchmark", "size", "baseMsPerOp", "headMsPerOp", "ratios", "medianRatio", "interval", "verdict", "skipReason"];
+            readonly properties: {
+                readonly benchmark: {
+                    readonly type: "string";
+                };
+                readonly size: {
+                    readonly type: "string";
+                };
+                readonly baseMsPerOp: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "number";
+                    };
+                };
+                readonly headMsPerOp: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "number";
+                    };
+                };
+                readonly ratios: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "number";
+                    };
+                };
+                readonly medianRatio: {
+                    readonly type: readonly ["number", "null"];
+                };
+                readonly basePlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
+                readonly headPlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
+                readonly interval: {
+                    readonly $ref: "#/$defs/interval";
+                };
+                readonly verdict: {
+                    readonly enum: readonly ["regression", "improvement", "no-regression", "inconclusive", "skipped"];
+                };
+                readonly skipReason: {
+                    readonly type: readonly ["string", "null"];
+                };
+            };
+        };
+        readonly comparison: {
+            readonly type: "object";
+            readonly required: readonly ["rounds", "threshold", "alpha", "inject", "cases"];
+            readonly properties: {
+                readonly rounds: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly threshold: {
+                    readonly type: "number";
+                };
+                readonly alpha: {
+                    readonly type: "number";
+                };
+                readonly inject: {
+                    readonly type: "number";
+                };
+                readonly cases: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly $ref: "#/$defs/caseComparison";
+                    };
+                };
+            };
+        };
+        readonly rate: {
+            readonly type: "object";
+            readonly required: readonly ["events", "trials", "rate", "upperBound"];
+            readonly properties: {
+                readonly events: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly trials: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly rate: {
+                    readonly type: "number";
+                };
+                readonly upperBound: {
+                    readonly type: "number";
+                };
+            };
+        };
+        readonly calibration: {
+            readonly type: "object";
+            readonly required: readonly ["inject", "comparisons", "runs", "perCase", "perComparison", "verdicts", "requiredRule"];
+            readonly properties: {
+                readonly inject: {
+                    readonly type: "number";
+                };
+                readonly comparisons: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly runs: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly $ref: "#/$defs/caseComparison";
+                        };
+                    };
+                };
+                readonly perCase: {
+                    readonly $ref: "#/$defs/rate";
+                };
+                readonly perComparison: {
+                    readonly $ref: "#/$defs/rate";
+                };
+                readonly skippedComparisons: {
+                    readonly $ref: "#/$defs/rate";
+                };
+                readonly verdicts: {
+                    readonly type: "object";
+                };
+                readonly observed: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "object";
+                        readonly required: readonly ["benchmark", "size", "rounds", "medianRatio"];
+                        readonly properties: {
+                            readonly benchmark: {
+                                readonly type: "string";
+                            };
+                            readonly size: {
+                                readonly type: "string";
+                            };
+                            readonly rounds: {
+                                readonly type: "integer";
+                                readonly minimum: 0;
+                            };
+                            readonly medianRatio: {
+                                readonly type: readonly ["number", "null"];
+                            };
+                        };
+                    };
+                };
+                readonly requiredRule: {
+                    readonly type: "object";
+                    readonly required: readonly ["applies", "met", "reason"];
+                    readonly properties: {
+                        readonly applies: {
+                            readonly type: "boolean";
+                        };
+                        readonly met: {
+                            readonly type: "boolean";
+                        };
+                        readonly reason: {
+                            readonly type: "string";
+                        };
+                    };
+                };
+            };
+        };
+        readonly reading: {
+            readonly type: "object";
+            readonly required: readonly ["processes", "cases", "importCost"];
+            readonly properties: {
+                readonly processes: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly cases: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly $ref: "#/$defs/caseReading";
+                    };
+                };
+                readonly importCost: Schema;
+            };
+        };
+        readonly caseReading: {
+            readonly type: "object";
+            readonly required: readonly ["benchmark", "size", "inputsPerPass", "bytesPerPass", "throughput", "latency", "allocations", "peak", "longRun"];
+            readonly properties: {
+                readonly benchmark: {
+                    readonly type: "string";
+                };
+                readonly size: {
+                    readonly type: "string";
+                };
+                readonly inputsPerPass: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly bytesPerPass: {
+                    readonly type: readonly ["number", "null"];
+                };
+                readonly throughput: Schema;
+                readonly latency: Schema;
+                readonly allocations: Schema;
+                readonly peak: Schema;
+                readonly longRun: Schema;
+            };
+        };
+    };
+};
+/**
+ * Parse and check a results document. Throws a `TypeError` listing every problem when the text is
+ * not JSON, is a version this package does not read, or does not match the schema.
+ *
+ * @example
+ * ```ts
+ * import { readResults } from "@cosyte/bench";
+ *
+ * try {
+ *   readResults('{"schemaVersion": 2}');
+ * } catch (error) {
+ *   String(error).includes("version 2"); // => true
+ * }
+ * ```
+ */
+declare function readResults(text: string): ResultsDocument;
+
+/**
+ * Starting measuring processes and reading what they measured.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * A run that could not produce a measurement. The command line reports it and exits 2.
+ *
+ * @example
+ * ```ts
+ * import { BenchError } from "@cosyte/bench";
+ *
+ * new BenchError("no result").name; // => "BenchError"
+ * ```
+ */
+declare class BenchError extends Error {
+    readonly name = "BenchError";
+}
+
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
+export type { AllocationReport, AnyBenchmark, BenchContractShape, BenchCorpus, BenchSize, BenchSuite, Benchmark, BuildRef, CalibrationBody, CaseComparison, CaseReading, CaseRef, ComparisonBody, CountedRate, Environment, ImportCostReport, LatencyReport, LoadedFile, LongRunReport, MemorySkipReason, PeakReport, ReadingBody, ResultsDocument, Runner, Throughput, Verdict, WarmupReport };