@cosyte/bench 0.1.1 → 0.1.3

package/dist/index.d.ts

@@ -43,6 +43,15 @@ interface BenchContractShape {
     readonly ALLOC_SEMI_SPACE_MIB: number;
     /** Calls measured one by one for the peak heap. */
     readonly PEAK_CALLS: number;
+    /**
+     * An input under this many bytes, on average, is budgeted on its median call's total peak above
+     * the empty-call floor's median, rather than its highest raw peak. On a GitHub-hosted runner the
+     * peak probe costs about 3.3 KiB on a typical empty call and 28 KiB on its highest (one call, the
+     * first), so the highest of a small input's calls reads the probe more than the code: 20 times a
+     * 1 KiB input is spent before the code runs. At 64 KiB even the probe's highest call is about 2%
+     * of a 20 times budget, so above it the raw highest peak is read as before.
+     */
+    readonly FLOOR_INPUT_BYTES: number;
     /** Rounds in the long run that checks the heap stays flat. */
     readonly LONG_RUN_ROUNDS: number;
     /** Rounds at the start of the long run left out of the slope, while caches and compiled code settle. */
@@ -59,6 +68,20 @@ interface BenchContractShape {
     readonly GC_MAX_ROUNDS: number;
     /** Fresh processes timed for the cold import, for the target and for an empty module each. */
     readonly IMPORT_RUNS: number;
+    /**
+     * A case whose last three warmup batches each took at least this many milliseconds a pass runs on
+     * the large-case plan: every batch is one pass, so the warmup rule's aggregation cannot happen.
+     */
+    readonly LARGE_PASS_MS: number;
+    /** Passes in each of the two windows whose medians the large-case warmup compares. */
+    readonly LARGE_WARMUP_WINDOW: number;
+    /** Hard cap on the large-case warmup, in milliseconds, from the start of the warmup. */
+    readonly LARGE_WARMUP_MAX_MS: number;
+    /**
+     * The long run also runs on a benchmark's largest size whose rounds fit in this many milliseconds,
+     * a round being at least one pass.
+     */
+    readonly LONG_RUN_MAX_MS: number;
     /** The timing comparison may become a required check when A/A false alarms are under this rate. */
     readonly REQUIRED_FALSE_ALARM_RATE: number;
     /** The confidence that the false alarm rate is under the line, for the rule above. */
@@ -199,16 +222,27 @@ declare const WARMUP: Readonly<{
     STABLE_TOL: number;
     STABLE_BATCHES: number;
 }>;
+/**
+ * Which warmup rule a process ended on. `standard` is ADR 0001 section 2's. `large` is the
+ * large-case plan, for a case whose every batch is a single pass: see {@link warmUp}.
+ */
+type WarmupPlan = "standard" | "large";
 /** What the warmup did. */
 interface WarmupReport {
-    /** Whether three batches in a row came within 5% of their median before 5 s ran out. */
+    /**
+     * Whether the warmup settled: on the standard plan, three batches in a row within 5% of their
+     * median before 5 s ran out; on the large-case plan, two windows of passes with medians within 5%
+     * of each other before 30 s ran out.
+     */
     readonly stable: boolean;
     /** Wall time spent warming, in milliseconds. */
     readonly elapsedMs: number;
     /** Passes over the corpus. */
     readonly passes: number;
-    /** Each batch's milliseconds per pass. */
+    /** Each batch's milliseconds per pass. On the large-case plan every batch after the switch is one pass. */
     readonly batches: readonly number[];
+    /** The rule the warmup ended on. */
+    readonly plan: WarmupPlan;
 }
 /** Per-call latency, or why there is none. */
 type LatencyReport = {
@@ -235,16 +269,32 @@ type LatencyReport = {
  *
  * They read the heap through a {@link HeapProbe}, so their tests can drive them with a scripted heap.
  * The real probe follows ADR 0001 section 8: a zero-argument `gc()` only, settled by rule rather than
- * by a count.
+ * by a count. The measuring process also runs with `--no-concurrent-array-buffer-sweeping`, so a
+ * collection frees the ArrayBuffers it found dead before it ends, and the external memory a
+ * collection reports before and after is what it held and what it kept.
  *
  * @packageDocumentation
  */
 
-/** Why a memory measurement has no number. */
+/**
+ * Why a memory measurement has no number. `allocation-window-gc` is no longer written: a case whose
+ * every window collects is now counted with its collections. It stays here because documents
+ * written before carry it.
+ */
 type MemorySkipReason = "gc-unavailable" | "gc-unsettled" | "allocation-window-gc";
+/**
+ * How allocations were counted. `no-gc-window`: the heap's growth over windows in which no
+ * collection ran. `gc-accounted`: every window collected, so the growth over each window plus what
+ * its collections reclaimed.
+ */
+type AllocationMethod = "no-gc-window" | "gc-accounted";
 /** Allocations per operation. */
 type AllocationReport = {
     readonly status: "measured";
+    /** How they were counted. Absent in a document written before it was recorded: `no-gc-window`. */
+    readonly method?: AllocationMethod;
+    /** Collections that ran in the two windows counted: 0 for `no-gc-window`. */
+    readonly collections?: number;
     /** The smaller of the two windows, in operations. */
     readonly windowOps: number;
     /** Bytes allocated on the V8 heap per operation. */
@@ -255,14 +305,40 @@ type AllocationReport = {
     readonly status: "skipped";
     readonly reason: MemorySkipReason;
 };
-/** Peak heap per call. */
+/** Peak memory per call: the V8 heap, external memory, and the two together. */
 type PeakReport = {
     readonly status: "measured";
     readonly calls: number;
-    /** The highest peak of any call, in bytes above the settled heap before it. */
+    /** The highest V8 heap peak of any call, in bytes above the settled heap before it. */
     readonly peakHeapBytes: number;
-    /** The median call's peak. */
+    /** The median call's V8 heap peak. */
     readonly medianPeakHeapBytes: number;
+    /**
+     * The highest external peak of any call (Buffers, ArrayBuffers), above the external memory
+     * held before it. Absent in a document written before external memory was read.
+     */
+    readonly peakExternalBytes?: number;
+    /** The median call's external peak. */
+    readonly medianPeakExternalBytes?: number;
+    /**
+     * The highest peak of the V8 heap and external memory together, above the two before the call:
+     * the figure a memory budget reads. Not the sum of the two peaks above, which need not come at
+     * one moment.
+     */
+    readonly peakTotalBytes?: number;
+    /** The median call's peak of the two together. */
+    readonly medianPeakTotalBytes?: number;
+    /**
+     * Each figure above, less the same figure of the run's empty-call floor (its median), and
+     * never below zero. Absent when the run measured no floor, and in a document written before it.
+     * See {@link aboveFloor}.
+     */
+    readonly peakHeapAboveFloorBytes?: number;
+    readonly medianPeakHeapAboveFloorBytes?: number;
+    readonly peakExternalAboveFloorBytes?: number;
+    readonly medianPeakExternalAboveFloorBytes?: number;
+    readonly peakTotalAboveFloorBytes?: number;
+    readonly medianPeakTotalAboveFloorBytes?: number;
 } | {
     readonly status: "skipped";
     readonly reason: MemorySkipReason;
@@ -356,6 +432,13 @@ interface CaseComparison extends CaseRef {
     readonly baseSamples: readonly (readonly number[])[];
     /** Each round's head samples, whole. */
     readonly headSamples: readonly (readonly number[])[];
+    /**
+     * Each round's base process: the warmup rule it ended on. Absent in a document written before it
+     * was recorded.
+     */
+    readonly basePlans?: readonly WarmupPlan[];
+    /** Each round's head process: the warmup rule it ended on. */
+    readonly headPlans?: readonly WarmupPlan[];
     /** Each round's `head / base`. */
     readonly ratios: readonly number[];
     readonly medianRatio: number | null;
@@ -474,6 +557,8 @@ type Throughput = {
     readonly processes: readonly {
         readonly samplesMsPerOp: readonly number[];
         readonly medianMsPerOp: number;
+        /** The warmup rule the process ended on. Absent in a document written before it was recorded. */
+        readonly plan?: WarmupPlan;
     }[];
     /** The median of the process medians. */
     readonly medianMsPerOp: number;
@@ -484,6 +569,19 @@ type Throughput = {
     readonly status: "skipped";
     readonly reason: string;
 };
+/**
+ * A part of a run that could not be measured: a case's timing or memory, or the empty-call floor.
+ * The message is the first line of what went wrong. Only the first line is kept, because the lines
+ * after it may be a measuring process's stderr, which may quote an input; the command line prints
+ * the whole of it to its own stderr.
+ */
+interface RunFailure {
+    readonly phase: "timing" | "memory" | "floor";
+    /** The case, or `null` for the floor. */
+    readonly benchmark: string | null;
+    readonly size: string | null;
+    readonly message: string;
+}
 /** Everything a reading measured for one case. */
 interface CaseReading extends CaseRef {
     readonly inputsPerPass: number;
@@ -499,9 +597,59 @@ interface ReadingBody {
     /** Fresh processes timed per case. */
     readonly processes: number;
     readonly cases: readonly CaseReading[];
+    /**
+     * The peak of an empty call, measured once per run through the same probe as the cases: what the
+     * probe itself costs. Each case's peak also gives its figures above this one. `null` when memory
+     * was not measured or the floor could not be. Absent in a document written before it.
+     */
+    readonly memoryFloor?: PeakReport | null;
+    /**
+     * What could not be measured. A case whose timing failed is here and not in `cases`; one whose
+     * memory failed is in both, with the memory it could not read as `null`. Every other case is
+     * written as usual. Empty when everything measured; absent in a document written before it.
+     */
+    readonly failures?: readonly RunFailure[];
     /** `null` when import cost was not asked for. */
     readonly importCost: ImportCostReport | null;
 }
+/**
+ * The peak, in bytes, a memory budget reads for a case, or `null` when the case has none. A case
+ * whose inputs average under `FLOOR_INPUT_BYTES` (its bytes a pass over its inputs a pass) reads its
+ * median call's total peak above the run's empty-call floor (`medianPeakTotalAboveFloorBytes`); any
+ * other case, and one whose suite gave no byte count, reads the highest raw total peak (the V8 heap
+ * and external memory together). The probe costs a few KiB on a typical call and tens of KiB on its
+ * highest, which is most of the budget of a small input and nothing the code under test did, and a
+ * rounding error on a large one. A document without a floor reads the raw highest.
+ *
+ * @example
+ * ```ts
+ * import { budgetPeakBytes } from "@cosyte/bench";
+ *
+ * // 300 bytes an input, and a floor whose median call is 3,400 bytes: the highest call read
+ * // 30,000 bytes, most of it the probe, and the median call 6,000 above the floor.
+ * budgetPeakBytes({
+ *   benchmark: "parse",
+ *   size: "300 B",
+ *   inputsPerPass: 1,
+ *   bytesPerPass: 300,
+ *   throughput: { status: "skipped", reason: "warmup-unstable" },
+ *   latency: null,
+ *   allocations: null,
+ *   peak: {
+ *     status: "measured",
+ *     calls: 20,
+ *     peakHeapBytes: 30_000,
+ *     medianPeakHeapBytes: 9_400,
+ *     peakTotalBytes: 30_000,
+ *     medianPeakTotalBytes: 9_400,
+ *     peakTotalAboveFloorBytes: 26_600,
+ *     medianPeakTotalAboveFloorBytes: 6_000,
+ *   },
+ *   longRun: null,
+ * }); // => 6000
+ * ```
+ */
+declare function budgetPeakBytes(c: CaseReading): number | null;
 
 /**
  * The machine a measurement was taken on. Every results document carries it, because a number
@@ -832,6 +980,18 @@ declare const RESULTS_SCHEMA: {
                 readonly medianRatio: {
                     readonly type: readonly ["number", "null"];
                 };
+                readonly basePlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
+                readonly headPlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
                 readonly interval: {
                     readonly $ref: "#/$defs/interval";
                 };
@@ -959,6 +1119,25 @@ declare const RESULTS_SCHEMA: {
                 };
             };
         };
+        readonly peak: Schema;
+        readonly failure: {
+            readonly type: "object";
+            readonly required: readonly ["phase", "benchmark", "size", "message"];
+            readonly properties: {
+                readonly phase: {
+                    readonly enum: readonly ["timing", "memory", "floor"];
+                };
+                readonly benchmark: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly size: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly message: {
+                    readonly type: "string";
+                };
+            };
+        };
         readonly reading: {
             readonly type: "object";
             readonly required: readonly ["processes", "cases", "importCost"];
@@ -973,6 +1152,13 @@ declare const RESULTS_SCHEMA: {
                         readonly $ref: "#/$defs/caseReading";
                     };
                 };
+                readonly memoryFloor: Schema;
+                readonly failures: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly $ref: "#/$defs/failure";
+                    };
+                };
                 readonly importCost: Schema;
             };
         };
@@ -1039,5 +1225,5 @@ declare class BenchError extends Error {
     readonly name = "BenchError";
 }
 
-export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
-export type { AllocationReport, AnyBenchmark, BenchContractShape, BenchCorpus, BenchSize, BenchSuite, Benchmark, BuildRef, CalibrationBody, CaseComparison, CaseReading, CaseRef, ComparisonBody, CountedRate, Environment, ImportCostReport, LatencyReport, LoadedFile, LongRunReport, MemorySkipReason, PeakReport, ReadingBody, ResultsDocument, Runner, Throughput, Verdict, WarmupReport };
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, budgetPeakBytes, defineBenchmark, defineSuite, judge, readResults };
+export type { AllocationReport, AnyBenchmark, BenchContractShape, BenchCorpus, BenchSize, BenchSuite, Benchmark, BuildRef, CalibrationBody, CaseComparison, CaseReading, CaseRef, ComparisonBody, CountedRate, Environment, ImportCostReport, LatencyReport, LoadedFile, LongRunReport, MemorySkipReason, PeakReport, ReadingBody, ResultsDocument, RunFailure, Runner, Throughput, Verdict, WarmupReport };

package/dist/index.mjs

@@ -21,6 +21,7 @@ var BENCH_CONTRACT = Object.freeze({
   ALLOC_WINDOW_MS: 100,
   ALLOC_SEMI_SPACE_MIB: 64,
   PEAK_CALLS: 20,
+  FLOOR_INPUT_BYTES: 64 * 1024,
   LONG_RUN_ROUNDS: 30,
   LONG_RUN_SETTLE_ROUNDS: 5,
   LONG_RUN_ROUND_MS: 100,
@@ -29,6 +30,10 @@ var BENCH_CONTRACT = Object.freeze({
   GC_SETTLE_ROUNDS: 2,
   GC_MAX_ROUNDS: 7,
   IMPORT_RUNS: 20,
+  LARGE_PASS_MS: 50,
+  LARGE_WARMUP_WINDOW: 5,
+  LARGE_WARMUP_MAX_MS: 3e4,
+  LONG_RUN_MAX_MS: 6e4,
   REQUIRED_FALSE_ALARM_RATE: 0.01,
   REQUIRED_CONFIDENCE: 0.95
 });
@@ -138,6 +143,19 @@ function judge(ratios, threshold = BENCH_CONTRACT.THRESHOLD, alpha = BENCH_CONTR
   else verdict = "inconclusive";
   return { verdict, interval, medianRatio };
 }
+
+// src/reading.ts
+function budgetPeakBytes(c) {
+  const p = c.peak;
+  return p === null || p.status !== "measured" ? null : budgetOf(c, p);
+}
+function budgetOf(c, p) {
+  const raw = p.peakTotalBytes ?? p.peakHeapBytes;
+  if (c.bytesPerPass === null) return raw;
+  const perInput = c.bytesPerPass / c.inputsPerPass;
+  if (perInput >= BENCH_CONTRACT.FLOOR_INPUT_BYTES) return raw;
+  return p.medianPeakTotalAboveFloorBytes ?? raw;
+}
 var SCHEMA_VERSION = 1;
 var num = { type: "number" };
 var int = { type: "integer", minimum: 0 };
@@ -151,6 +169,7 @@ var skipped = {
   properties: { status: { const: "skipped" }, reason: str }
 };
 var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var plan = { enum: ["standard", "large"] };
 var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
 var RESULTS_SCHEMA = {
   $schema: "https://json-schema.org/draft/2020-12/schema",
@@ -267,6 +286,9 @@ var RESULTS_SCHEMA = {
         headMsPerOp: nums,
         ratios: nums,
         medianRatio: numOrNull,
+        // Added within version 1, so not required: a document written before them is still valid.
+        basePlans: { type: "array", items: plan },
+        headPlans: { type: "array", items: plan },
         interval: { $ref: "#/$defs/interval" },
         verdict: {
           enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
@@ -328,12 +350,46 @@ var RESULTS_SCHEMA = {
         }
       }
     },
+    peak: measuredOrSkipped({
+      type: "object",
+      required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+      properties: {
+        status: { const: "measured" },
+        calls: int,
+        peakHeapBytes: num,
+        medianPeakHeapBytes: num,
+        // Added within version 1, so not required.
+        peakExternalBytes: num,
+        medianPeakExternalBytes: num,
+        peakTotalBytes: num,
+        medianPeakTotalBytes: num,
+        peakHeapAboveFloorBytes: num,
+        medianPeakHeapAboveFloorBytes: num,
+        peakExternalAboveFloorBytes: num,
+        medianPeakExternalAboveFloorBytes: num,
+        peakTotalAboveFloorBytes: num,
+        medianPeakTotalAboveFloorBytes: num
+      }
+    }),
+    failure: {
+      type: "object",
+      required: ["phase", "benchmark", "size", "message"],
+      properties: {
+        phase: { enum: ["timing", "memory", "floor"] },
+        benchmark: strOrNull,
+        size: strOrNull,
+        message: str
+      }
+    },
     reading: {
       type: "object",
       required: ["processes", "cases", "importCost"],
       properties: {
         processes: int,
         cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        // Added within version 1, so not required.
+        memoryFloor: orNull({ $ref: "#/$defs/peak" }),
+        failures: { type: "array", items: { $ref: "#/$defs/failure" } },
         importCost: orNull({
           type: "object",
           required: [
@@ -396,7 +452,8 @@ var RESULTS_SCHEMA = {
               items: {
                 type: "object",
                 required: ["samplesMsPerOp", "medianMsPerOp"],
-                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+                // `plan` was added within version 1, so it is not required.
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num, plan }
               }
             },
             medianMsPerOp: num,
@@ -423,24 +480,16 @@ var RESULTS_SCHEMA = {
             required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
             properties: {
               status: { const: "measured" },
+              // Added within version 1, so not required.
+              method: { enum: ["no-gc-window", "gc-accounted"] },
+              collections: int,
               windowOps: int,
               heapBytesPerOp: num,
               externalBytesPerOp: num
             }
           })
         ),
-        peak: orNull(
-          measuredOrSkipped({
-            type: "object",
-            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
-            properties: {
-              status: { const: "measured" },
-              calls: int,
-              peakHeapBytes: num,
-              medianPeakHeapBytes: num
-            }
-          })
-        ),
+        peak: orNull({ $ref: "#/$defs/peak" }),
         longRun: orNull(
           measuredOrSkipped({
             type: "object",
@@ -550,6 +599,6 @@ Object.freeze({
   importer: fileURLToPath(new URL("./import-entry.mjs", import.meta.url))
 });
 
-export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, budgetPeakBytes, defineBenchmark, defineSuite, judge, readResults };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map