@cosyte/bench 0.1.1 → 0.1.2

package/dist/index.d.ts

@@ -59,6 +59,20 @@ interface BenchContractShape {
     readonly GC_MAX_ROUNDS: number;
     /** Fresh processes timed for the cold import, for the target and for an empty module each. */
     readonly IMPORT_RUNS: number;
+    /**
+     * A case whose last three warmup batches each took at least this many milliseconds a pass runs on
+     * the large-case plan: every batch is one pass, so the warmup rule's aggregation cannot happen.
+     */
+    readonly LARGE_PASS_MS: number;
+    /** Passes in each of the two windows whose medians the large-case warmup compares. */
+    readonly LARGE_WARMUP_WINDOW: number;
+    /** Hard cap on the large-case warmup, in milliseconds, from the start of the warmup. */
+    readonly LARGE_WARMUP_MAX_MS: number;
+    /**
+     * The long run also runs on a benchmark's largest size whose rounds fit in this many milliseconds,
+     * a round being at least one pass.
+     */
+    readonly LONG_RUN_MAX_MS: number;
     /** The timing comparison may become a required check when A/A false alarms are under this rate. */
     readonly REQUIRED_FALSE_ALARM_RATE: number;
     /** The confidence that the false alarm rate is under the line, for the rule above. */
@@ -199,16 +213,27 @@ declare const WARMUP: Readonly<{
     STABLE_TOL: number;
     STABLE_BATCHES: number;
 }>;
+/**
+ * Which warmup rule a process ended on. `standard` is ADR 0001 section 2's. `large` is the
+ * large-case plan, for a case whose every batch is a single pass: see {@link warmUp}.
+ */
+type WarmupPlan = "standard" | "large";
 /** What the warmup did. */
 interface WarmupReport {
-    /** Whether three batches in a row came within 5% of their median before 5 s ran out. */
+    /**
+     * Whether the warmup settled: on the standard plan, three batches in a row within 5% of their
+     * median before 5 s ran out; on the large-case plan, two windows of passes with medians within 5%
+     * of each other before 30 s ran out.
+     */
     readonly stable: boolean;
     /** Wall time spent warming, in milliseconds. */
     readonly elapsedMs: number;
     /** Passes over the corpus. */
     readonly passes: number;
-    /** Each batch's milliseconds per pass. */
+    /** Each batch's milliseconds per pass. On the large-case plan every batch after the switch is one pass. */
     readonly batches: readonly number[];
+    /** The rule the warmup ended on. */
+    readonly plan: WarmupPlan;
 }
 /** Per-call latency, or why there is none. */
 type LatencyReport = {
@@ -235,16 +260,32 @@ type LatencyReport = {
  *
  * They read the heap through a {@link HeapProbe}, so their tests can drive them with a scripted heap.
  * The real probe follows ADR 0001 section 8: a zero-argument `gc()` only, settled by rule rather than
- * by a count.
+ * by a count. The measuring process also runs with `--no-concurrent-array-buffer-sweeping`, so a
+ * collection frees the ArrayBuffers it found dead before it ends, and the external memory a
+ * collection reports before and after is what it held and what it kept.
  *
  * @packageDocumentation
  */
 
-/** Why a memory measurement has no number. */
+/**
+ * Why a memory measurement has no number. `allocation-window-gc` is no longer written: a case whose
+ * every window collects is now counted with its collections. It stays here because documents
+ * written before carry it.
+ */
 type MemorySkipReason = "gc-unavailable" | "gc-unsettled" | "allocation-window-gc";
+/**
+ * How allocations were counted. `no-gc-window`: the heap's growth over windows in which no
+ * collection ran. `gc-accounted`: every window collected, so the growth over each window plus what
+ * its collections reclaimed.
+ */
+type AllocationMethod = "no-gc-window" | "gc-accounted";
 /** Allocations per operation. */
 type AllocationReport = {
     readonly status: "measured";
+    /** How they were counted. Absent in a document written before it was recorded: `no-gc-window`. */
+    readonly method?: AllocationMethod;
+    /** Collections that ran in the two windows counted: 0 for `no-gc-window`. */
+    readonly collections?: number;
     /** The smaller of the two windows, in operations. */
     readonly windowOps: number;
     /** Bytes allocated on the V8 heap per operation. */
@@ -255,14 +296,29 @@ type AllocationReport = {
     readonly status: "skipped";
     readonly reason: MemorySkipReason;
 };
-/** Peak heap per call. */
+/** Peak memory per call: the V8 heap, external memory, and the two together. */
 type PeakReport = {
     readonly status: "measured";
     readonly calls: number;
-    /** The highest peak of any call, in bytes above the settled heap before it. */
+    /** The highest V8 heap peak of any call, in bytes above the settled heap before it. */
     readonly peakHeapBytes: number;
-    /** The median call's peak. */
+    /** The median call's V8 heap peak. */
     readonly medianPeakHeapBytes: number;
+    /**
+     * The highest external peak of any call (Buffers, ArrayBuffers), above the external memory
+     * held before it. Absent in a document written before external memory was read.
+     */
+    readonly peakExternalBytes?: number;
+    /** The median call's external peak. */
+    readonly medianPeakExternalBytes?: number;
+    /**
+     * The highest peak of the V8 heap and external memory together, above the two before the call:
+     * the figure a memory budget reads. Not the sum of the two peaks above, which need not come at
+     * one moment.
+     */
+    readonly peakTotalBytes?: number;
+    /** The median call's peak of the two together. */
+    readonly medianPeakTotalBytes?: number;
 } | {
     readonly status: "skipped";
     readonly reason: MemorySkipReason;
@@ -356,6 +412,13 @@ interface CaseComparison extends CaseRef {
     readonly baseSamples: readonly (readonly number[])[];
     /** Each round's head samples, whole. */
     readonly headSamples: readonly (readonly number[])[];
+    /**
+     * Each round's base process: the warmup rule it ended on. Absent in a document written before it
+     * was recorded.
+     */
+    readonly basePlans?: readonly WarmupPlan[];
+    /** Each round's head process: the warmup rule it ended on. */
+    readonly headPlans?: readonly WarmupPlan[];
     /** Each round's `head / base`. */
     readonly ratios: readonly number[];
     readonly medianRatio: number | null;
@@ -474,6 +537,8 @@ type Throughput = {
     readonly processes: readonly {
         readonly samplesMsPerOp: readonly number[];
         readonly medianMsPerOp: number;
+        /** The warmup rule the process ended on. Absent in a document written before it was recorded. */
+        readonly plan?: WarmupPlan;
     }[];
     /** The median of the process medians. */
     readonly medianMsPerOp: number;
@@ -832,6 +897,18 @@ declare const RESULTS_SCHEMA: {
                 readonly medianRatio: {
                     readonly type: readonly ["number", "null"];
                 };
+                readonly basePlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
+                readonly headPlans: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly enum: readonly ["standard", "large"];
+                    };
+                };
                 readonly interval: {
                     readonly $ref: "#/$defs/interval";
                 };

package/dist/index.mjs

@@ -29,6 +29,10 @@ var BENCH_CONTRACT = Object.freeze({
   GC_SETTLE_ROUNDS: 2,
   GC_MAX_ROUNDS: 7,
   IMPORT_RUNS: 20,
+  LARGE_PASS_MS: 50,
+  LARGE_WARMUP_WINDOW: 5,
+  LARGE_WARMUP_MAX_MS: 3e4,
+  LONG_RUN_MAX_MS: 6e4,
   REQUIRED_FALSE_ALARM_RATE: 0.01,
   REQUIRED_CONFIDENCE: 0.95
 });
@@ -151,6 +155,7 @@ var skipped = {
   properties: { status: { const: "skipped" }, reason: str }
 };
 var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var plan = { enum: ["standard", "large"] };
 var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
 var RESULTS_SCHEMA = {
   $schema: "https://json-schema.org/draft/2020-12/schema",
@@ -267,6 +272,9 @@ var RESULTS_SCHEMA = {
         headMsPerOp: nums,
         ratios: nums,
         medianRatio: numOrNull,
+        // Added within version 1, so not required: a document written before them is still valid.
+        basePlans: { type: "array", items: plan },
+        headPlans: { type: "array", items: plan },
         interval: { $ref: "#/$defs/interval" },
         verdict: {
           enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
@@ -396,7 +404,8 @@ var RESULTS_SCHEMA = {
               items: {
                 type: "object",
                 required: ["samplesMsPerOp", "medianMsPerOp"],
-                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+                // `plan` was added within version 1, so it is not required.
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num, plan }
               }
             },
             medianMsPerOp: num,
@@ -423,6 +432,9 @@ var RESULTS_SCHEMA = {
             required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
             properties: {
               status: { const: "measured" },
+              // Added within version 1, so not required.
+              method: { enum: ["no-gc-window", "gc-accounted"] },
+              collections: int,
               windowOps: int,
               heapBytesPerOp: num,
               externalBytesPerOp: num
@@ -437,7 +449,12 @@ var RESULTS_SCHEMA = {
               status: { const: "measured" },
               calls: int,
               peakHeapBytes: num,
-              medianPeakHeapBytes: num
+              medianPeakHeapBytes: num,
+              // Added within version 1, so not required.
+              peakExternalBytes: num,
+              medianPeakExternalBytes: num,
+              peakTotalBytes: num,
+              medianPeakTotalBytes: num
             }
           })
         ),