@cosyte/bench 0.1.2 → 0.1.3

package/dist/index.d.ts

@@ -43,6 +43,15 @@ interface BenchContractShape {
     readonly ALLOC_SEMI_SPACE_MIB: number;
     /** Calls measured one by one for the peak heap. */
     readonly PEAK_CALLS: number;
+    /**
+     * An input under this many bytes, on average, is budgeted on its median call's total peak above
+     * the empty-call floor's median, rather than its highest raw peak. On a GitHub-hosted runner the
+     * peak probe costs about 3.3 KiB on a typical empty call and 28 KiB on its highest (one call, the
+     * first), so the highest of a small input's calls reads the probe more than the code: 20 times a
+     * 1 KiB input is spent before the code runs. At 64 KiB even the probe's highest call is about 2%
+     * of a 20 times budget, so above it the raw highest peak is read as before.
+     */
+    readonly FLOOR_INPUT_BYTES: number;
     /** Rounds in the long run that checks the heap stays flat. */
     readonly LONG_RUN_ROUNDS: number;
     /** Rounds at the start of the long run left out of the slope, while caches and compiled code settle. */
@@ -319,6 +328,17 @@ type PeakReport = {
     readonly peakTotalBytes?: number;
     /** The median call's peak of the two together. */
     readonly medianPeakTotalBytes?: number;
+    /**
+     * Each figure above, less the same figure of the run's empty-call floor (its median), and
+     * never below zero. Absent when the run measured no floor, and in a document written before it.
+     * See {@link aboveFloor}.
+     */
+    readonly peakHeapAboveFloorBytes?: number;
+    readonly medianPeakHeapAboveFloorBytes?: number;
+    readonly peakExternalAboveFloorBytes?: number;
+    readonly medianPeakExternalAboveFloorBytes?: number;
+    readonly peakTotalAboveFloorBytes?: number;
+    readonly medianPeakTotalAboveFloorBytes?: number;
 } | {
     readonly status: "skipped";
     readonly reason: MemorySkipReason;
@@ -549,6 +569,19 @@ type Throughput = {
     readonly status: "skipped";
     readonly reason: string;
 };
+/**
+ * A part of a run that could not be measured: a case's timing or memory, or the empty-call floor.
+ * The message is the first line of what went wrong. Only the first line is kept, because the lines
+ * after it may be a measuring process's stderr, which may quote an input; the command line prints
+ * the whole of it to its own stderr.
+ */
+interface RunFailure {
+    readonly phase: "timing" | "memory" | "floor";
+    /** The case, or `null` for the floor. */
+    readonly benchmark: string | null;
+    readonly size: string | null;
+    readonly message: string;
+}
 /** Everything a reading measured for one case. */
 interface CaseReading extends CaseRef {
     readonly inputsPerPass: number;
@@ -564,9 +597,59 @@ interface ReadingBody {
     /** Fresh processes timed per case. */
     readonly processes: number;
     readonly cases: readonly CaseReading[];
+    /**
+     * The peak of an empty call, measured once per run through the same probe as the cases: what the
+     * probe itself costs. Each case's peak also gives its figures above this one. `null` when memory
+     * was not measured or the floor could not be. Absent in a document written before it.
+     */
+    readonly memoryFloor?: PeakReport | null;
+    /**
+     * What could not be measured. A case whose timing failed is here and not in `cases`; one whose
+     * memory failed is in both, with the memory it could not read as `null`. Every other case is
+     * written as usual. Empty when everything measured; absent in a document written before it.
+     */
+    readonly failures?: readonly RunFailure[];
     /** `null` when import cost was not asked for. */
     readonly importCost: ImportCostReport | null;
 }
+/**
+ * The peak, in bytes, a memory budget reads for a case, or `null` when the case has none. A case
+ * whose inputs average under `FLOOR_INPUT_BYTES` (its bytes a pass over its inputs a pass) reads its
+ * median call's total peak above the run's empty-call floor (`medianPeakTotalAboveFloorBytes`); any
+ * other case, and one whose suite gave no byte count, reads the highest raw total peak (the V8 heap
+ * and external memory together). The probe costs a few KiB on a typical call and tens of KiB on its
+ * highest, which is most of the budget of a small input and nothing the code under test did, and a
+ * rounding error on a large one. A document without a floor reads the raw highest.
+ *
+ * @example
+ * ```ts
+ * import { budgetPeakBytes } from "@cosyte/bench";
+ *
+ * // 300 bytes an input, and a floor whose median call is 3,400 bytes: the highest call read
+ * // 30,000 bytes, most of it the probe, and the median call 6,000 above the floor.
+ * budgetPeakBytes({
+ *   benchmark: "parse",
+ *   size: "300 B",
+ *   inputsPerPass: 1,
+ *   bytesPerPass: 300,
+ *   throughput: { status: "skipped", reason: "warmup-unstable" },
+ *   latency: null,
+ *   allocations: null,
+ *   peak: {
+ *     status: "measured",
+ *     calls: 20,
+ *     peakHeapBytes: 30_000,
+ *     medianPeakHeapBytes: 9_400,
+ *     peakTotalBytes: 30_000,
+ *     medianPeakTotalBytes: 9_400,
+ *     peakTotalAboveFloorBytes: 26_600,
+ *     medianPeakTotalAboveFloorBytes: 6_000,
+ *   },
+ *   longRun: null,
+ * }); // => 6000
+ * ```
+ */
+declare function budgetPeakBytes(c: CaseReading): number | null;
 
 /**
  * The machine a measurement was taken on. Every results document carries it, because a number
@@ -1036,6 +1119,25 @@ declare const RESULTS_SCHEMA: {
                 };
             };
         };
+        readonly peak: Schema;
+        readonly failure: {
+            readonly type: "object";
+            readonly required: readonly ["phase", "benchmark", "size", "message"];
+            readonly properties: {
+                readonly phase: {
+                    readonly enum: readonly ["timing", "memory", "floor"];
+                };
+                readonly benchmark: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly size: {
+                    readonly type: readonly ["string", "null"];
+                };
+                readonly message: {
+                    readonly type: "string";
+                };
+            };
+        };
         readonly reading: {
             readonly type: "object";
             readonly required: readonly ["processes", "cases", "importCost"];
@@ -1050,6 +1152,13 @@ declare const RESULTS_SCHEMA: {
                         readonly $ref: "#/$defs/caseReading";
                     };
                 };
+                readonly memoryFloor: Schema;
+                readonly failures: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly $ref: "#/$defs/failure";
+                    };
+                };
                 readonly importCost: Schema;
             };
         };
@@ -1116,5 +1225,5 @@ declare class BenchError extends Error {
     readonly name = "BenchError";
 }
 
-export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
-export type { AllocationReport, AnyBenchmark, BenchContractShape, BenchCorpus, BenchSize, BenchSuite, Benchmark, BuildRef, CalibrationBody, CaseComparison, CaseReading, CaseRef, ComparisonBody, CountedRate, Environment, ImportCostReport, LatencyReport, LoadedFile, LongRunReport, MemorySkipReason, PeakReport, ReadingBody, ResultsDocument, Runner, Throughput, Verdict, WarmupReport };
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, budgetPeakBytes, defineBenchmark, defineSuite, judge, readResults };
+export type { AllocationReport, AnyBenchmark, BenchContractShape, BenchCorpus, BenchSize, BenchSuite, Benchmark, BuildRef, CalibrationBody, CaseComparison, CaseReading, CaseRef, ComparisonBody, CountedRate, Environment, ImportCostReport, LatencyReport, LoadedFile, LongRunReport, MemorySkipReason, PeakReport, ReadingBody, ResultsDocument, RunFailure, Runner, Throughput, Verdict, WarmupReport };

package/dist/index.mjs

@@ -21,6 +21,7 @@ var BENCH_CONTRACT = Object.freeze({
   ALLOC_WINDOW_MS: 100,
   ALLOC_SEMI_SPACE_MIB: 64,
   PEAK_CALLS: 20,
+  FLOOR_INPUT_BYTES: 64 * 1024,
   LONG_RUN_ROUNDS: 30,
   LONG_RUN_SETTLE_ROUNDS: 5,
   LONG_RUN_ROUND_MS: 100,
@@ -142,6 +143,19 @@ function judge(ratios, threshold = BENCH_CONTRACT.THRESHOLD, alpha = BENCH_CONTR
   else verdict = "inconclusive";
   return { verdict, interval, medianRatio };
 }
+
+// src/reading.ts
+function budgetPeakBytes(c) {
+  const p = c.peak;
+  return p === null || p.status !== "measured" ? null : budgetOf(c, p);
+}
+function budgetOf(c, p) {
+  const raw = p.peakTotalBytes ?? p.peakHeapBytes;
+  if (c.bytesPerPass === null) return raw;
+  const perInput = c.bytesPerPass / c.inputsPerPass;
+  if (perInput >= BENCH_CONTRACT.FLOOR_INPUT_BYTES) return raw;
+  return p.medianPeakTotalAboveFloorBytes ?? raw;
+}
 var SCHEMA_VERSION = 1;
 var num = { type: "number" };
 var int = { type: "integer", minimum: 0 };
@@ -336,12 +350,46 @@ var RESULTS_SCHEMA = {
         }
       }
     },
+    peak: measuredOrSkipped({
+      type: "object",
+      required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+      properties: {
+        status: { const: "measured" },
+        calls: int,
+        peakHeapBytes: num,
+        medianPeakHeapBytes: num,
+        // Added within version 1, so not required.
+        peakExternalBytes: num,
+        medianPeakExternalBytes: num,
+        peakTotalBytes: num,
+        medianPeakTotalBytes: num,
+        peakHeapAboveFloorBytes: num,
+        medianPeakHeapAboveFloorBytes: num,
+        peakExternalAboveFloorBytes: num,
+        medianPeakExternalAboveFloorBytes: num,
+        peakTotalAboveFloorBytes: num,
+        medianPeakTotalAboveFloorBytes: num
+      }
+    }),
+    failure: {
+      type: "object",
+      required: ["phase", "benchmark", "size", "message"],
+      properties: {
+        phase: { enum: ["timing", "memory", "floor"] },
+        benchmark: strOrNull,
+        size: strOrNull,
+        message: str
+      }
+    },
     reading: {
       type: "object",
       required: ["processes", "cases", "importCost"],
       properties: {
         processes: int,
         cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        // Added within version 1, so not required.
+        memoryFloor: orNull({ $ref: "#/$defs/peak" }),
+        failures: { type: "array", items: { $ref: "#/$defs/failure" } },
         importCost: orNull({
           type: "object",
           required: [
@@ -441,23 +489,7 @@ var RESULTS_SCHEMA = {
             }
           })
         ),
-        peak: orNull(
-          measuredOrSkipped({
-            type: "object",
-            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
-            properties: {
-              status: { const: "measured" },
-              calls: int,
-              peakHeapBytes: num,
-              medianPeakHeapBytes: num,
-              // Added within version 1, so not required.
-              peakExternalBytes: num,
-              medianPeakExternalBytes: num,
-              peakTotalBytes: num,
-              medianPeakTotalBytes: num
-            }
-          })
-        ),
+        peak: orNull({ $ref: "#/$defs/peak" }),
         longRun: orNull(
           measuredOrSkipped({
             type: "object",
@@ -567,6 +599,6 @@ Object.freeze({
   importer: fileURLToPath(new URL("./import-entry.mjs", import.meta.url))
 });
 
-export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, budgetPeakBytes, defineBenchmark, defineSuite, judge, readResults };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map