@cosyte/bench 0.1.2 → 0.1.4

package/dist/cli.mjs

@@ -7,6 +7,7 @@ import 'perf_hooks';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { gzipSync } from 'zlib';
 import { spawn } from 'child_process';
+import 'v8';
 
 function readCgroupCpuMax(path = "/sys/fs/cgroup/cpu.max") {
   try {
@@ -59,6 +60,7 @@ var BENCH_CONTRACT = Object.freeze({
   ALLOC_WINDOW_MS: 100,
   ALLOC_SEMI_SPACE_MIB: 64,
   PEAK_CALLS: 20,
+  FLOOR_INPUT_BYTES: 64 * 1024,
   LONG_RUN_ROUNDS: 30,
   LONG_RUN_SETTLE_ROUNDS: 5,
   LONG_RUN_ROUND_MS: 100,
@@ -115,11 +117,12 @@ function runNode(entry, args, execArgv, timeoutMs, execPath = process.execPath)
 function flagsFor(job) {
   if (job.mode === "timing") return [];
   const memory = ["--expose-gc", "--no-concurrent-array-buffer-sweeping"];
-  if (job.mode === "heap") return memory;
+  if (job.mode === "heap" || job.mode === "floor") return memory;
   const mib = String(BENCH_CONTRACT.ALLOC_SEMI_SPACE_MIB);
   return [...memory, `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
 }
 function describe(job) {
+  if (job.mode === "floor") return "floor job";
   return `${job.mode} job for "${job.case.benchmark} @ ${job.case.size}"`;
 }
 async function runInChild(job, options = {}) {
@@ -554,18 +557,55 @@ async function calibrateComparison(options) {
   }
   return summarizeCalibration(options.inject ?? 0, runs);
 }
+function aboveFloor(peak, floor) {
+  if (peak.status !== "measured" || floor === null || floor.status !== "measured") return peak;
+  const less = (value, base) => value === void 0 || base === void 0 ? void 0 : Math.max(0, value - base);
+  const out = {
+    ...peak,
+    peakHeapAboveFloorBytes: less(peak.peakHeapBytes, floor.medianPeakHeapBytes),
+    medianPeakHeapAboveFloorBytes: less(peak.medianPeakHeapBytes, floor.medianPeakHeapBytes),
+    peakExternalAboveFloorBytes: less(peak.peakExternalBytes, floor.medianPeakExternalBytes),
+    medianPeakExternalAboveFloorBytes: less(
+      peak.medianPeakExternalBytes,
+      floor.medianPeakExternalBytes
+    ),
+    peakTotalAboveFloorBytes: less(peak.peakTotalBytes, floor.medianPeakTotalBytes),
+    medianPeakTotalAboveFloorBytes: less(peak.medianPeakTotalBytes, floor.medianPeakTotalBytes)
+  };
+  return Object.fromEntries(
+    Object.entries(out).filter(([, v]) => v !== void 0)
+  );
+}
 
 // src/reading.ts
+var WrongKind = class extends Error {
+};
 function expectTiming(result) {
   if (result.mode !== "timing")
-    throw new Error("@cosyte/bench: a timing job returned another kind");
+    throw new WrongKind("@cosyte/bench: a timing job returned another kind");
   return result;
 }
 function expectMemory(result) {
-  if (result.mode === "timing")
-    throw new Error("@cosyte/bench: a memory job returned a timing result");
+  if (result.mode === "timing" || result.mode === "floor")
+    throw new WrongKind("@cosyte/bench: a memory job returned another kind");
   return result;
 }
+function expectFloor(result) {
+  if (result.mode !== "floor")
+    throw new WrongKind("@cosyte/bench: a floor job returned another kind");
+  return result.peak;
+}
+function firstLine(error) {
+  const text = error instanceof Error ? error.message : String(error);
+  return text.split("\n")[0];
+}
+function budgetOf(c, p) {
+  const raw = p.peakTotalBytes ?? p.peakHeapBytes;
+  if (c.bytesPerPass === null) return raw;
+  const perInput = c.bytesPerPass / c.inputsPerPass;
+  if (perInput >= BENCH_CONTRACT.FLOOR_INPUT_BYTES) return raw;
+  return p.medianPeakTotalAboveFloorBytes ?? raw;
+}
 function summarizeThroughput(first, rest) {
   const measured = [];
   for (const t of [first, ...rest]) {
@@ -601,37 +641,56 @@ async function readBuild(options) {
   if (!Number.isInteger(processes) || processes < 1) {
     throw new TypeError("@cosyte/bench: processes must be a positive integer");
   }
+  const memory = options.memory ?? true;
   const base = { suite: options.suiteFile, target: options.target };
   const listed = listCases(options.suite, options.only);
   const cases = [];
+  const failures = [];
+  const fail2 = (phase, c, error) => {
+    if (error instanceof WrongKind) throw error;
+    const message = firstLine(error);
+    failures.push({ phase, benchmark: c?.benchmark ?? null, size: c?.size ?? null, message });
+    const what = c === null ? "the memory floor" : `${phase} of ${c.benchmark} @ ${c.size}`;
+    options.onProgress?.(
+      `could not measure ${what}: ${error instanceof Error ? error.message : String(error)}`
+    );
+  };
   for (const name of new Set(listed.map((c) => c.benchmark))) {
     const timed = [];
     for (const c of listed.filter((l) => l.benchmark === name)) {
       const timing = async (latency) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency, inject: 0 }));
-      const first = await timing(true);
-      const rest = [];
-      for (let p = 1; p < processes; p++) rest.push(await timing(false));
-      timed.push({ c, first, throughput: summarizeThroughput(first, rest) });
+      try {
+        const first = await timing(true);
+        const rest = [];
+        for (let p = 1; p < processes; p++) rest.push(await timing(false));
+        timed.push({ c, first, throughput: summarizeThroughput(first, rest) });
+      } catch (error) {
+        fail2("timing", c, error);
+      }
     }
     const largest = timed.filter((t) => longRunFits(t.throughput, t.first.inputsPerPass)).pop();
     for (const { c, first, throughput } of timed) {
       let allocations = null;
       let peak = null;
       let longRun = null;
-      if (options.memory ?? true) {
-        allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
-        const { benchmark } = findCase(options.suite, c);
-        const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
-        const heap = expectMemory(
-          await runJob({
-            mode: "heap",
-            ...base,
-            case: c,
-            longRun: c.size === longRunSize || c.size === largest?.c.size
-          })
-        );
-        peak = heap.peak;
-        longRun = heap.longRun;
+      if (memory) {
+        try {
+          allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
+          const { benchmark } = findCase(options.suite, c);
+          const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
+          const heap = expectMemory(
+            await runJob({
+              mode: "heap",
+              ...base,
+              case: c,
+              longRun: c.size === longRunSize || c.size === largest?.c.size
+            })
+          );
+          peak = heap.peak;
+          longRun = heap.longRun;
+        } catch (error) {
+          fail2("memory", c, error);
+        }
       }
       cases.push({
         ...c,
@@ -646,8 +705,24 @@ async function readBuild(options) {
       options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
     }
   }
+  let memoryFloor = null;
+  if (memory) {
+    try {
+      memoryFloor = expectFloor(await runJob({ mode: "floor" }));
+    } catch (error) {
+      fail2("floor", null, error);
+    }
+  }
   const importCost = options.importCost ?? true ? await options.measureImport(options.target) : null;
-  return { processes, cases, importCost };
+  return {
+    processes,
+    cases: cases.map(
+      (c) => c.peak === null ? c : { ...c, peak: aboveFloor(c.peak, memoryFloor) }
+    ),
+    memoryFloor,
+    failures,
+    importCost
+  };
 }
 
 // src/report.ts
@@ -732,10 +807,19 @@ function renderMarkdown(doc) {
     const r = doc.reading;
     lines.push(...header(doc, "Reading"));
     lines.push(
-      `Throughput is the median of ${String(r.processes)} fresh processes. "Large-case plan" marks a case whose calls each outlast a warmup batch. Latency is per call, nearest rank. Memory: allocations per operation ("collections counted" when every window collected), the peak of one call above what was in use before it (the V8 heap, external memory such as Buffers, and the two together, which a memory budget reads), and whether the settled heap stays flat over a long run.`,
-      "",
-      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Peak external | Peak total | Long run |",
-      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
+      `Throughput is the median of ${String(r.processes)} fresh processes. "Large-case plan" marks a case whose calls each outlast a warmup batch. Latency is per call, nearest rank. Memory: allocations per operation ("collections counted" when every window collected), the peak of one call above what was in use before it (the V8 heap, external memory such as Buffers, and the two together), the total above the empty-call floor, the peak a memory budget reads (the median call's total above the floor for an input under ${bytes(BENCH_CONTRACT.FLOOR_INPUT_BYTES)}, the highest raw total for any other), and whether the settled heap stays flat over a long run.`,
+      ""
+    );
+    const f = r.memoryFloor;
+    if (f !== void 0 && f !== null) {
+      lines.push(
+        f.status === "measured" ? `Empty-call floor, what the peak probe itself costs: ${bytes(f.medianPeakHeapBytes)} heap, ${f.medianPeakTotalBytes === void 0 ? "n/a" : bytes(f.medianPeakTotalBytes)} total (the median of ${String(f.calls)} calls).` : `Empty-call floor: skipped (${f.reason}).`,
+        ""
+      );
+    }
+    lines.push(
+      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Peak external | Peak total | Above floor | Budget reads | Long run |",
+      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
     );
     for (const c of r.cases) {
       const t = c.throughput;
@@ -749,13 +833,30 @@ function renderMarkdown(doc) {
       const alloc = a === null ? "n/a" : a.status === "measured" ? `${bytes(a.heapBytesPerOp + a.externalBytesPerOp)}${a.method === "gc-accounted" ? " (collections counted)" : ""}` : `skipped (${a.reason})`;
       const p = c.peak;
       const optional = (n) => n === void 0 ? "n/a" : bytes(n);
-      const peak = p === null ? ["n/a", "n/a", "n/a"] : p.status === "measured" ? [bytes(p.peakHeapBytes), optional(p.peakExternalBytes), optional(p.peakTotalBytes)] : Array.from({ length: 3 }, () => `skipped (${p.reason})`);
+      const peak = p === null ? ["n/a", "n/a", "n/a", "n/a", "n/a"] : p.status === "measured" ? [
+        bytes(p.peakHeapBytes),
+        optional(p.peakExternalBytes),
+        optional(p.peakTotalBytes),
+        optional(p.peakTotalAboveFloorBytes),
+        bytes(budgetOf(c, p))
+      ] : Array.from({ length: 5 }, () => `skipped (${p.reason})`);
       const lr = c.longRun;
       const long = lr === null ? "" : lr.status === "measured" ? `${lr.verdict} (${bytes(lr.growthBytes)} over ${String(lr.ops)} ops, ${String(lr.slopeBytesPerOp)} B/op)` : `skipped (${lr.reason})`;
       lines.push(
         `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak.join(" | ")} | ${long} |`
       );
     }
+    const failures = r.failures ?? [];
+    if (failures.length > 0) {
+      lines.push(
+        "",
+        `Could not measure ${String(failures.length)} part(s) of the run; every other case above was measured and written:`,
+        "",
+        ...failures.map(
+          (x) => x.benchmark === null ? `- the empty-call floor: ${x.message}` : `- ${x.phase} of ${x.benchmark} @ ${String(x.size)}: ${x.message}`
+        )
+      );
+    }
     if (r.importCost !== null) {
       const i = r.importCost;
       lines.push(
@@ -1000,12 +1101,46 @@ var RESULTS_SCHEMA = {
         }
       }
     },
+    peak: measuredOrSkipped({
+      type: "object",
+      required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+      properties: {
+        status: { const: "measured" },
+        calls: int,
+        peakHeapBytes: num,
+        medianPeakHeapBytes: num,
+        // Added within version 1, so not required.
+        peakExternalBytes: num,
+        medianPeakExternalBytes: num,
+        peakTotalBytes: num,
+        medianPeakTotalBytes: num,
+        peakHeapAboveFloorBytes: num,
+        medianPeakHeapAboveFloorBytes: num,
+        peakExternalAboveFloorBytes: num,
+        medianPeakExternalAboveFloorBytes: num,
+        peakTotalAboveFloorBytes: num,
+        medianPeakTotalAboveFloorBytes: num
+      }
+    }),
+    failure: {
+      type: "object",
+      required: ["phase", "benchmark", "size", "message"],
+      properties: {
+        phase: { enum: ["timing", "memory", "floor"] },
+        benchmark: strOrNull,
+        size: strOrNull,
+        message: str
+      }
+    },
     reading: {
       type: "object",
       required: ["processes", "cases", "importCost"],
       properties: {
         processes: int,
         cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        // Added within version 1, so not required.
+        memoryFloor: orNull({ $ref: "#/$defs/peak" }),
+        failures: { type: "array", items: { $ref: "#/$defs/failure" } },
         importCost: orNull({
           type: "object",
           required: [
@@ -1105,23 +1240,7 @@ var RESULTS_SCHEMA = {
             }
           })
         ),
-        peak: orNull(
-          measuredOrSkipped({
-            type: "object",
-            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
-            properties: {
-              status: { const: "measured" },
-              calls: int,
-              peakHeapBytes: num,
-              medianPeakHeapBytes: num,
-              // Added within version 1, so not required.
-              peakExternalBytes: num,
-              medianPeakExternalBytes: num,
-              peakTotalBytes: num,
-              medianPeakTotalBytes: num
-            }
-          })
-        ),
+        peak: orNull({ $ref: "#/$defs/peak" }),
         longRun: orNull(
           measuredOrSkipped({
             type: "object",
@@ -1235,7 +1354,8 @@ var USAGE = `Usage:
   cosyte-bench check <results.json>
 
 Exit codes: 0 clean, 1 a finding (a regression, or results that do not match the schema),
-2 could not measure or could not read its input.
+2 could not measure or could not read its input. A run that could not measure a case still
+writes every other case, lists what failed, and exits 2.
 `;
 var UsageError = class extends Error {
 };
@@ -1423,7 +1543,8 @@ async function main(argv, io) {
           subject: { target: rel(target), commit },
           reading
         },
-        0
+        // Whatever measured is written; a part that could not be is still exit 2.
+        reading.failures.length > 0 ? 2 : 0
       );
     }
     {